Device node: a device's ports are wired in flows

- flow/nodes/device.py: `device` node with params `device` and `config`;
  provides the device's `in` ports, requires its `out` ports without
  waking anything, never run (Node.passive), down "Offline" from presence
- DeviceDef.io and DevicePort go; GET /devices/{id} computes `io` and
  `bindings` from the device nodes of published flows, in a fixed order,
  cached by FlowStore.revision; the credential's scope comes from them
- issues `unknown_device` and `device_port` (wired twice; against a report
  the wrong direction or dtype, or unlisted as advisory), rechecked when a
  device is created or deleted; ValidationIssue.advisory is a field
- a device publishing on a bound port is credited to its node by the
  server; the panel endpoint kind is gone from `_endpoints` and the canvas
- a rebuild that moves a device's bindings sends `dashboard_changed` with
  an empty name, which the firmware refetches on
- startup moves each device's `io` rows into device nodes in the flows
  declaring their messages, draft and published, one commit; rows no
  flow declares are logged and left out
- frontend: Device node settings (device, Publishes, Follows, a port
  picker from a report), palette shortcuts per reported port, DeviceDetail
  lists bindings with "Use in flow" in place of the I/O editor
- docs: devices, flow editor, node types, issues, API

backend/tests/flow/test_device_node.py

@@ -0,0 +1,94 @@
+"""A device node: never run by the engine, down while its device is away."""
+
+import asyncio
+from pathlib import Path
+
+from fluksio.flow.controller import FlowController
+from fluksio.flow.devices import presence
+from fluksio.flow.messages import DType, MessageSpec
+from fluksio.flow.schemas import FlowDef, FlowInput, NodeDef
+from fluksio.flow.store import FlowStore
+
+
+def _controller(tmp_path: Path) -> FlowController:
+    store = FlowStore(tmp_path / "flows")
+    store.write_flow(
+        FlowDef(
+            name="hall",
+            inputs=[
+                FlowInput(spec=MessageSpec(name="level", dtype=DType.INT), initial=50)
+            ],
+            nodes=[
+                NodeDef(
+                    id="panel",
+                    type="device",
+                    params={"device": "wall"},
+                    provides=[
+                        MessageSpec(name="bell", port="gpio.4", dtype=DType.BOOL)
+                    ],
+                    requires=[
+                        MessageSpec(
+                            name="level",
+                            port="backlight",
+                            dtype=DType.INT,
+                            trigger=False,
+                        )
+                    ],
+                ),
+                NodeDef(
+                    id="ring",
+                    type="change",
+                    requires=[MessageSpec(name="bell", dtype=DType.BOOL)],
+                    provides=[MessageSpec(name="rung", dtype=DType.BOOL)],
+                ),
+            ],
+        )
+    )
+    return FlowController(store)
+
+
+def test_the_engine_never_runs_a_device_node(tmp_path: Path) -> None:
+    controller = _controller(tmp_path)
+    asyncio.run(controller.reload())
+    assert controller.issues == []
+    node = controller.get_node("hall.panel")
+    assert node is not None and node.passive
+    ran: list[object] = []
+    node.f = lambda **kwargs: ran.append(kwargs)  # type: ignore[method-assign]
+
+    # The message its out port follows changes, and a manual run of the flow.
+    controller.publish_message("hall.level", 80)
+    controller.run_flow("hall")
+    assert ran == []
+
+    # What the device publishes on its in port runs what reads it.
+    controller.publish_message("hall.bell", True)
+    assert controller.values("hall")["hall.rung"]["value"] is True
+    asyncio.run(controller._teardown())
+
+
+def test_a_device_node_is_down_while_its_device_is_away(tmp_path: Path) -> None:
+    controller = _controller(tmp_path)
+
+    async def scenario() -> list[tuple[str, str | None]]:
+        await controller.reload()
+        entry = controller.loaded["hall.panel"]
+        seen = [(entry.health, entry.health_detail)]
+        presence.opened("wall")
+        seen.append((entry.health, entry.health_detail))
+        presence.closed("wall")
+        seen.append((entry.health, entry.health_detail))
+        await controller._teardown()
+        # Stopped, it no longer follows the device.
+        presence.opened("wall")
+        seen.append((entry.health, entry.health_detail))
+        presence.closed("wall")
+        return seen
+
+    assert asyncio.run(scenario()) == [
+        ("down", "Offline"),
+        ("ok", None),
+        ("down", "Offline"),
+        ("down", "Offline"),
+    ]
+    presence.forget("wall")

backend/tests/flow/test_device_store.py

@@ -56,3 +56,186 @@ def test_panels_move_into_devices_once(tmp_path: Path) -> None:
             text=True,
         ).stdout
     )
+
+
+def _flow(flows: FlowStore, name: str, draft: dict | None = None, **published) -> None:
+    """Write a published flow document, and a draft beside it if given."""
+    folder = flows.root / name
+    folder.mkdir(parents=True, exist_ok=True)
+    (folder / "flow.json").write_text(json.dumps({"name": name, **published}))
+    if draft is not None:
+        (folder / "flow.draft.json").write_text(json.dumps({"name": name, **draft}))
+
+
+def test_device_ports_move_into_flows_once(tmp_path: Path) -> None:
+    flows = FlowStore(tmp_path / "flows")
+    _flow(
+        flows,
+        "home",
+        version=3,
+        inputs=[
+            {"spec": {"name": "brightness", "dtype": "int"}, "initial": 80},
+        ],
+        nodes=[
+            {
+                "id": "bell",
+                "type": "python",
+                "requires": [{"name": "doorbell", "dtype": "bool"}],
+            }
+        ],
+        draft={
+            "version": 5,
+            "inputs": [{"spec": {"name": "brightness", "dtype": "int"}, "initial": 80}],
+            "nodes": [
+                {
+                    "id": "bell",
+                    "type": "python",
+                    "requires": [{"name": "doorbell", "dtype": "bool"}],
+                },
+                {"id": "hall", "type": "inject"},
+            ],
+        },
+    )
+    # Declares `home.light` from outside: a row on it goes to the owner first,
+    # and only here when the owner does not declare it.
+    _flow(
+        flows,
+        "lights",
+        nodes=[
+            {
+                "id": "dim",
+                "type": "python",
+                "provides": [{"name": "home.light", "dtype": "float"}],
+            }
+        ],
+    )
+    devices = flows.root / "_devices"
+    devices.mkdir()
+    (devices / "hall.json").write_text(
+        json.dumps(
+            {
+                "id": "hall",
+                "title": "Hall",
+                "version": 4,
+                "io": [
+                    {
+                        "port": "backlight",
+                        "message": "home.brightness",
+                        "direction": "out",
+                    },
+                    {
+                        "port": "gpio.4",
+                        "message": "home.doorbell",
+                        "direction": "in",
+                        "config": {"pull": "up"},
+                    },
+                    {"port": "dimmer", "message": "home.light", "direction": "out"},
+                    {"port": "mic", "message": "home.nowhere", "direction": "in"},
+                ],
+            }
+        )
+    )
+    store = DeviceStore(flows, tmp_path / "devices.json")
+    store.move_io()
+
+    published = flows.read_flow("home")
+    draft = flows.read_flow("home", draft=True)
+    assert (published.version, draft.version) == (4, 6)
+    for doc in (published, draft):
+        # `hall` is taken by an inject node in the draft, so the id moves on in
+        # both documents alike.
+        node = next(n for n in doc.nodes if n.type == "device")
+        assert node.id == "hall_2" and node.title == "Hall"
+        assert node.params == {"device": "hall", "config": {"gpio.4": {"pull": "up"}}}
+        assert [(s.port, s.name, s.dtype.value) for s in node.provides] == [
+            ("gpio.4", "doorbell", "bool")
+        ]
+        assert [(s.port, s.name, s.dtype.value, s.trigger) for s in node.requires] == [
+            ("backlight", "brightness", "int", False)
+        ]
+    lights = flows.read_flow("lights")
+    node = next(n for n in lights.nodes if n.type == "device")
+    assert node.id == "hall" and lights.version == 2
+    assert [(s.port, s.name) for s in node.requires] == [("dimmer", "home.light")]
+
+    # The device lost `io` and kept its version; the row nothing declares is
+    # gone. What it reads back is the same rows, in port order.
+    assert "io" not in json.loads((devices / "hall.json").read_text())
+    assert store.find("hall").version == 4  # type: ignore[union-attr]
+    assert [(r.port, r.message, r.direction, r.config) for r in store.io("hall")] == [
+        ("backlight", "home.brightness", "out", {}),
+        ("dimmer", "home.light", "out", {}),
+        ("gpio.4", "home.doorbell", "in", {"pull": "up"}),
+    ]
+    log = subprocess.run(
+        ["git", "-C", str(flows.root), "log", "--format=%s"],
+        capture_output=True,
+        text=True,
+    ).stdout.splitlines()
+    assert log[0] == "Move device ports into flows"
+
+    # Nothing left to move, and nothing written twice.
+    store.move_io()
+    assert flows.read_flow("home").version == 4
+    assert (
+        subprocess.run(
+            ["git", "-C", str(flows.root), "log", "--format=%s"],
+            capture_output=True,
+            text=True,
+        ).stdout.splitlines()
+        == log
+    )
+
+
+def test_a_crash_after_the_flows_were_written_adds_nothing_twice(
+    tmp_path: Path,
+) -> None:
+    flows = FlowStore(tmp_path / "flows")
+    _flow(flows, "home", inputs=[{"spec": {"name": "level", "dtype": "int"}}])
+    devices = flows.root / "_devices"
+    devices.mkdir()
+    device = {
+        "id": "hall",
+        "io": [{"port": "backlight", "message": "home.level", "direction": "out"}],
+    }
+    (devices / "hall.json").write_text(json.dumps(device))
+    store = DeviceStore(flows, tmp_path / "devices.json")
+    store.move_io()
+    # As if the device file had not been rewritten before the process died.
+    (devices / "hall.json").write_text(json.dumps(device))
+    store.move_io()
+    node = next(n for n in flows.read_flow("home").nodes if n.type == "device")
+    assert len(node.requires) == 1
+    assert flows.read_flow("home").version == 2
+
+
+def test_panels_with_io_reach_the_flows_in_one_start(tmp_path: Path) -> None:
+    """An instance from before devices: panels.json, then device nodes."""
+    flows = FlowStore(tmp_path / "flows")
+    _flow(flows, "kiosk", inputs=[{"spec": {"name": "brightness", "dtype": "int"}}])
+    panels = tmp_path / "panels.json"
+    panels.write_text(
+        json.dumps(
+            {
+                "panels": [
+                    {
+                        "id": "knob",
+                        "io": [
+                            {
+                                "port": "backlight",
+                                "message": "kiosk.brightness",
+                                "direction": "out",
+                                "config": {"wake_s": 30},
+                            }
+                        ],
+                    }
+                ]
+            }
+        )
+    )
+    store = DeviceStore(flows, tmp_path / "devices.json")
+    store.migrate(panels)
+    store.move_io()
+    assert [(r.port, r.message, r.config) for r in store.io("knob")] == [
+        ("backlight", "kiosk.brightness", {"wake_s": 30})
+    ]

backend/tests/flow/test_endpoints.py

@@ -11,7 +11,6 @@ import pytest
 
 from fluksio.api.routes.flows import _endpoints
 from fluksio.flow.controller import FlowController
-from fluksio.flow.devices import DeviceDef, DevicePort, DeviceStore
 from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.schemas import FlowDef, NodeDef
 from fluksio.flow.store import FlowStore
@@ -110,29 +109,3 @@ def test_the_wiring_index_is_read_once_per_store_revision(controller):
     controller.store.write_flow(FlowDef(name="shed", nodes=[]))
     _endpoints(controller, controller.store.read_flow("house"))
     assert reads == 2
-
-
-def test_a_device_port_is_drawn_on_the_side_it_wires(controller, tmp_path):
-    controller.devices = DeviceStore(controller.store, tmp_path / "devices.json")
-    controller.devices.create(
-        DeviceDef(
-            id="hall",
-            title="Hall",
-            io=[
-                DevicePort(port="gpio.4", message="house.temp", direction="in"),
-                DevicePort(port="backlight", message="house.temp", direction="out"),
-                DevicePort(port="mic", message="garage.open", direction="in"),
-            ],
-        )
-    )
-
-    endpoints = {
-        one.id: one
-        for one in _endpoints(controller, controller.store.read_flow("house"))
-    }
-
-    # The id is what the device publishes under, so its edges pulse for it.
-    assert endpoints["panel:hall:gpio.4"].provides == ["house.temp"]
-    assert endpoints["panel:hall:backlight"].requires == ["house.temp"]
-    assert endpoints["panel:hall:backlight"].detail == "Hall"
-    assert "panel:hall:mic" not in endpoints

backend/tests/flow/test_node_types.py

@@ -100,6 +100,13 @@ FIXTURES: dict[str, dict] = {
         "requires": [MessageSpec(name="prompt", dtype=DType.STR)],
         "provides": [MessageSpec(name="kitchen_light", dtype=DType.BOOL)],
     },
+    "device": {
+        "params": {"device": "hall", "config": {"gpio.4": {"pull": "up"}}},
+        "requires": [
+            MessageSpec(name="level", port="backlight", dtype=DType.INT, trigger=False)
+        ],
+        "provides": [MessageSpec(name="bell", port="gpio.4", dtype=DType.BOOL)],
+    },
 }