Device node: a device's ports are wired in flows
- flow/nodes/device.py: `device` node with params `device` and `config`;
provides the device's `in` ports, requires its `out` ports without
waking anything, never run (Node.passive), down "Offline" from presence
- DeviceDef.io and DevicePort go; GET /devices/{id} computes `io` and
`bindings` from the device nodes of published flows, in a fixed order,
cached by FlowStore.revision; the credential's scope comes from them
- issues `unknown_device` and `device_port` (wired twice; against a report
the wrong direction or dtype, or unlisted as advisory), rechecked when a
device is created or deleted; ValidationIssue.advisory is a field
- a device publishing on a bound port is credited to its node by the
server; the panel endpoint kind is gone from `_endpoints` and the canvas
- a rebuild that moves a device's bindings sends `dashboard_changed` with
an empty name, which the firmware refetches on
- startup moves each device's `io` rows into device nodes in the flows
declaring their messages, draft and published, one commit; rows no
flow declares are logged and left out
- frontend: Device node settings (device, Publishes, Follows, a port
picker from a report), palette shortcuts per reported port, DeviceDetail
lists bindings with "Use in flow" in place of the I/O editor
- docs: devices, flow editor, node types, issues, API
This commit is contained in:
+346
-29
@@ -11,6 +11,12 @@ committed like a dashboard, so it is versioned with the dashboards it names.
|
||||
What a revert must never undo lives outside git, in ``devices.json``: the nonce
|
||||
a credential is held against, and the device's own report of what it has.
|
||||
Whether a device is connected right now is kept in memory only.
|
||||
|
||||
A device's own ports, a button on its GPIO or its backlight, are wired in
|
||||
flows rather than here: a ``device`` node names the device, publishes what its
|
||||
``in`` ports send and reads what its ``out`` ports follow. What the device
|
||||
reads back as its ``io``, and what its credential may reach, is worked out
|
||||
from those nodes in the published flows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -22,6 +28,8 @@ import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
@@ -30,8 +38,8 @@ from pydantic import BaseModel, Field, TypeAdapter, ValidationError, field_valid
|
||||
|
||||
from fluksio.flow.dashboards import DashboardDef, DashboardNotFound, DashboardStore
|
||||
from fluksio.flow.events import event_bus
|
||||
from fluksio.flow.messages import validate_qualified
|
||||
from fluksio.flow.schemas import _validate_name
|
||||
from fluksio.flow.messages import MessageSpec, flow_of, qualify, validate_qualified
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef, _validate_name
|
||||
from fluksio.flow.store import FlowStore, InvalidName, StaleVersion, checked
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -42,32 +50,75 @@ DEVICE_DIR = "_devices"
|
||||
#: Free text the device interprets: ``gpio.4``, ``backlight``, ``mic``.
|
||||
PORT_PATTERN = re.compile(r"^[a-z0-9_.]+$")
|
||||
|
||||
#: The node type that stands for a device in a flow.
|
||||
DEVICE_TYPE = "device"
|
||||
|
||||
class DevicePort(BaseModel):
|
||||
"""One piece of the device's own hardware, wired to a message.
|
||||
|
||||
A wall panel is a node as well as a screen: a button on its GPIO, a
|
||||
microphone, a speaker. Which ports exist is the device's business, so the
|
||||
name is free text and ``config`` is passed through to it unread.
|
||||
"""
|
||||
@dataclass(frozen=True)
|
||||
class Binding:
|
||||
"""One port of a device, wired to a message by a device node."""
|
||||
|
||||
device: str
|
||||
flow: str
|
||||
node: str
|
||||
port: str
|
||||
#: Qualified, ``flow.name``. A flow still has to declare it, as for any
|
||||
#: widget, before a publish to it lands.
|
||||
#: Qualified, ``flow.name``.
|
||||
message: str
|
||||
#: ``in`` is the device publishing into fluksio, ``out`` is the device
|
||||
#: following the message.
|
||||
direction: Literal["in", "out"]
|
||||
dtype: str
|
||||
#: Passed through to the device unread.
|
||||
config: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class DeviceIo(BaseModel):
|
||||
"""One port as the device reads it: the message it publishes or follows."""
|
||||
|
||||
port: str
|
||||
message: str
|
||||
direction: Literal["in", "out"]
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("port")
|
||||
@classmethod
|
||||
def _check_port(cls, value: str) -> str:
|
||||
if not PORT_PATTERN.fullmatch(value):
|
||||
raise ValueError("Use lowercase letters, digits, underscores and dots")
|
||||
return value
|
||||
|
||||
_check_message = field_validator("message")(validate_qualified)
|
||||
class DeviceBinding(BaseModel):
|
||||
"""Where a port is wired: the device node holding it."""
|
||||
|
||||
flow: str
|
||||
node: str
|
||||
port: str
|
||||
message: str
|
||||
direction: Literal["in", "out"]
|
||||
|
||||
|
||||
def bindings_of(flow: FlowDef) -> list[Binding]:
|
||||
"""The device ports this flow's device nodes wire, in document order."""
|
||||
found: list[Binding] = []
|
||||
for node in flow.nodes:
|
||||
if node.type != DEVICE_TYPE:
|
||||
continue
|
||||
device = str(node.params.get("device") or "")
|
||||
config = node.params.get("config")
|
||||
config = config if isinstance(config, dict) else {}
|
||||
sides = (("in", node.provides), ("out", node.requires))
|
||||
for direction, specs in sides:
|
||||
for spec in specs:
|
||||
if not spec.name:
|
||||
continue
|
||||
own = config.get(spec.port)
|
||||
found.append(
|
||||
Binding(
|
||||
device=device,
|
||||
flow=flow.name,
|
||||
node=node.id,
|
||||
port=spec.port,
|
||||
message=qualify(flow.name, spec.name),
|
||||
direction=direction, # type: ignore[arg-type]
|
||||
dtype=spec.dtype.value,
|
||||
config=own if isinstance(own, dict) else {},
|
||||
)
|
||||
)
|
||||
return found
|
||||
|
||||
|
||||
class DeviceDef(BaseModel):
|
||||
@@ -83,10 +134,6 @@ class DeviceDef(BaseModel):
|
||||
#: touched rather than pointed at. A fact about the screen, not about any
|
||||
#: dashboard on it.
|
||||
touch: bool = False
|
||||
#: The device's own inputs and outputs. An ``in`` port may publish whether
|
||||
#: or not a dashboard is locked, since a lock covers the controls on a
|
||||
#: dashboard and not the sensors on the wall. An ``out`` port only reads.
|
||||
io: list[DevicePort] = Field(default_factory=list)
|
||||
#: Bumped on every save. A save based on an older one is refused.
|
||||
version: int = 0
|
||||
|
||||
@@ -139,6 +186,12 @@ class DeviceStore:
|
||||
# Read-modify-write of the nonce file. An unpair interleaved with
|
||||
# anything else writing it must not be undone by that write.
|
||||
self._state_lock = threading.Lock()
|
||||
#: Every device's bindings, at the flow store revision they were read
|
||||
#: at. A device reads its own on every refetch and its credential is
|
||||
#: checked against them per request.
|
||||
self._wiring: dict[str, list[Binding]] = {}
|
||||
self._wiring_at = -1
|
||||
self._wiring_lock = threading.Lock()
|
||||
|
||||
def _file(self, device_id: str) -> Path:
|
||||
return self.root / f"{checked(device_id)}.json"
|
||||
@@ -148,6 +201,78 @@ class DeviceStore:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(device.model_dump_json(indent=2) + "\n")
|
||||
|
||||
# -- what the published flows wire ------------------------------------
|
||||
|
||||
def wiring(self) -> dict[str, list[Binding]]:
|
||||
"""Every device's bound ports, keyed by device id.
|
||||
|
||||
Each list is in one fixed order, port first, so what a device reads is
|
||||
the same bytes for as long as the flows are the same: the firmware
|
||||
compares the whole document and rebuilds its screen on any change.
|
||||
Read again when the flow store commits, a device save included.
|
||||
"""
|
||||
with self._wiring_lock:
|
||||
revision = self.flows.revision
|
||||
if self._wiring_at != revision:
|
||||
found: dict[str, list[Binding]] = {}
|
||||
for flow in self.flows.read_all():
|
||||
for binding in bindings_of(flow):
|
||||
found.setdefault(binding.device, []).append(binding)
|
||||
for rows in found.values():
|
||||
rows.sort(
|
||||
key=lambda b: (b.port, b.message, b.direction, b.flow, b.node)
|
||||
)
|
||||
self._wiring, self._wiring_at = found, revision
|
||||
return self._wiring
|
||||
|
||||
def bound(self, device_id: str) -> list[Binding]:
|
||||
return self.wiring().get(device_id, [])
|
||||
|
||||
def io(self, device_id: str) -> list[DeviceIo]:
|
||||
"""The ports as the device reads them."""
|
||||
return [
|
||||
DeviceIo(
|
||||
port=b.port, message=b.message, direction=b.direction, config=b.config
|
||||
)
|
||||
for b in self.bound(device_id)
|
||||
]
|
||||
|
||||
def bindings(self, device_id: str) -> list[DeviceBinding]:
|
||||
"""The same ports by the node holding each, for whoever wires them."""
|
||||
return sorted(
|
||||
(
|
||||
DeviceBinding(
|
||||
flow=b.flow,
|
||||
node=b.node,
|
||||
port=b.port,
|
||||
message=b.message,
|
||||
direction=b.direction,
|
||||
)
|
||||
for b in self.bound(device_id)
|
||||
),
|
||||
key=lambda b: (b.flow, b.node, b.port, b.message),
|
||||
)
|
||||
|
||||
def reported_ports(self, device_id: str) -> dict[str, dict[str, Any]] | None:
|
||||
"""The ports the device says it has, by name, or None without a report.
|
||||
|
||||
Read leniently: the report is the device's own word, and a shape this
|
||||
version does not know is no report rather than an error.
|
||||
"""
|
||||
try:
|
||||
report = self.state(device_id).report
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not isinstance(report, dict) or not isinstance(report.get("ports"), list):
|
||||
return None
|
||||
return {
|
||||
str(port["port"]): port
|
||||
for port in report["ports"]
|
||||
if isinstance(port, dict) and port.get("port")
|
||||
}
|
||||
|
||||
# -- the definitions ---------------------------------------------------
|
||||
|
||||
def list(self) -> list[DeviceDef]:
|
||||
found = []
|
||||
for path in sorted(self.root.glob("*.json")):
|
||||
@@ -266,6 +391,7 @@ class DeviceStore:
|
||||
return
|
||||
|
||||
devices: list[tuple[DeviceDef, int]] = []
|
||||
legacy: dict[str, Any] = {}
|
||||
for raw in panels:
|
||||
try:
|
||||
device = DeviceDef.model_validate({**raw, "version": 1})
|
||||
@@ -273,11 +399,23 @@ class DeviceStore:
|
||||
logger.error("Not migrating panel %r: %s", raw, exc)
|
||||
continue
|
||||
devices.append((device, int(raw.get("nonce") or 0)))
|
||||
if raw.get("io"):
|
||||
legacy[device.id] = raw["io"]
|
||||
|
||||
with self._lock, self.flows._write_lock:
|
||||
written = [d.id for d, _ in devices if not self._file(d.id).exists()]
|
||||
for device, _ in devices:
|
||||
if device.id in written:
|
||||
if device.id in legacy and device.id in written:
|
||||
# Kept for `move_io`, which wires the rows into flows next.
|
||||
self._file(device.id).parent.mkdir(parents=True, exist_ok=True)
|
||||
self._file(device.id).write_text(
|
||||
json.dumps(
|
||||
{**device.model_dump(mode="json"), "io": legacy[device.id]},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
elif device.id in written:
|
||||
self._write(device)
|
||||
if written:
|
||||
self.flows._commit("Migrate panels to devices")
|
||||
@@ -295,6 +433,167 @@ class DeviceStore:
|
||||
", ".join(written) or "all were there already",
|
||||
)
|
||||
|
||||
def move_io(self) -> None:
|
||||
"""Wire each device's own ``io`` rows into flows as device nodes, once.
|
||||
|
||||
Before device nodes, a device carried its ports itself. Each row goes
|
||||
to the flow declaring its message, the one the message is named after
|
||||
if that one does, as a port of one device node per device and flow,
|
||||
in the published document and the draft alike, with the dtype that
|
||||
flow declares it with and the row's config under ``params.config``.
|
||||
Every document changed moves one version on, the device files lose
|
||||
``io``, and all of it is one commit.
|
||||
|
||||
A row whose message no flow declares is left out and logged, since
|
||||
there is nowhere to wire it. Running it again finds no ``io`` left,
|
||||
and a document already holding a port is not given it twice.
|
||||
"""
|
||||
with self._lock, self.flows._write_lock:
|
||||
legacy: list[tuple[str, dict[str, Any]]] = []
|
||||
for path in sorted(self.root.glob("*.json")):
|
||||
try:
|
||||
raw = json.loads(path.read_text())
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if isinstance(raw, dict) and "io" in raw:
|
||||
legacy.append((path.stem, raw))
|
||||
if not legacy:
|
||||
return
|
||||
|
||||
docs = _flow_documents(self.flows.root)
|
||||
changed: set[Path] = set()
|
||||
moved: list[str] = []
|
||||
for device_id, raw in legacy:
|
||||
title = str(raw.get("title") or "")
|
||||
for row in raw.get("io") or []:
|
||||
where = _place(docs, device_id, title, row)
|
||||
if where is None:
|
||||
logger.warning(
|
||||
"Device '%s': left out port %r, since no flow declares "
|
||||
"its message. Wire it again in a flow.",
|
||||
device_id,
|
||||
row,
|
||||
)
|
||||
continue
|
||||
changed |= where
|
||||
moved.append(f"{device_id}.{row.get('port')}")
|
||||
|
||||
for path in sorted(changed):
|
||||
flow = docs[path]
|
||||
flow.version += 1
|
||||
path.write_text(flow.model_dump_json(indent=2) + "\n")
|
||||
for device_id, raw in legacy:
|
||||
try:
|
||||
self._write(DeviceDef.model_validate(raw))
|
||||
except ValidationError as exc:
|
||||
logger.error("Device '%s' is unreadable: %s", device_id, exc)
|
||||
self.flows._commit("Move device ports into flows")
|
||||
logger.warning(
|
||||
"Moved device ports into flows: %s",
|
||||
", ".join(moved) or "none could be placed",
|
||||
)
|
||||
|
||||
|
||||
def _flow_documents(root: Path) -> dict[Path, FlowDef]:
|
||||
"""Every published and draft flow document, by its file."""
|
||||
docs: dict[Path, FlowDef] = {}
|
||||
for path in sorted([*root.glob("*/flow.json"), *root.glob("*/flow.draft.json")]):
|
||||
if path.parent.name.startswith("_"):
|
||||
continue
|
||||
try:
|
||||
docs[path] = FlowDef.model_validate_json(path.read_text())
|
||||
except (OSError, ValidationError):
|
||||
logger.warning("Skipping unreadable flow document %s", path)
|
||||
return docs
|
||||
|
||||
|
||||
def _declared(flow: FlowDef, message: str) -> MessageSpec | None:
|
||||
"""How this flow declares a qualified message, if it does."""
|
||||
for declared in flow.inputs:
|
||||
if declared.spec.name and qualify(flow.name, declared.spec.name) == message:
|
||||
return declared.spec
|
||||
for node in flow.nodes:
|
||||
for spec in [*node.provides, *node.requires]:
|
||||
if spec.name and qualify(flow.name, spec.name) == message:
|
||||
return spec
|
||||
return None
|
||||
|
||||
|
||||
def _place(
|
||||
docs: dict[Path, FlowDef], device_id: str, title: str, row: Any
|
||||
) -> set[Path] | None:
|
||||
"""Put one legacy ``io`` row on its flow's device node. The files changed,
|
||||
or None when no flow declares its message."""
|
||||
try:
|
||||
port = str(row["port"])
|
||||
message = validate_qualified(str(row["message"]))
|
||||
direction = row["direction"]
|
||||
if not PORT_PATTERN.fullmatch(port) or direction not in ("in", "out"):
|
||||
raise ValueError
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
owner = flow_of(message)
|
||||
names = sorted({doc.name for doc in docs.values()}, key=lambda n: (n != owner, n))
|
||||
for name in names:
|
||||
found = [
|
||||
(path, doc, spec)
|
||||
for path, doc in docs.items()
|
||||
if doc.name == name and (spec := _declared(doc, message)) is not None
|
||||
]
|
||||
if found:
|
||||
break
|
||||
else:
|
||||
return None
|
||||
# The published declaration first: that is what the engine runs.
|
||||
_, _, declared = sorted(found, key=lambda one: one[0].name != "flow.json")[0]
|
||||
rest = message[len(name) + 1 :] if message.startswith(f"{name}.") else ""
|
||||
spec = MessageSpec(
|
||||
name=rest if rest and "." not in rest else message,
|
||||
port=port,
|
||||
dtype=declared.dtype,
|
||||
item=declared.item,
|
||||
trigger=direction == "in",
|
||||
)
|
||||
config = row.get("config") if isinstance(row.get("config"), dict) else {}
|
||||
|
||||
targets = [(path, doc) for path, doc in docs.items() if doc.name == name]
|
||||
# One node per device and flow, under the same id in both documents.
|
||||
ours = [
|
||||
[
|
||||
n
|
||||
for n in doc.nodes
|
||||
if n.type == DEVICE_TYPE and n.params.get("device") == device_id
|
||||
]
|
||||
for _, doc in targets
|
||||
]
|
||||
existing = next((nodes[0].id for nodes in ours if nodes), None)
|
||||
taken = {node.id for _, doc in targets for node in doc.nodes}
|
||||
node_id, count = existing or device_id, 1
|
||||
while existing is None and node_id in taken:
|
||||
count += 1
|
||||
node_id = f"{device_id}_{count}"
|
||||
|
||||
changed: set[Path] = set()
|
||||
for (path, doc), nodes in zip(targets, ours, strict=True):
|
||||
node = nodes[0] if nodes else None
|
||||
if node is None:
|
||||
node = NodeDef(
|
||||
id=node_id,
|
||||
type=DEVICE_TYPE,
|
||||
title=title,
|
||||
params={"device": device_id, "config": {}},
|
||||
)
|
||||
doc.nodes.append(node)
|
||||
side = node.provides if direction == "in" else node.requires
|
||||
if any(s.port == spec.port and s.name == spec.name for s in side):
|
||||
continue
|
||||
side.append(spec)
|
||||
if config:
|
||||
node.params.setdefault("config", {})[port] = config
|
||||
changed.add(path)
|
||||
return changed
|
||||
|
||||
|
||||
class Presence:
|
||||
"""Which devices are connected now, and when each was last heard from.
|
||||
@@ -308,6 +607,17 @@ class Presence:
|
||||
self._sockets: dict[str, int] = {}
|
||||
self._seen: dict[str, float] = {}
|
||||
self._lock = threading.Lock()
|
||||
#: Told when a device comes or goes: its device nodes, whose health
|
||||
#: is whether it is there.
|
||||
self._watchers: dict[str, set[Callable[[bool], None]]] = {}
|
||||
|
||||
def watch(self, device_id: str, callback: Callable[[bool], None]) -> None:
|
||||
with self._lock:
|
||||
self._watchers.setdefault(device_id, set()).add(callback)
|
||||
|
||||
def unwatch(self, device_id: str, callback: Callable[[bool], None]) -> None:
|
||||
with self._lock:
|
||||
self._watchers.get(device_id, set()).discard(callback)
|
||||
|
||||
def seen(self, device_id: str) -> None:
|
||||
self._seen[device_id] = time.time()
|
||||
@@ -345,15 +655,20 @@ class Presence:
|
||||
|
||||
def _announce(self, device_id: str) -> None:
|
||||
last = self.last_seen(device_id)
|
||||
online = self.online(device_id)
|
||||
event_bus.publish(
|
||||
{
|
||||
"type": "device_status",
|
||||
"device": device_id,
|
||||
"online": self.online(device_id),
|
||||
"online": online,
|
||||
"last_seen": last.isoformat() if last else None,
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
with self._lock:
|
||||
watchers = list(self._watchers.get(device_id, ()))
|
||||
for callback in watchers:
|
||||
callback(online)
|
||||
|
||||
|
||||
presence = Presence()
|
||||
@@ -427,19 +742,21 @@ def targets_of(defn: DashboardDef) -> set[str]:
|
||||
return {w.target for w in defn.widgets if w.target}
|
||||
|
||||
|
||||
def io_in(device: DeviceDef | None) -> set[str]:
|
||||
"""The messages this device's own hardware publishes to."""
|
||||
def io_in(device: DeviceDef | None, devices: DeviceStore) -> set[str]:
|
||||
"""The messages this device's own ports publish to."""
|
||||
if device is None:
|
||||
return set()
|
||||
return {port.message for port in device.io if port.direction == "in"}
|
||||
return {b.message for b in devices.bound(device.id) if b.direction == "in"}
|
||||
|
||||
|
||||
def messages_for(device: DeviceDef | None, store: DashboardStore) -> set[str]:
|
||||
"""Every message this device's dashboards and its own I/O read or write.
|
||||
def messages_for(
|
||||
device: DeviceDef | None, store: DashboardStore, devices: DeviceStore
|
||||
) -> set[str]:
|
||||
"""Every message this device's dashboards and its own ports read or write.
|
||||
|
||||
Empty for a device that is gone, which is the same answer as "nothing".
|
||||
"""
|
||||
own = {port.message for port in device.io} if device else set()
|
||||
own = {b.message for b in devices.bound(device.id)} if device else set()
|
||||
return own | {
|
||||
name for defn in dashboards_for(device, store) for name in messages_of(defn)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user