Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Device node: a device's ports are wired in flows

Browse Source 
- flow/nodes/device.py: `device` node with params `device` and `config`;
  provides the device's `in` ports, requires its `out` ports without
  waking anything, never run (Node.passive), down "Offline" from presence
- DeviceDef.io and DevicePort go; GET /devices/{id} computes `io` and
  `bindings` from the device nodes of published flows, in a fixed order,
  cached by FlowStore.revision; the credential's scope comes from them
- issues `unknown_device` and `device_port` (wired twice; against a report
  the wrong direction or dtype, or unlisted as advisory), rechecked when a
  device is created or deleted; ValidationIssue.advisory is a field
- a device publishing on a bound port is credited to its node by the
  server; the panel endpoint kind is gone from `_endpoints` and the canvas
- a rebuild that moves a device's bindings sends `dashboard_changed` with
  an empty name, which the firmware refetches on
- startup moves each device's `io` rows into device nodes in the flows
  declaring their messages, draft and published, one commit; rows no
  flow declares are logged and left out
- frontend: Device node settings (device, Publishes, Follows, a port
  picker from a report), palette shortcuts per reported port, DeviceDetail
  lists bindings with "Use in flow" in place of the I/O editor
- docs: devices, flow editor, node types, issues, API
This commit is contained in:
Melvin Strobl
2026-10-16 16:40:31 +02:00
parent 91e3b703f9
commit 29d3ec467f
35 changed files with 2195 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/fluksio/api/deps.py
+22 -7
View File
@@ -87,9 +87,16 @@ def _device_messages(device: DeviceDef, request: Request) -> set[str]:
the answer to give when the answer cannot be worked out.
"""
store: DashboardStore | None = getattr(request.app.state, "dashboard_store", None)
if store is None:
owned: DeviceStore | None = getattr(request.app.state, "device_store", None)
if store is None or owned is None:
return set()
return devices.messages_for(device, store)
return devices.messages_for(device, store, owned)
def _device_in(device: DeviceDef, request: Request) -> set[str]:
"""The messages this device's own ``in`` ports are wired to publish."""
owned: DeviceStore | None = getattr(request.app.state, "device_store", None)
return devices.io_in(device, owned) if owned is not None else set()
def _locked(defn: DashboardDef, live: dict[str, Any]) -> bool:
@@ -128,14 +135,15 @@ def _device_writable(device: DeviceDef, request: Request) -> set[str]:
one is what entitles the screen to it. Reads are untouched — read-only is
not blind, and a locked dashboard has to keep drawing live data.
The panel's own ``in`` ports are added outside the lock: it covers the
controls on a dashboard, not the sensors on the device. Its ``out`` ports
are read, not written, unless a widget targets them too.
The device's own ``in`` ports, as flows wire them, are added outside the
lock: it covers the controls on a dashboard, not the sensors on the
device. Its ``out`` ports are read, not written, unless a widget targets
them too.
"""
store: DashboardStore | None = getattr(request.app.state, "dashboard_store", None)
if store is None:
return set()
writable = devices.io_in(device)
writable = _device_in(device, request)
defns = devices.dashboards_for(device, store)
bound = sorted(
{s.message for d in defns if (s := d.settings.get("locked")) and s.message}
@@ -197,11 +205,18 @@ def _device_may(payload: dict[str, Any], request: Request) -> None:
a panel to read every message it names and to publish to none of them but
the requests its own charts ask by.
Its own ports are the device nodes the published flows hold for it, so
wiring a port in a flow is what lets the device publish on it.
The device is left on ``request.state`` for the route, which credits a
value published on one of its ports to the node holding that port.
# ponytail: the device file, its nonce and the published dashboards are
# re-read per request. Cache them behind the store's revision if this
# shows up in a profile.
"""
device = _device_for(payload, request.app)
request.state.device = device
api = settings.API_V1_STR
path = request.url.path
@@ -252,7 +267,7 @@ def _device_may(payload: dict[str, Any], request: Request) -> None:
elif method == "PUT" and path == f"{api}/artifacts":
# What the device's own sensor recorded — a microphone's utterance —
# uploaded under the ``in`` message it is about to publish it to.
allowed = request.query_params.get("name", "") in devices.io_in(device)
allowed = request.query_params.get("name", "") in _device_in(device, request)
if not allowed:
raise HTTPException(
backend/fluksio/api/routes/devices.py
+72 -21
View File
@@ -13,7 +13,8 @@ comes. Anyone on the internet can ask the portal for a code, so a request that
came that way is adopted by its code only.
The credential is scoped: ``fluksio.api.deps`` lets it reach the dashboards
that device was assigned, the messages their widgets bind to, and nothing else.
that device was assigned, the messages their widgets bind to, the messages the
device nodes in published flows wire its own ports to, and nothing else.
Deleting the device revokes it, and so does unpairing, which bumps the
device's nonce and leaves what it shows as it was.
@@ -25,6 +26,7 @@ from __future__ import annotations
import secrets
import time
from contextlib import suppress
from datetime import datetime, timedelta
from typing import Any, cast
@@ -43,9 +45,12 @@ from fluksio.api.deps import (
from fluksio.cloud import config as cloud_config
from fluksio.core import security
from fluksio.core.config import settings
from fluksio.flow.controller import RebuildBusy
from fluksio.flow.devices import (
DeviceBinding,
DeviceDef,
DeviceExists,
DeviceIo,
DeviceNotFound,
DeviceStore,
is_lan,
@@ -288,12 +293,27 @@ class DeviceCreate(BaseModel):
title: str = ""
class DevicePublic(DeviceDef):
class DeviceDocument(DeviceDef):
"""A device as it reads itself: what it shows, and what its ports carry.
``io`` is worked out from the device nodes in published flows, in a fixed
order, so an unchanged state reads back as the same bytes.
"""
#: Each port the flows wire, and the message it publishes or follows.
io: list[DeviceIo] = Field(default_factory=list)
#: The same ports by the flow and node holding them.
bindings: list[DeviceBinding] = Field(default_factory=list)
class DevicePublic(DeviceDocument):
"""A device as the Devices tab lists it, with whether it is there."""
online: bool = False
#: The last request it made, since this instance started.
last_seen: datetime | None = None
#: What the device says it has, as it last said so. None until it reports.
report: dict[str, Any] | None = None
class DevicesPublic(BaseModel):
@@ -364,6 +384,26 @@ def _not_found(device_id: str) -> HTTPException:
return HTTPException(status_code=404, detail=f"No device named {device_id!r}")
def _document(device: DeviceDef, store: DeviceStore) -> DeviceDocument:
return DeviceDocument(
**device.model_dump(),
io=store.io(device.id),
bindings=store.bindings(device.id),
)
async def _recheck(request: Request) -> None:
"""Validate the flows again: a device node may name this device.
The device is saved either way, so a busy engine is not this request's
failure: the next rebuild validates again.
"""
controller = getattr(request.app.state, "flow_controller", None)
if controller is not None:
with suppress(RebuildBusy):
await controller.recheck()
def _changed() -> None:
"""Tell every device socket that what it shows may have moved.
@@ -457,15 +497,19 @@ panels_router.add_api_route(
@router.get("/", response_model=DevicesPublic, dependencies=[Depends(get_current_user)])
async def read_devices(store: DeviceStoreDep) -> Any:
"""Every device, what each one shows, and whether it is connected."""
devices = await run_in_threadpool(store.list)
data = [
DevicePublic(
**device.model_dump(),
online=presence.online(device.id),
last_seen=presence.last_seen(device.id),
)
for device in devices
]
def gather() -> list[DevicePublic]:
return [
DevicePublic(
**_document(device, store).model_dump(),
online=presence.online(device.id),
last_seen=presence.last_seen(device.id),
report=store.state(device.id).report,
)
for device in store.list()
]
data = await run_in_threadpool(gather)
return DevicesPublic(
data=data,
count=len(data),
@@ -503,7 +547,10 @@ def pending_device(code: str) -> Any:
dependencies=[Depends(get_current_active_superuser)],
)
async def create_device(
device_id: str, store: DeviceStoreDep, body: DeviceCreate | None = None
device_id: str,
store: DeviceStoreDep,
request: Request,
body: DeviceCreate | None = None,
) -> Any:
"""Add a device. It shows nothing until it is given dashboards."""
if device_id in RESERVED:
@@ -513,35 +560,38 @@ async def create_device(
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc))
try:
return await run_in_threadpool(store.create, device)
created = await run_in_threadpool(store.create, device)
except DeviceExists:
raise HTTPException(status_code=409, detail=f"{device_id!r} already exists")
await _recheck(request)
return created
@router.get(
"/{device_id}",
response_model=DeviceDef,
response_model=DeviceDocument,
dependencies=[Depends(get_current_user)],
)
async def read_device(device_id: str, store: DeviceStoreDep) -> Any:
"""One device, read by a person or by the device holding its credential.
The definition alone: a device compares what it reads with what it is
showing, and a field that changes on every request would rebuild its
screen each time. Declared after the pairing routes and ``/pending``, so
neither is mistaken for a device of that name.
The definition and what the flows wire its ports to, nothing about this
moment: a device compares what it reads with what it is showing, and a
field that changes on every request would rebuild its screen each time.
Declared after the pairing routes and ``/pending``, so neither is mistaken
for a device of that name.
"""
device = await run_in_threadpool(store.find, device_id)
if device is None:
raise _not_found(device_id)
return device
return await run_in_threadpool(_document, device, store)
panels_router.add_api_route(
"/{device_id}",
read_device,
methods=["GET"],
response_model=DeviceDef,
response_model=DeviceDocument,
dependencies=[Depends(get_current_user)],
)
@@ -580,12 +630,13 @@ async def save_device(device_id: str, body: DeviceDef, store: DeviceStoreDep) ->
response_model=Message,
dependencies=[Depends(get_current_active_superuser)],
)
async def delete_device(device_id: str, store: DeviceStoreDep) -> Any:
async def delete_device(device_id: str, store: DeviceStoreDep, request: Request) -> Any:
"""Remove the device and refuse its credential, wherever it was minted here."""
try:
await run_in_threadpool(store.delete, device_id)
except DeviceNotFound:
raise _not_found(device_id)
await _recheck(request)
_changed()
return Message(message=f"Deleted {device_id}")
backend/fluksio/api/routes/flows.py
+5 -29
View File
@@ -79,19 +79,18 @@ ws_router = APIRouter(prefix="/flows", tags=["flows"])
class Endpoint(BaseModel):
"""Something wired into this flow that is not a node in it.
A dashboard control setting one of its messages, a tile showing one, a
port on a panel's own hardware, or a node in another flow on the far side
of a dotted name. The canvas draws these so a value never appears to come
A dashboard control setting one of its messages, a tile showing one, or a
node in another flow on the far side of a dotted name. The canvas draws these so a value never appears to come
from nowhere — or worse, appears to come from whichever node happens to be
drawn as a producer.
"""
#: dashboard, panel or flow.
#: dashboard or flow.
kind: str
#: Stable within its kind, and used as the canvas node id.
id: str
label: str
#: What sort of widget, the panel, or the node type in the other flow.
#: What sort of widget, or the node type in the other flow.
detail: str = ""
#: Messages of this flow it publishes, and ones it reads.
provides: list[str] = []
@@ -157,29 +156,6 @@ def _endpoints(controller: FlowController, definition: FlowDef) -> list[Endpoint
requires=binding["requires"],
)
# A device's own hardware: an ``in`` port publishes like a control, an
# ``out`` port reads like a tile. The id is the source id the device
# publishes under, so its edges pulse for its values.
prefix = f"{flow}."
devices: DeviceStore | None = getattr(controller, "devices", None)
for device in devices.list() if devices is not None else []:
for port in device.io:
if not port.message.startswith(prefix):
continue
key = f"panel:{device.id}:{port.port}"
endpoint = found.setdefault(
key,
Endpoint(
kind="panel",
id=key,
label=port.port,
detail=device.title or device.id,
),
)
side = endpoint.provides if port.direction == "in" else endpoint.requires
if port.message not in side:
side.append(port.message)
def _flow_endpoint(key: str, detail: str) -> Endpoint:
"""One label per node on the far side, however many names reach it."""
return found.setdefault(
@@ -1102,7 +1078,7 @@ def panel_scope(token: str, app: Any) -> set[str] | None:
devices: DeviceStore | None = getattr(app.state, "device_store", None)
if store is None or devices is None:
return set()
return messages_for(devices.find(device_id), store)
return messages_for(devices.find(device_id), store, devices)
def event_for_panel(event: dict[str, Any], only: set[str]) -> bool:
backend/fluksio/api/routes/messages.py
+26 -3
View File
@@ -8,11 +8,12 @@ into it without owning a node.
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.concurrency import run_in_threadpool
from pydantic import BaseModel
from fluksio.api.deps import FlowControllerDep, get_current_user
from fluksio.flow.devices import DeviceDef, DeviceStore
from fluksio.flow.messages import flow_of
from fluksio.flow.pipeline import ValueSource
from fluksio.flow.state import as_number
@@ -71,16 +72,38 @@ async def read_messages(controller: FlowControllerDep) -> Any:
return MessagesPublic(data=infos, count=len(infos))
def _port_source(request: Request, name: str) -> ValueSource | None:
"""The device node a device's own publish on a bound ``in`` port is from.
Said by the server rather than by the device: the node is what the canvas
draws the value leaving from, and the device does not know which node in
which flow holds its port.
"""
device: DeviceDef | None = getattr(request.state, "device", None)
devices: DeviceStore | None = getattr(request.app.state, "device_store", None)
if device is None or devices is None:
return None
for bound in devices.bound(device.id):
if bound.direction == "in" and bound.message == name:
return ValueSource(
kind="node",
id=f"{bound.flow}.{bound.node}",
label=device.title or device.id,
detail=bound.port,
)
return None
@router.post("/{name}", response_model=MessageValue)
async def publish_message(
name: str, body: PublishRequest, controller: FlowControllerDep
name: str, body: PublishRequest, controller: FlowControllerDep, request: Request
) -> Any:
"""Put a value into the graph, as a dashboard control does.
Only a message some flow declares can be published to: flows own the
namespace, and a dashboard is a client of it rather than a second author.
"""
source = ValueSource(
source = _port_source(request, name) or ValueSource(
kind=body.source_kind,
id=body.source_id,
label=body.source_label or body.source_id or "API",
backend/fluksio/api/routes/observability.py
+1 -4
View File
@@ -28,7 +28,6 @@ from fluksio.api.deps import (
from fluksio.api.routes.runs import elapsed_ms
from fluksio.core.config import settings
from fluksio.flow.controller import NodeStatus
from fluksio.flow.pipeline import ADVISORY_ISSUES
from fluksio.models import EngineEvent, FlowRun, Message, MetricBucket
router = APIRouter(
@@ -236,9 +235,7 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
{
issue.flow
for issue in controller.issues
if issue.flow
and issue.code != "node_error"
and issue.code not in ADVISORY_ISSUES
if issue.flow and issue.code != "node_error" and not issue.advisory
}
)
if invalid:
backend/fluksio/flow/controller.py
+150 -8
View File
@@ -22,6 +22,7 @@ import logging
import threading
import time
import traceback
from collections import Counter
from collections.abc import Callable
from contextlib import suppress
from dataclasses import dataclass, field
@@ -42,6 +43,7 @@ from fluksio.flow.nodes import (
ChangeNode,
DebounceNode,
DelayNode,
DeviceNode,
ExecNode,
FileNode,
HttpNode,
@@ -58,7 +60,6 @@ from fluksio.flow.nodes import (
WrenNode,
)
from fluksio.flow.pipeline import (
ADVISORY_ISSUES,
NodeOutcome,
Pipeline,
RunCacheLookup,
@@ -435,6 +436,12 @@ NODE_TYPES: dict[str, NodeType] = {
cls=NtfyNode,
params_schema=_schema_of(NtfyNode),
),
"device": NodeType(
title="Device",
description="A paired device's own ports: what it publishes, what it follows.",
cls=DeviceNode,
params_schema=_schema_of(DeviceNode),
),
}
@@ -492,9 +499,12 @@ class FlowController:
# Set by the lifespan once the store exists; the canvas asks the
# controller which dashboards are wired into a flow.
self.dashboards: Any = None
#: The device store, for the same reason: a device's own ports are
#: drawn on the canvas of the flow they wire into.
#: The device store, for the same reason: validation checks a device
#: node against the device it names.
self.devices: Any = None
#: Every device's bound ports as of the last build, so a rebuild that
#: moves them can tell the devices to read theirs again.
self._device_wiring: Any = None
self.pipeline: Pipeline | None = None
self.loaded: dict[str, LoadedNode] = {}
@@ -696,7 +706,9 @@ class FlowController:
self.pipeline.history_limits = self.history_limits
if self.execution is not None:
self.execution.bind(self.pipeline)
self.issues = _collect_issues(loaded, self.pipeline, flow_inputs)
self.issues = _collect_issues(
loaded, self.pipeline, flow_inputs, self.devices
)
await self._activate()
@@ -718,6 +730,7 @@ class FlowController:
"paused": self.paused_flows(),
}
)
self._rewired()
async def reload_flow(
self, name: str, wait: float | None = FLOW_REBUILD_WAIT
@@ -800,7 +813,7 @@ class FlowController:
# Over the whole graph, because it has to be: taking a producer out
# of one flow is what leaves another flow's input unconnected.
self.issues = _collect_issues(
self.loaded, pipeline, self._all_flow_inputs()
self.loaded, pipeline, self._all_flow_inputs(), self.devices
)
await self._activate(name)
@@ -821,6 +834,47 @@ class FlowController:
"paused": self.paused_flows(),
}
)
self._rewired()
async def recheck(self) -> None:
"""Validate the built graph again without rebuilding anything.
For what validation reads from outside the flows: a device created or
deleted changes whether a device node names one that exists.
"""
if self.pipeline is None:
return
await self._acquire(FLOW_REBUILD_WAIT)
try:
self.issues = _collect_issues(
self.loaded, self.pipeline, self._all_flow_inputs(), self.devices
)
finally:
self._lock.release()
self._publish(
{
"type": "pipeline_rebuilt",
"issues": [issue.model_dump() for issue in self.issues],
"nodes": [status.model_dump() for status in self.node_statuses()],
"paused": self.paused_flows(),
}
)
def _rewired(self) -> None:
"""Tell the devices when a rebuild moved what their ports are wired to.
An empty dashboard name is what a device refetches its own document
on, and what its socket rescopes on. The first build only takes note:
a device connecting later reads the document as it is.
"""
if self.devices is None:
return
wiring = self.devices.wiring()
known, self._device_wiring = self._device_wiring, wiring
if known is not None and known != wiring:
self._publish(
{"type": "dashboard_changed", "dashboard": "", "ts": time.time()}
)
async def reload_failed_flows(self) -> list[str]:
"""Rebuild only the flows holding a node that would not load.
@@ -1375,7 +1429,7 @@ class FlowController:
max_workers=self.max_workers,
initial_values=initial_values,
)
issues = _collect_issues(loaded, pipeline, flow_inputs)
issues = _collect_issues(loaded, pipeline, flow_inputs, self.devices)
return Preview(
nodes=[
NodeStatusPublic(
@@ -1591,7 +1645,7 @@ class FlowController:
# it, so all of them are marked rather than an arbitrary one.
troubled: dict[str, str] = {}
for issue in self.issues + self._health_issues():
if issue.code in ADVISORY_ISSUES:
if issue.advisory:
continue
for member in (*issue.nodes, *filter(None, [issue.node])):
troubled.setdefault(member, issue.message)
@@ -1983,9 +2037,10 @@ def _collect_issues(
loaded: dict[str, LoadedNode],
pipeline: Pipeline,
flow_inputs: dict[str, bool],
devices: Any = None,
) -> list[ValidationIssue]:
"""Everything wrong with a built graph, whether it is running or not."""
issues = pipeline.validate(flow_inputs)
issues = pipeline.validate(flow_inputs) + _device_issues(loaded, devices)
issues += [
ValidationIssue(
code="node_error",
@@ -2046,6 +2101,93 @@ def _collect_issues(
return issues
def _device_issues(
loaded: dict[str, LoadedNode], devices: Any
) -> list[ValidationIssue]:
"""Device nodes naming no device, and ports that cannot be what they say.
A port is wired to one message in one place, so one bound twice across
the graph is an error on each. Against a report, a port in the wrong
direction or of another dtype is an error too, and one the device does not
list is a warning: the device may know it all the same. Without a report
a port is the device's business and nothing is said about it.
"""
if devices is None:
return []
nodes = [
(entry, entry.node)
for entry in loaded.values()
if isinstance(entry.node, DeviceNode)
]
if not nodes:
return []
known = {device.id for device in devices.list()}
bound = Counter(
(node.cfg.device, spec.port)
for _, node in nodes
for spec in (*node.input_ports, *node.output_ports)
if spec.name
)
issues: list[ValidationIssue] = []
for entry, node in nodes:
device = node.cfg.device
if device not in known:
issues.append(
ValidationIssue(
code="unknown_device",
message=(
f"'{node.local_id}' names device '{device}', which does "
"not exist."
if device
else f"Pick the device '{node.local_id}' stands for."
),
flow=entry.flow,
node=entry.id,
)
)
continue
reported = devices.reported_ports(device)
for direction, specs in (("in", node.output_ports), ("out", node.input_ports)):
for spec in specs:
if not spec.name:
continue
problem, advisory = "", False
said = reported.get(spec.port) if reported is not None else None
if bound[(device, spec.port)] > 1:
problem = (
f"Port '{spec.port}' of '{device}' is wired more than "
"once. A port carries one message, in one flow."
)
elif reported is None:
pass
elif said is None:
problem = f"'{device}' does not report a port '{spec.port}'."
advisory = True
elif said.get("direction") and said["direction"] != direction:
problem = (
f"'{spec.port}' of '{device}' is an {said['direction']} "
f"port, wired here as {direction}."
)
elif said.get("dtype") and said["dtype"] != spec.dtype.value:
problem = (
f"'{spec.port}' of '{device}' carries {said['dtype']}, "
f"wired here to a {spec.dtype.value} message."
)
if problem:
issues.append(
ValidationIssue(
code="device_port",
message=problem,
flow=entry.flow,
node=entry.id,
port=spec.port,
message_name=spec.name,
advisory=advisory,
)
)
return issues
def _short_error(exc: Exception) -> str:
"""One line a node author can act on: what broke, and where in their code.
backend/fluksio/flow/devices.py
+346 -29
View File
@@ -11,6 +11,12 @@ committed like a dashboard, so it is versioned with the dashboards it names.
What a revert must never undo lives outside git, in ``devices.json``: the nonce
a credential is held against, and the device's own report of what it has.
Whether a device is connected right now is kept in memory only.
A device's own ports, a button on its GPIO or its backlight, are wired in
flows rather than here: a ``device`` node names the device, publishes what its
``in`` ports send and reads what its ``out`` ports follow. What the device
reads back as its ``io``, and what its credential may reach, is worked out
from those nodes in the published flows.
"""
from __future__ import annotations
@@ -22,6 +28,8 @@ import os
import re
import threading
import time
from collections.abc import Callable
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Literal
@@ -30,8 +38,8 @@ from pydantic import BaseModel, Field, TypeAdapter, ValidationError, field_valid
from fluksio.flow.dashboards import DashboardDef, DashboardNotFound, DashboardStore
from fluksio.flow.events import event_bus
from fluksio.flow.messages import validate_qualified
from fluksio.flow.schemas import _validate_name
from fluksio.flow.messages import MessageSpec, flow_of, qualify, validate_qualified
from fluksio.flow.schemas import FlowDef, NodeDef, _validate_name
from fluksio.flow.store import FlowStore, InvalidName, StaleVersion, checked
logger = logging.getLogger(__name__)
@@ -42,32 +50,75 @@ DEVICE_DIR = "_devices"
#: Free text the device interprets: ``gpio.4``, ``backlight``, ``mic``.
PORT_PATTERN = re.compile(r"^[a-z0-9_.]+$")
#: The node type that stands for a device in a flow.
DEVICE_TYPE = "device"
class DevicePort(BaseModel):
"""One piece of the device's own hardware, wired to a message.
A wall panel is a node as well as a screen: a button on its GPIO, a
microphone, a speaker. Which ports exist is the device's business, so the
name is free text and ``config`` is passed through to it unread.
"""
@dataclass(frozen=True)
class Binding:
"""One port of a device, wired to a message by a device node."""
device: str
flow: str
node: str
port: str
#: Qualified, ``flow.name``. A flow still has to declare it, as for any
#: widget, before a publish to it lands.
#: Qualified, ``flow.name``.
message: str
#: ``in`` is the device publishing into fluksio, ``out`` is the device
#: following the message.
direction: Literal["in", "out"]
dtype: str
#: Passed through to the device unread.
config: dict[str, Any] = field(default_factory=dict)
class DeviceIo(BaseModel):
"""One port as the device reads it: the message it publishes or follows."""
port: str
message: str
direction: Literal["in", "out"]
config: dict[str, Any] = Field(default_factory=dict)
@field_validator("port")
@classmethod
def _check_port(cls, value: str) -> str:
if not PORT_PATTERN.fullmatch(value):
raise ValueError("Use lowercase letters, digits, underscores and dots")
return value
_check_message = field_validator("message")(validate_qualified)
class DeviceBinding(BaseModel):
"""Where a port is wired: the device node holding it."""
flow: str
node: str
port: str
message: str
direction: Literal["in", "out"]
def bindings_of(flow: FlowDef) -> list[Binding]:
"""The device ports this flow's device nodes wire, in document order."""
found: list[Binding] = []
for node in flow.nodes:
if node.type != DEVICE_TYPE:
continue
device = str(node.params.get("device") or "")
config = node.params.get("config")
config = config if isinstance(config, dict) else {}
sides = (("in", node.provides), ("out", node.requires))
for direction, specs in sides:
for spec in specs:
if not spec.name:
continue
own = config.get(spec.port)
found.append(
Binding(
device=device,
flow=flow.name,
node=node.id,
port=spec.port,
message=qualify(flow.name, spec.name),
direction=direction, # type: ignore[arg-type]
dtype=spec.dtype.value,
config=own if isinstance(own, dict) else {},
)
)
return found
class DeviceDef(BaseModel):
@@ -83,10 +134,6 @@ class DeviceDef(BaseModel):
#: touched rather than pointed at. A fact about the screen, not about any
#: dashboard on it.
touch: bool = False
#: The device's own inputs and outputs. An ``in`` port may publish whether
#: or not a dashboard is locked, since a lock covers the controls on a
#: dashboard and not the sensors on the wall. An ``out`` port only reads.
io: list[DevicePort] = Field(default_factory=list)
#: Bumped on every save. A save based on an older one is refused.
version: int = 0
@@ -139,6 +186,12 @@ class DeviceStore:
# Read-modify-write of the nonce file. An unpair interleaved with
# anything else writing it must not be undone by that write.
self._state_lock = threading.Lock()
#: Every device's bindings, at the flow store revision they were read
#: at. A device reads its own on every refetch and its credential is
#: checked against them per request.
self._wiring: dict[str, list[Binding]] = {}
self._wiring_at = -1
self._wiring_lock = threading.Lock()
def _file(self, device_id: str) -> Path:
return self.root / f"{checked(device_id)}.json"
@@ -148,6 +201,78 @@ class DeviceStore:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(device.model_dump_json(indent=2) + "\n")
# -- what the published flows wire ------------------------------------
def wiring(self) -> dict[str, list[Binding]]:
"""Every device's bound ports, keyed by device id.
Each list is in one fixed order, port first, so what a device reads is
the same bytes for as long as the flows are the same: the firmware
compares the whole document and rebuilds its screen on any change.
Read again when the flow store commits, a device save included.
"""
with self._wiring_lock:
revision = self.flows.revision
if self._wiring_at != revision:
found: dict[str, list[Binding]] = {}
for flow in self.flows.read_all():
for binding in bindings_of(flow):
found.setdefault(binding.device, []).append(binding)
for rows in found.values():
rows.sort(
key=lambda b: (b.port, b.message, b.direction, b.flow, b.node)
)
self._wiring, self._wiring_at = found, revision
return self._wiring
def bound(self, device_id: str) -> list[Binding]:
return self.wiring().get(device_id, [])
def io(self, device_id: str) -> list[DeviceIo]:
"""The ports as the device reads them."""
return [
DeviceIo(
port=b.port, message=b.message, direction=b.direction, config=b.config
)
for b in self.bound(device_id)
]
def bindings(self, device_id: str) -> list[DeviceBinding]:
"""The same ports by the node holding each, for whoever wires them."""
return sorted(
(
DeviceBinding(
flow=b.flow,
node=b.node,
port=b.port,
message=b.message,
direction=b.direction,
)
for b in self.bound(device_id)
),
key=lambda b: (b.flow, b.node, b.port, b.message),
)
def reported_ports(self, device_id: str) -> dict[str, dict[str, Any]] | None:
"""The ports the device says it has, by name, or None without a report.
Read leniently: the report is the device's own word, and a shape this
version does not know is no report rather than an error.
"""
try:
report = self.state(device_id).report
except (OSError, ValueError):
return None
if not isinstance(report, dict) or not isinstance(report.get("ports"), list):
return None
return {
str(port["port"]): port
for port in report["ports"]
if isinstance(port, dict) and port.get("port")
}
# -- the definitions ---------------------------------------------------
def list(self) -> list[DeviceDef]:
found = []
for path in sorted(self.root.glob("*.json")):
@@ -266,6 +391,7 @@ class DeviceStore:
return
devices: list[tuple[DeviceDef, int]] = []
legacy: dict[str, Any] = {}
for raw in panels:
try:
device = DeviceDef.model_validate({**raw, "version": 1})
@@ -273,11 +399,23 @@ class DeviceStore:
logger.error("Not migrating panel %r: %s", raw, exc)
continue
devices.append((device, int(raw.get("nonce") or 0)))
if raw.get("io"):
legacy[device.id] = raw["io"]
with self._lock, self.flows._write_lock:
written = [d.id for d, _ in devices if not self._file(d.id).exists()]
for device, _ in devices:
if device.id in written:
if device.id in legacy and device.id in written:
# Kept for `move_io`, which wires the rows into flows next.
self._file(device.id).parent.mkdir(parents=True, exist_ok=True)
self._file(device.id).write_text(
json.dumps(
{**device.model_dump(mode="json"), "io": legacy[device.id]},
indent=2,
)
+ "\n"
)
elif device.id in written:
self._write(device)
if written:
self.flows._commit("Migrate panels to devices")
@@ -295,6 +433,167 @@ class DeviceStore:
", ".join(written) or "all were there already",
)
def move_io(self) -> None:
"""Wire each device's own ``io`` rows into flows as device nodes, once.
Before device nodes, a device carried its ports itself. Each row goes
to the flow declaring its message, the one the message is named after
if that one does, as a port of one device node per device and flow,
in the published document and the draft alike, with the dtype that
flow declares it with and the row's config under ``params.config``.
Every document changed moves one version on, the device files lose
``io``, and all of it is one commit.
A row whose message no flow declares is left out and logged, since
there is nowhere to wire it. Running it again finds no ``io`` left,
and a document already holding a port is not given it twice.
"""
with self._lock, self.flows._write_lock:
legacy: list[tuple[str, dict[str, Any]]] = []
for path in sorted(self.root.glob("*.json")):
try:
raw = json.loads(path.read_text())
except (OSError, ValueError):
continue
if isinstance(raw, dict) and "io" in raw:
legacy.append((path.stem, raw))
if not legacy:
return
docs = _flow_documents(self.flows.root)
changed: set[Path] = set()
moved: list[str] = []
for device_id, raw in legacy:
title = str(raw.get("title") or "")
for row in raw.get("io") or []:
where = _place(docs, device_id, title, row)
if where is None:
logger.warning(
"Device '%s': left out port %r, since no flow declares "
"its message. Wire it again in a flow.",
device_id,
row,
)
continue
changed |= where
moved.append(f"{device_id}.{row.get('port')}")
for path in sorted(changed):
flow = docs[path]
flow.version += 1
path.write_text(flow.model_dump_json(indent=2) + "\n")
for device_id, raw in legacy:
try:
self._write(DeviceDef.model_validate(raw))
except ValidationError as exc:
logger.error("Device '%s' is unreadable: %s", device_id, exc)
self.flows._commit("Move device ports into flows")
logger.warning(
"Moved device ports into flows: %s",
", ".join(moved) or "none could be placed",
)
def _flow_documents(root: Path) -> dict[Path, FlowDef]:
"""Every published and draft flow document, by its file."""
docs: dict[Path, FlowDef] = {}
for path in sorted([*root.glob("*/flow.json"), *root.glob("*/flow.draft.json")]):
if path.parent.name.startswith("_"):
continue
try:
docs[path] = FlowDef.model_validate_json(path.read_text())
except (OSError, ValidationError):
logger.warning("Skipping unreadable flow document %s", path)
return docs
def _declared(flow: FlowDef, message: str) -> MessageSpec | None:
"""How this flow declares a qualified message, if it does."""
for declared in flow.inputs:
if declared.spec.name and qualify(flow.name, declared.spec.name) == message:
return declared.spec
for node in flow.nodes:
for spec in [*node.provides, *node.requires]:
if spec.name and qualify(flow.name, spec.name) == message:
return spec
return None
def _place(
docs: dict[Path, FlowDef], device_id: str, title: str, row: Any
) -> set[Path] | None:
"""Put one legacy ``io`` row on its flow's device node. The files changed,
or None when no flow declares its message."""
try:
port = str(row["port"])
message = validate_qualified(str(row["message"]))
direction = row["direction"]
if not PORT_PATTERN.fullmatch(port) or direction not in ("in", "out"):
raise ValueError
except (KeyError, TypeError, ValueError):
return None
owner = flow_of(message)
names = sorted({doc.name for doc in docs.values()}, key=lambda n: (n != owner, n))
for name in names:
found = [
(path, doc, spec)
for path, doc in docs.items()
if doc.name == name and (spec := _declared(doc, message)) is not None
]
if found:
break
else:
return None
# The published declaration first: that is what the engine runs.
_, _, declared = sorted(found, key=lambda one: one[0].name != "flow.json")[0]
rest = message[len(name) + 1 :] if message.startswith(f"{name}.") else ""
spec = MessageSpec(
name=rest if rest and "." not in rest else message,
port=port,
dtype=declared.dtype,
item=declared.item,
trigger=direction == "in",
)
config = row.get("config") if isinstance(row.get("config"), dict) else {}
targets = [(path, doc) for path, doc in docs.items() if doc.name == name]
# One node per device and flow, under the same id in both documents.
ours = [
[
n
for n in doc.nodes
if n.type == DEVICE_TYPE and n.params.get("device") == device_id
]
for _, doc in targets
]
existing = next((nodes[0].id for nodes in ours if nodes), None)
taken = {node.id for _, doc in targets for node in doc.nodes}
node_id, count = existing or device_id, 1
while existing is None and node_id in taken:
count += 1
node_id = f"{device_id}_{count}"
changed: set[Path] = set()
for (path, doc), nodes in zip(targets, ours, strict=True):
node = nodes[0] if nodes else None
if node is None:
node = NodeDef(
id=node_id,
type=DEVICE_TYPE,
title=title,
params={"device": device_id, "config": {}},
)
doc.nodes.append(node)
side = node.provides if direction == "in" else node.requires
if any(s.port == spec.port and s.name == spec.name for s in side):
continue
side.append(spec)
if config:
node.params.setdefault("config", {})[port] = config
changed.add(path)
return changed
class Presence:
"""Which devices are connected now, and when each was last heard from.
@@ -308,6 +607,17 @@ class Presence:
self._sockets: dict[str, int] = {}
self._seen: dict[str, float] = {}
self._lock = threading.Lock()
#: Told when a device comes or goes: its device nodes, whose health
#: is whether it is there.
self._watchers: dict[str, set[Callable[[bool], None]]] = {}
def watch(self, device_id: str, callback: Callable[[bool], None]) -> None:
with self._lock:
self._watchers.setdefault(device_id, set()).add(callback)
def unwatch(self, device_id: str, callback: Callable[[bool], None]) -> None:
with self._lock:
self._watchers.get(device_id, set()).discard(callback)
def seen(self, device_id: str) -> None:
self._seen[device_id] = time.time()
@@ -345,15 +655,20 @@ class Presence:
def _announce(self, device_id: str) -> None:
last = self.last_seen(device_id)
online = self.online(device_id)
event_bus.publish(
{
"type": "device_status",
"device": device_id,
"online": self.online(device_id),
"online": online,
"last_seen": last.isoformat() if last else None,
"ts": time.time(),
}
)
with self._lock:
watchers = list(self._watchers.get(device_id, ()))
for callback in watchers:
callback(online)
presence = Presence()
@@ -427,19 +742,21 @@ def targets_of(defn: DashboardDef) -> set[str]:
return {w.target for w in defn.widgets if w.target}
def io_in(device: DeviceDef | None) -> set[str]:
"""The messages this device's own hardware publishes to."""
def io_in(device: DeviceDef | None, devices: DeviceStore) -> set[str]:
"""The messages this device's own ports publish to."""
if device is None:
return set()
return {port.message for port in device.io if port.direction == "in"}
return {b.message for b in devices.bound(device.id) if b.direction == "in"}
def messages_for(device: DeviceDef | None, store: DashboardStore) -> set[str]:
"""Every message this device's dashboards and its own I/O read or write.
def messages_for(
device: DeviceDef | None, store: DashboardStore, devices: DeviceStore
) -> set[str]:
"""Every message this device's dashboards and its own ports read or write.
Empty for a device that is gone, which is the same answer as "nothing".
"""
own = {port.message for port in device.io} if device else set()
own = {b.message for b in devices.bound(device.id)} if device else set()
return own | {
name for defn in dashboards_for(device, store) for name in messages_of(defn)
}
backend/fluksio/flow/nodes/__init__.py
+2
View File
@@ -7,6 +7,7 @@ Split by the outside world each one talks to. Importing from
from fluksio.flow.nodes.base import RESERVED_SETTINGS, Node
from fluksio.flow.nodes.debounce import DebounceNode
from fluksio.flow.nodes.delay import DelayNode
from fluksio.flow.nodes.device import DeviceNode
from fluksio.flow.nodes.exec import ExecNode
from fluksio.flow.nodes.file import FileNode
from fluksio.flow.nodes.http import HttpNode
@@ -23,6 +24,7 @@ __all__ = [
"ChangeNode",
"DebounceNode",
"DelayNode",
"DeviceNode",
"ExecNode",
"FileNode",
"HttpNode",
backend/fluksio/flow/nodes/base.py
+5
View File
@@ -182,6 +182,11 @@ class Node:
# to the device, which at-least-once delivery must not cause.
idempotent: bool = True
# A node standing for something outside the engine, which publishes its
# outputs and reads its inputs itself: a paired device. The pipeline never
# runs one, so its inputs wait for nothing.
passive: bool = False
@classmethod
def target_key(cls, params: dict[str, Any]) -> str | None:
"""Which outside thing these parameters point at, if any.
backend/fluksio/flow/nodes/device.py
+87
View File
@@ -0,0 +1,87 @@
"""Device: a paired device's own ports, as a node in a flow.
A wall panel's button, its backlight, its microphone. The device publishes on
its ``in`` ports and follows its ``out`` ports, and this node is where a flow
says which message each one is: its outputs are the ``in`` ports, its inputs
the ``out`` ports, each named by the port.
The engine never runs it. What it provides is published by the device itself,
through its own credential, and what it requires is read by the device off its
socket. The same wiring is what that credential may reach, and what the device
reads back as its ``io`` (see ``fluksio.flow.devices``).
"""
from __future__ import annotations
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any
from pydantic import BaseModel, Field
from fluksio.flow.devices import PORT_PATTERN, presence
from fluksio.flow.messages import MessageSpec
from fluksio.flow.nodes.base import Node
if TYPE_CHECKING:
from fastapi import FastAPI
def _never(**_: Any) -> None:
"""What the node would compute, were it ever run: nothing."""
class DeviceNode(Node):
"""A device's ports, wired to messages. Down while the device is away."""
passive = True
class Params(BaseModel):
device: str = Field(default="", description="The device this node stands for.")
config: dict[str, dict[str, Any]] = Field(
default_factory=dict,
description=(
"Per port, settings handed to the device as they are, such as a "
"pin's pull-up or how long the screen stays lit."
),
)
__slots__ = ("cfg",)
def __init__(
self,
requires: MessageSpec | Iterable[MessageSpec] = (),
provides: MessageSpec | Iterable[MessageSpec] = (),
params: dict[str, Any] | None = None,
name: str | None = None,
):
self.cfg = self.Params.model_validate(params or {})
super().__init__(
f=_never, requires=requires, provides=provides, params=params, name=name
)
for spec in [*self.input_ports, *self.output_ports]:
if not PORT_PATTERN.fullmatch(spec.port):
raise ValueError(
f"'{spec.port}' is not a port name. Use lowercase letters, "
"digits, underscores and dots, such as gpio.4"
)
@classmethod
def target_key(cls, params: dict[str, Any]) -> str | None:
"""One neuron per device, however many flows wire its ports."""
return str(params.get("device") or "") or None
async def start(self, app: FastAPI | None = None) -> None:
if not self.cfg.device:
return
presence.watch(self.cfg.device, self._presence)
self._presence(presence.online(self.cfg.device))
async def stop(self, app: FastAPI | None = None) -> None:
if self.cfg.device:
presence.unwatch(self.cfg.device, self._presence)
def _presence(self, online: bool) -> None:
if online:
self.report_health("ok")
else:
self.report_health("down", "Offline")
backend/fluksio/flow/pipeline.py
+20 -8
View File
@@ -27,7 +27,7 @@ from datetime import UTC, datetime
from functools import lru_cache
from typing import Any, Literal, Protocol
from pydantic import BaseModel, computed_field
from pydantic import BaseModel, model_validator
from fluksio.flow import logs
from fluksio.flow.artifacts import is_reference
@@ -63,6 +63,8 @@ class ValidationIssue(BaseModel):
"missing_source",
"node_unhealthy",
"ignored_device",
"unknown_device",
"device_port",
]
message: str
flow: str = ""
@@ -70,12 +72,15 @@ class ValidationIssue(BaseModel):
node: str | None = None
port: str | None = None
message_name: str | None = None
#: Worth saying, but not a fault — the UI says so in a softer tone. Set
#: by the code for most, by the issue itself where one code covers both.
advisory: bool = False
@computed_field # type: ignore[prop-decorator]
@property
def advisory(self) -> bool:
"""Worth saying, but not a fault — the UI says so in a softer tone."""
return self.code in ADVISORY_ISSUES
@model_validator(mode="after")
def _advisory_by_code(self) -> ValidationIssue:
if self.code in ADVISORY_ISSUES:
self.advisory = True
return self
class ValueSource(BaseModel):
@@ -229,7 +234,8 @@ def _derive(
for node in nodes:
for msg in node.provides:
produces.setdefault(msg, []).append(node)
for msg in node.requires:
# A passive node is never run, so nothing published wakes it.
for msg in () if node.passive else node.requires:
consumes.setdefault(msg, []).append(node)
by_id[node.id] = node
@@ -557,7 +563,8 @@ class Pipeline:
)
for node in nodes:
for msg_name, spec in node.requires.items():
# A passive node never runs, so it waits for nothing.
for msg_name, spec in () if node.passive else node.requires.items():
if msg_name in produces:
# A message a node both reads and writes carries state
# between its runs. If the node is the only one writing it,
@@ -1369,6 +1376,11 @@ class Pipeline:
# not spend the synchronous claim it needs once it may run.
if self._gate_blocks(n) or in_degree[n] != 0:
continue
if n.passive:
# Something outside publishes for it; a run is not it.
complete(n)
progressed = True
continue
if fresh is not None and untouched(n):
# Ahead of the readiness check, so an unchanged node
# spends neither a delivery stamp nor a synchronous
backend/fluksio/main.py
+3
View File
@@ -354,6 +354,9 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
# Once, for an instance from before devices: its panels file moves
# into the repository and the nonce file, and is renamed `.migrated`.
await run_in_threadpool(devices.migrate, settings.PANELS_FILE)
# Once, for an instance from before device nodes: each device's own
# ports become a device node in the flow declaring their message.
await run_in_threadpool(devices.move_io)
app.state.device_store = devices
controller.devices = devices
# Charts need a deeper series than the default; tell the engine
backend/tests/api/routes/test_devices.py
+264 -32
View File
@@ -7,6 +7,7 @@ from fastapi.testclient import TestClient
from fluksio.api.routes import devices as devices_route
from fluksio.cloud import config as cloud_config
from fluksio.core.config import settings
from fluksio.flow.devices import DeviceState
from fluksio.main import app
PREFIX = f"{settings.API_V1_STR}/devices"
@@ -522,36 +523,115 @@ def test_a_locked_dashboard_entitles_a_panel_to_read_and_not_to_publish(
)
PANEL_IO = [
{
"port": "gpio.4",
"message": "demo.doorbell",
"direction": "in",
"config": {"pull": "up", "debounce_ms": 50},
},
{"port": "backlight", "message": "demo.backlight", "direction": "out"},
]
FLOWS = f"{settings.API_V1_STR}/flows"
def test_a_panels_io_is_saved_and_read_back(
def _device_node(device: str, node: str = "porch", **ports: list) -> dict:
"""A device node: ``provides`` its ``in`` ports, ``requires`` its ``out`` ones."""
return {
"id": node,
"type": "device",
"params": {"device": device, "config": ports.pop("config", {})},
"provides": ports.get("provides", []),
"requires": ports.get("requires", []),
}
def _publish_flow(
client: TestClient, headers: dict[str, str], name: str, nodes: list, **more
) -> dict:
"""Write a flow and publish it, replacing whatever was there."""
client.delete(f"{FLOWS}/{name}", headers=headers)
saved = client.put(
f"{FLOWS}/{name}",
headers=headers,
json={"name": name, "nodes": nodes, "version": 0, **more},
)
assert saved.status_code == 200, saved.text
published = client.post(
f"{FLOWS}/{name}/publish",
headers=headers,
json={"version": saved.json()["definition"]["version"]},
)
assert published.status_code == 200, published.text
return published.json()
#: A doorbell button and the backlight, as a device node in flow ``porch``.
PORCH_NODE = _device_node(
"porch_io",
provides=[{"name": "doorbell", "port": "gpio.4", "dtype": "bool"}],
requires=[
{"name": "backlight", "port": "backlight", "dtype": "int", "trigger": False}
],
config={"gpio.4": {"pull": "up", "debounce_ms": 50}},
)
def test_a_devices_io_comes_from_its_device_nodes(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
_panels(
client,
superuser_token_headers,
{"panels": [{"id": "porch_io", "io": PANEL_IO}]},
"""What the device reads as ``io`` is what published flows wire, in port order.
The same bytes for as long as nothing changes: the firmware compares the
whole document and rebuilds its screen on any difference.
"""
_panels(client, superuser_token_headers, {"panels": [{"id": "porch_io"}]})
_publish_flow(client, superuser_token_headers, "porch", [PORCH_NODE])
# A draft wires nothing until it is published.
client.put(
f"{FLOWS}/porch_draft",
headers=superuser_token_headers,
json={
"name": "porch_draft",
"nodes": [
_device_node(
"porch_io",
provides=[{"name": "mic", "port": "mic", "dtype": "audio"}],
)
],
"version": 0,
},
)
read = client.get(f"{PREFIX}/porch_io", headers=superuser_token_headers).json()
assert read["io"] == [PANEL_IO[0], {**PANEL_IO[1], "config": {}}]
first = client.get(f"{PREFIX}/porch_io", headers=superuser_token_headers)
read = first.json()
assert read["io"] == [
{
"port": "backlight",
"message": "porch.backlight",
"direction": "out",
"config": {},
},
{
"port": "gpio.4",
"message": "porch.doorbell",
"direction": "in",
"config": {"pull": "up", "debounce_ms": 50},
},
]
assert [(b["flow"], b["node"], b["port"]) for b in read["bindings"]] == [
("porch", "porch", "backlight"),
("porch", "porch", "gpio.4"),
]
again = client.get(f"{PANELS}/porch_io", headers=superuser_token_headers)
assert again.content == first.content
# The list carries the same, for the Devices tab.
listed = client.get(f"{PREFIX}/", headers=superuser_token_headers).json()
assert next(d for d in listed["data"] if d["id"] == "porch_io")["io"] == read["io"]
for flow in ("porch", "porch_draft"):
client.delete(f"{FLOWS}/{flow}", headers=superuser_token_headers)
def test_a_panels_own_ports_publish_through_a_lock(
def test_a_devices_own_ports_publish_through_a_lock(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
"""An ``in`` port is the device's own sensor, which a dashboard lock is not about.
An ``out`` port is the device following a message, so it reads and does
not write — and only an ``in`` port's name opens the artifact upload.
not write, and only an ``in`` port's name opens the artifact upload. A
value published on a bound port is the device node's, whatever the device
says it is from.
"""
_dashboard_with(client, superuser_token_headers, "panel_io_locked", PANEL_WIDGETS)
_publish_settings(
@@ -560,36 +640,65 @@ def test_a_panels_own_ports_publish_through_a_lock(
_panels(
client,
superuser_token_headers,
{"panels": [{"id": "door", "dashboards": ["panel_io_locked"], "io": PANEL_IO}]},
{
"panels": [
{"id": "porch_io", "title": "Porch", "dashboards": ["panel_io_locked"]}
]
},
)
panel_headers = _pair(client, superuser_token_headers, "door")
_publish_flow(client, superuser_token_headers, "porch", [PORCH_NODE])
panel_headers = _pair(client, superuser_token_headers, "porch_io")
messages = f"{settings.API_V1_STR}/messages"
# 404: past the gate, refused by an engine no flow here declares it in.
controller = app.state.flow_controller
sources = []
original = controller.publish_message
def recorded(name, value, source=None): # type: ignore[no-untyped-def]
sources.append(source)
return original(name, value, source)
controller.publish_message = recorded
try:
published = client.post(
f"{messages}/porch.doorbell",
headers=panel_headers,
json={"value": True, "source_kind": "panel", "source_id": "panel:x:y"},
)
finally:
del controller.publish_message
assert published.status_code == 200, published.text
assert sources[0].model_dump() == {
"kind": "node",
"id": "porch.porch",
"label": "Porch",
"detail": "gpio.4",
}
assert (
client.post(
f"{messages}/demo.doorbell", headers=panel_headers, json={"value": True}
).status_code
== 404
)
assert (
client.post(
f"{messages}/demo.backlight", headers=panel_headers, json={"value": 50}
f"{messages}/porch.backlight", headers=panel_headers, json={"value": 50}
).status_code
== 403
)
assert (
client.get(
f"{messages}/demo.backlight/history", headers=panel_headers
f"{messages}/porch.backlight/history", headers=panel_headers
).status_code
== 200
)
assert (
client.post(
f"{messages}/porch.unrelated", headers=panel_headers, json={"value": 1}
).status_code
== 403
)
artifacts = f"{settings.API_V1_STR}/artifacts"
for name, status in (
("demo.doorbell", 200),
("demo.backlight", 403),
("demo.unrelated", 403),
("porch.doorbell", 200),
("porch.backlight", 403),
("porch.unrelated", 403),
):
uploaded = client.put(
artifacts,
@@ -599,6 +708,129 @@ def test_a_panels_own_ports_publish_through_a_lock(
)
assert uploaded.status_code == status, name
# Unwired, the port is nothing the credential reaches any more.
_publish_flow(client, superuser_token_headers, "porch", [])
assert (
client.post(
f"{messages}/porch.doorbell", headers=panel_headers, json={"value": True}
).status_code
== 403
)
client.delete(f"{FLOWS}/porch", headers=superuser_token_headers)
def test_a_port_wired_twice_or_an_unknown_device_is_an_issue(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
_panels(client, superuser_token_headers, {"panels": [{"id": "porch_io"}]})
_publish_flow(client, superuser_token_headers, "porch", [PORCH_NODE])
_publish_flow(
client,
superuser_token_headers,
"porch_too",
[
_device_node(
"porch_io",
provides=[{"name": "bell", "port": "gpio.4", "dtype": "bool"}],
),
_device_node("ghost", node="ghost"),
],
)
def codes(flow: str) -> set[tuple[str, str | None, bool]]:
issues = client.get(f"{FLOWS}/{flow}", headers=superuser_token_headers).json()[
"issues"
]
return {
(i["code"], i["port"], i["advisory"])
for i in issues
if i["code"] in ("device_port", "unknown_device")
}
assert codes("porch") == {("device_port", "gpio.4", False)}
assert codes("porch_too") == {
("device_port", "gpio.4", False),
("unknown_device", None, False),
}
# Creating the device is enough, nothing has to be published again.
_panels(
client,
superuser_token_headers,
{"panels": [{"id": "porch_io"}, {"id": "ghost"}]},
)
assert codes("porch_too") == {("device_port", "gpio.4", False)}
# With a report a port it does not list is a warning, and one it lists the
# other way round an error. Seen in a draft, which is validated as it is.
store = app.state.device_store
with store._state_lock:
states = store._read_states()
states.setdefault("ghost", DeviceState()).report = {
"schema": 1,
"ports": [{"port": "touch", "direction": "in", "dtype": "bool"}],
}
store._write_states(states)
client.put(
f"{FLOWS}/porch_too",
headers=superuser_token_headers,
json={
"name": "porch_too",
"version": client.get(
f"{FLOWS}/porch_too", headers=superuser_token_headers
).json()["definition"]["version"],
"nodes": [
_device_node(
"ghost",
node="ghost",
provides=[{"name": "relay", "port": "relay", "dtype": "bool"}],
requires=[
{
"name": "touch",
"port": "touch",
"dtype": "bool",
"trigger": False,
}
],
)
],
},
)
assert codes("porch_too") == {
("device_port", "relay", True),
("device_port", "touch", False),
}
for flow in ("porch", "porch_too"):
client.delete(f"{FLOWS}/{flow}", headers=superuser_token_headers)
def test_publishing_a_flow_with_device_nodes_tells_the_devices(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
"""An empty dashboard name is what a device refetches its document on."""
_panels(client, superuser_token_headers, {"panels": [{"id": "porch_io"}]})
controller = app.state.flow_controller
sent = []
original = controller._publish
def recorded(event): # type: ignore[no-untyped-def]
sent.append(event)
original(event)
controller._publish = recorded
try:
_publish_flow(client, superuser_token_headers, "porch", [PORCH_NODE])
told = [e for e in sent if e["type"] == "dashboard_changed"]
assert told and told[-1]["dashboard"] == ""
sent.clear()
# A flow with no device in it moves nothing a device reads.
_publish_flow(client, superuser_token_headers, "porch_plain", [])
assert not [e for e in sent if e["type"] == "dashboard_changed"]
client.delete(f"{FLOWS}/porch", headers=superuser_token_headers)
assert [e for e in sent if e["type"] == "dashboard_changed"]
finally:
del controller._publish
client.delete(f"{FLOWS}/porch_plain", headers=superuser_token_headers)
def test_a_panel_may_read_its_dashboards_own_settings(
client: TestClient, superuser_token_headers: dict[str, str]
backend/tests/flow/test_device_node.py
+94
View File
@@ -0,0 +1,94 @@
"""A device node: never run by the engine, down while its device is away."""
import asyncio
from pathlib import Path
from fluksio.flow.controller import FlowController
from fluksio.flow.devices import presence
from fluksio.flow.messages import DType, MessageSpec
from fluksio.flow.schemas import FlowDef, FlowInput, NodeDef
from fluksio.flow.store import FlowStore
def _controller(tmp_path: Path) -> FlowController:
store = FlowStore(tmp_path / "flows")
store.write_flow(
FlowDef(
name="hall",
inputs=[
FlowInput(spec=MessageSpec(name="level", dtype=DType.INT), initial=50)
],
nodes=[
NodeDef(
id="panel",
type="device",
params={"device": "wall"},
provides=[
MessageSpec(name="bell", port="gpio.4", dtype=DType.BOOL)
],
requires=[
MessageSpec(
name="level",
port="backlight",
dtype=DType.INT,
trigger=False,
)
],
),
NodeDef(
id="ring",
type="change",
requires=[MessageSpec(name="bell", dtype=DType.BOOL)],
provides=[MessageSpec(name="rung", dtype=DType.BOOL)],
),
],
)
)
return FlowController(store)
def test_the_engine_never_runs_a_device_node(tmp_path: Path) -> None:
controller = _controller(tmp_path)
asyncio.run(controller.reload())
assert controller.issues == []
node = controller.get_node("hall.panel")
assert node is not None and node.passive
ran: list[object] = []
node.f = lambda **kwargs: ran.append(kwargs) # type: ignore[method-assign]
# The message its out port follows changes, and a manual run of the flow.
controller.publish_message("hall.level", 80)
controller.run_flow("hall")
assert ran == []
# What the device publishes on its in port runs what reads it.
controller.publish_message("hall.bell", True)
assert controller.values("hall")["hall.rung"]["value"] is True
asyncio.run(controller._teardown())
def test_a_device_node_is_down_while_its_device_is_away(tmp_path: Path) -> None:
controller = _controller(tmp_path)
async def scenario() -> list[tuple[str, str | None]]:
await controller.reload()
entry = controller.loaded["hall.panel"]
seen = [(entry.health, entry.health_detail)]
presence.opened("wall")
seen.append((entry.health, entry.health_detail))
presence.closed("wall")
seen.append((entry.health, entry.health_detail))
await controller._teardown()
# Stopped, it no longer follows the device.
presence.opened("wall")
seen.append((entry.health, entry.health_detail))
presence.closed("wall")
return seen
assert asyncio.run(scenario()) == [
("down", "Offline"),
("ok", None),
("down", "Offline"),
("down", "Offline"),
]
presence.forget("wall")
backend/tests/flow/test_device_store.py
+183
View File
@@ -56,3 +56,186 @@ def test_panels_move_into_devices_once(tmp_path: Path) -> None:
text=True,
).stdout
)
def _flow(flows: FlowStore, name: str, draft: dict | None = None, **published) -> None:
"""Write a published flow document, and a draft beside it if given."""
folder = flows.root / name
folder.mkdir(parents=True, exist_ok=True)
(folder / "flow.json").write_text(json.dumps({"name": name, **published}))
if draft is not None:
(folder / "flow.draft.json").write_text(json.dumps({"name": name, **draft}))
def test_device_ports_move_into_flows_once(tmp_path: Path) -> None:
flows = FlowStore(tmp_path / "flows")
_flow(
flows,
"home",
version=3,
inputs=[
{"spec": {"name": "brightness", "dtype": "int"}, "initial": 80},
],
nodes=[
{
"id": "bell",
"type": "python",
"requires": [{"name": "doorbell", "dtype": "bool"}],
}
],
draft={
"version": 5,
"inputs": [{"spec": {"name": "brightness", "dtype": "int"}, "initial": 80}],
"nodes": [
{
"id": "bell",
"type": "python",
"requires": [{"name": "doorbell", "dtype": "bool"}],
},
{"id": "hall", "type": "inject"},
],
},
)
# Declares `home.light` from outside: a row on it goes to the owner first,
# and only here when the owner does not declare it.
_flow(
flows,
"lights",
nodes=[
{
"id": "dim",
"type": "python",
"provides": [{"name": "home.light", "dtype": "float"}],
}
],
)
devices = flows.root / "_devices"
devices.mkdir()
(devices / "hall.json").write_text(
json.dumps(
{
"id": "hall",
"title": "Hall",
"version": 4,
"io": [
{
"port": "backlight",
"message": "home.brightness",
"direction": "out",
},
{
"port": "gpio.4",
"message": "home.doorbell",
"direction": "in",
"config": {"pull": "up"},
},
{"port": "dimmer", "message": "home.light", "direction": "out"},
{"port": "mic", "message": "home.nowhere", "direction": "in"},
],
}
)
)
store = DeviceStore(flows, tmp_path / "devices.json")
store.move_io()
published = flows.read_flow("home")
draft = flows.read_flow("home", draft=True)
assert (published.version, draft.version) == (4, 6)
for doc in (published, draft):
# `hall` is taken by an inject node in the draft, so the id moves on in
# both documents alike.
node = next(n for n in doc.nodes if n.type == "device")
assert node.id == "hall_2" and node.title == "Hall"
assert node.params == {"device": "hall", "config": {"gpio.4": {"pull": "up"}}}
assert [(s.port, s.name, s.dtype.value) for s in node.provides] == [
("gpio.4", "doorbell", "bool")
]
assert [(s.port, s.name, s.dtype.value, s.trigger) for s in node.requires] == [
("backlight", "brightness", "int", False)
]
lights = flows.read_flow("lights")
node = next(n for n in lights.nodes if n.type == "device")
assert node.id == "hall" and lights.version == 2
assert [(s.port, s.name) for s in node.requires] == [("dimmer", "home.light")]
# The device lost `io` and kept its version; the row nothing declares is
# gone. What it reads back is the same rows, in port order.
assert "io" not in json.loads((devices / "hall.json").read_text())
assert store.find("hall").version == 4 # type: ignore[union-attr]
assert [(r.port, r.message, r.direction, r.config) for r in store.io("hall")] == [
("backlight", "home.brightness", "out", {}),
("dimmer", "home.light", "out", {}),
("gpio.4", "home.doorbell", "in", {"pull": "up"}),
]
log = subprocess.run(
["git", "-C", str(flows.root), "log", "--format=%s"],
capture_output=True,
text=True,
).stdout.splitlines()
assert log[0] == "Move device ports into flows"
# Nothing left to move, and nothing written twice.
store.move_io()
assert flows.read_flow("home").version == 4
assert (
subprocess.run(
["git", "-C", str(flows.root), "log", "--format=%s"],
capture_output=True,
text=True,
).stdout.splitlines()
== log
)
def test_a_crash_after_the_flows_were_written_adds_nothing_twice(
tmp_path: Path,
) -> None:
flows = FlowStore(tmp_path / "flows")
_flow(flows, "home", inputs=[{"spec": {"name": "level", "dtype": "int"}}])
devices = flows.root / "_devices"
devices.mkdir()
device = {
"id": "hall",
"io": [{"port": "backlight", "message": "home.level", "direction": "out"}],
}
(devices / "hall.json").write_text(json.dumps(device))
store = DeviceStore(flows, tmp_path / "devices.json")
store.move_io()
# As if the device file had not been rewritten before the process died.
(devices / "hall.json").write_text(json.dumps(device))
store.move_io()
node = next(n for n in flows.read_flow("home").nodes if n.type == "device")
assert len(node.requires) == 1
assert flows.read_flow("home").version == 2
def test_panels_with_io_reach_the_flows_in_one_start(tmp_path: Path) -> None:
"""An instance from before devices: panels.json, then device nodes."""
flows = FlowStore(tmp_path / "flows")
_flow(flows, "kiosk", inputs=[{"spec": {"name": "brightness", "dtype": "int"}}])
panels = tmp_path / "panels.json"
panels.write_text(
json.dumps(
{
"panels": [
{
"id": "knob",
"io": [
{
"port": "backlight",
"message": "kiosk.brightness",
"direction": "out",
"config": {"wake_s": 30},
}
],
}
]
}
)
)
store = DeviceStore(flows, tmp_path / "devices.json")
store.migrate(panels)
store.move_io()
assert [(r.port, r.message, r.config) for r in store.io("knob")] == [
("backlight", "kiosk.brightness", {"wake_s": 30})
]
backend/tests/flow/test_endpoints.py
-27
View File
@@ -11,7 +11,6 @@ import pytest
from fluksio.api.routes.flows import _endpoints
from fluksio.flow.controller import FlowController
from fluksio.flow.devices import DeviceDef, DevicePort, DeviceStore
from fluksio.flow.messages import DType, MessageSpec
from fluksio.flow.schemas import FlowDef, NodeDef
from fluksio.flow.store import FlowStore
@@ -110,29 +109,3 @@ def test_the_wiring_index_is_read_once_per_store_revision(controller):
controller.store.write_flow(FlowDef(name="shed", nodes=[]))
_endpoints(controller, controller.store.read_flow("house"))
assert reads == 2
def test_a_device_port_is_drawn_on_the_side_it_wires(controller, tmp_path):
controller.devices = DeviceStore(controller.store, tmp_path / "devices.json")
controller.devices.create(
DeviceDef(
id="hall",
title="Hall",
io=[
DevicePort(port="gpio.4", message="house.temp", direction="in"),
DevicePort(port="backlight", message="house.temp", direction="out"),
DevicePort(port="mic", message="garage.open", direction="in"),
],
)
)
endpoints = {
one.id: one
for one in _endpoints(controller, controller.store.read_flow("house"))
}
# The id is what the device publishes under, so its edges pulse for it.
assert endpoints["panel:hall:gpio.4"].provides == ["house.temp"]
assert endpoints["panel:hall:backlight"].requires == ["house.temp"]
assert endpoints["panel:hall:backlight"].detail == "Hall"
assert "panel:hall:mic" not in endpoints
backend/tests/flow/test_node_types.py
+7
View File
@@ -100,6 +100,13 @@ FIXTURES: dict[str, dict] = {
"requires": [MessageSpec(name="prompt", dtype=DType.STR)],
"provides": [MessageSpec(name="kitchen_light", dtype=DType.BOOL)],
},
"device": {
"params": {"device": "hall", "config": {"gpio.4": {"pull": "up"}}},
"requires": [
MessageSpec(name="level", port="backlight", dtype=DType.INT, trigger=False)
],
"provides": [MessageSpec(name="bell", port="gpio.4", dtype=DType.BOOL)],
},
}