Cameras on devices (openpanel C1): frames in memory, a binary uplink, demand

- Volatile frames are held in memory with a byte-bounded queue instead of
  the /dev/shm ring scanned on every put; workers keep writing to the
  directory, which is trimmed on a timer. A volatile upload no longer spools
  to disk. ARTIFACT_VOLATILE_BYTES=0 turns the ring off.
- A device sends camera frames as binary messages on its socket, held and
  published in one step and credited to its device node; refused frames are
  logged and dropped. Its snapshot carries demand and the uplink (binary, or
  http through the portal).
- One demand registry for every socket's wanted media, the tunnel's
  included, plus what running flows read, mapped onto on-demand ports and
  published as device_demand; at most 8 media names per socket.
- Per-message deflate off in serve and dev; a device may fetch the last 8
  digests of a message; published dashboards cached per store revision; a
  revoked device socket is closed with 1008.
- Media tiles want bytes only while on screen; the Devices tab previews a
  wired camera; the device node warns when a flow reads an on-demand port.
This commit is contained in:
Melvin Strobl
2026-10-16 20:43:14 +02:00
parent d79203a5f7
commit ecc4bee4bb
29 changed files with 1425 additions and 220 deletions
+107 -39
View File
@@ -19,7 +19,9 @@ import hashlib
import logging
import os
import tempfile
import threading
import time
from collections import OrderedDict
from collections.abc import Iterable, Iterator
from pathlib import Path
from typing import Any
@@ -52,9 +54,10 @@ class ArtifactStore:
def __init__(self, root: Path) -> None:
self.root = root
self.root.mkdir(parents=True, exist_ok=True)
#: A ring of frames held in memory, for media a flow only shows live.
#: Set by the engine at startup; every lookup below falls through to
#: it, which is what lets a volatile reference be an ordinary one.
#: Frames held in memory, for media a flow only shows live. Set by the
#: engine at startup unless the ring is turned off; every lookup below
#: falls through to it, which is what lets a volatile reference be an
#: ordinary one.
self.volatile: VolatileStore | None = None
def _path(self, digest: str) -> Path:
@@ -79,7 +82,8 @@ class ArtifactStore:
``volatile`` puts it in the ring instead, where it is held in memory
and falls out once newer frames need the room. The reference is the
same shape either way — what differs is how long the bytes last, and a
caller that wants one kept returns it from a run.
caller that wants one kept returns it from a run. Without a ring it
lands here like anything else.
"""
if volatile and self.volatile is not None:
return self.volatile.put(chunks, name=name, media_type=media_type)
@@ -128,20 +132,26 @@ class ArtifactStore:
)
def path(self, digest: str) -> Path | None:
"""Where the bytes are, or None if this store does not have them.
"""Where the stored bytes are, or None if this store does not have them.
The ring is looked in second, so everything that resolves a digest —
serving one over HTTP, checking a run input still exists, a panel's
scope — reaches a live frame without knowing there are two stores.
Only the durable store: a live frame has no file. ``has`` and ``read``
are what reach both.
"""
if not valid_digest(digest):
return None
target = self._path(digest)
if target.exists():
return target
if self.volatile is not None:
return self.volatile.path(digest)
return None
return target if target.exists() else None
def has(self, digest: str) -> bool:
"""Whether the bytes are here, stored or held in the ring.
Everything that only asks whether a digest resolves — a run input
that has to still exist, a cached result — reaches a live frame
without knowing there are two stores.
"""
if self.path(digest) is not None:
return True
return self.volatile is not None and self.volatile.has(digest)
def adopt(self, digest: str) -> bool:
"""Copy a volatile artifact into this store, so it outlives the ring.
@@ -154,10 +164,10 @@ class ArtifactStore:
return False
if self._path(digest).exists():
return True
source = self.volatile.path(digest)
if source is None:
data = self.volatile.get(digest)
if data is None:
return False
self.put_file(source)
self.put([data])
return True
def delete(self, digest: str) -> bool:
@@ -176,7 +186,11 @@ class ArtifactStore:
def read(self, digest: str) -> Iterator[bytes]:
target = self.path(digest)
if target is None:
raise FileNotFoundError(digest)
data = self.volatile.get(digest) if self.volatile is not None else None
if data is None:
raise FileNotFoundError(digest)
yield data
return
with target.open("rb") as handle:
while chunk := handle.read(CHUNK):
yield chunk
@@ -211,44 +225,98 @@ class ArtifactStore:
return removed
class VolatileStore(ArtifactStore):
"""A bounded ring of artifacts, held wherever memory is cheaper than disk.
class VolatileStore:
"""Live frames, held in memory and bounded by size.
A camera publishing ten frames a second is ten files a second, and on the
wall panel this is built for that disk is an SD card. So the frames go to a
memory-backed directory instead and the oldest fall out once the ring is
full: nothing sweeps it, because a frame nobody kept is not worth a pass
over the store to find.
A camera publishing five frames a second is five artifacts a second that
nobody keeps, so they never reach a disk: each one is held here in the
order it came and the oldest fall out once the bound is reached. Nothing
sweeps it, and a ``put`` costs the same however full it is.
It is an ``ArtifactStore``, digest layout and all, which is what lets a
frame be an ordinary reference — the dtype check, the panel's scope, a
node opening one with ``load_artifact`` and the widget fetching one all
work on it unchanged.
``root`` is a memory-backed directory beside it for the frames a worker in
this container writes itself (``FLUKSIO_ARTIFACT_VOLATILE_DIR``), since a
worker is another process. Read from there once and held here after, so
a frame pushed to every open screen is read from the file once. The
engine trims that directory on a timer.
A frame is an ordinary reference all the same: the dtype check, the
device's scope, a node opening one with ``load_artifact`` and the widget
fetching one all work on it unchanged, through ``ArtifactStore``.
"""
def __init__(self, root: Path, limit_bytes: int) -> None:
super().__init__(root)
self.root = root
self.root.mkdir(parents=True, exist_ok=True)
self.limit_bytes = limit_bytes
self._frames: OrderedDict[str, bytes] = OrderedDict()
self._size = 0
self._lock = threading.Lock()
def put(
self,
chunks: Iterable[bytes],
name: str = "",
media_type: str = "",
volatile: bool = False,
) -> dict[str, Any]:
reference = super().put(chunks, name=name, media_type=media_type)
self.trim()
return reference
"""Hold a frame and return the reference to it."""
data = b"".join(chunks)
digest = DIGEST_PREFIX + hashlib.sha256(data).hexdigest()
self._hold(digest, data)
return {
"digest": digest,
"size": len(data),
"media_type": media_type or "application/octet-stream",
"name": name,
}
def _hold(self, digest: str, data: bytes) -> None:
with self._lock:
if digest in self._frames:
# The same bytes again are the newest frame again.
self._frames.move_to_end(digest)
return
self._frames[digest] = data
self._size += len(data)
# The one just held stays, even past the bound: its reference is
# about to be published.
while self._size > self.limit_bytes and len(self._frames) > 1:
_, dropped = self._frames.popitem(last=False)
self._size -= len(dropped)
def _file(self, digest: str) -> Path:
body = digest[len(DIGEST_PREFIX) :]
return self.root / body[:2] / body
def get(self, digest: str) -> bytes | None:
"""The frame's bytes, or None once it has fallen out."""
with self._lock:
data = self._frames.get(digest)
if data is not None or not valid_digest(digest):
return data
try:
data = self._file(digest).read_bytes()
except OSError:
return None
self._hold(digest, data)
return data
def has(self, digest: str) -> bool:
with self._lock:
if digest in self._frames:
return True
return valid_digest(digest) and self._file(digest).is_file()
@property
def size(self) -> int:
"""Bytes held in memory."""
return self._size
def trim(self) -> int:
"""Drop the oldest until the ring is inside its bound.
"""Drop the oldest of the workers' frames until the directory is
inside the bound. Returns how many went.
Also called on a timer, because a worker in this container writes here
itself and the engine never sees that ``put``.
# ponytail: a scandir per trim. An in-memory index if a ring of
# thousands of frames ever shows up in a profile.
A scan of the directory, so it runs on a timer and never per frame:
only frames a worker in this container wrote are there.
"""
if self.limit_bytes <= 0:
return 0
+25
View File
@@ -35,6 +35,7 @@ from fluksio_worker.worker_main import load_function
from fluksio.core.config import settings
from fluksio.flow.alerts import AlertManager
from fluksio.flow.demand import demand
from fluksio.flow.events import EventBus
from fluksio.flow.executor import ExecutionService
from fluksio.flow.messages import MessageSpec, flow_of, qualify
@@ -853,6 +854,9 @@ class FlowController:
)
finally:
self._lock.release()
# A device created, deleted or reporting changes whose ports send on
# demand.
demand.refresh()
self._publish(
{
"type": "pipeline_rebuilt",
@@ -868,9 +872,13 @@ class FlowController:
An empty dashboard name is what a device refetches its own document
on, and what its socket rescopes on. The first build only takes note:
a device connecting later reads the document as it is.
What a running flow reads may have moved with it, so demand on the
devices' ports is worked out again too.
"""
if self.devices is None:
return
demand.refresh()
wiring = self.devices.wiring()
known, self._device_wiring = self._device_wiring, wiring
if known is not None and known != wiring:
@@ -1488,6 +1496,23 @@ class FlowController:
def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]:
return self.pipeline.values(flow) if self.pipeline else {}
def consumed_messages(self) -> set[str]:
"""The messages a node of a running flow reads.
What keeps a device's on-demand port sending whether or not anyone is
watching: a node that reads a camera wants every frame. A stopped or
batch flow reads nothing until it runs, and a device node is passive.
"""
pipeline = self.pipeline
if pipeline is None:
return set()
idle = self.disabled | self.batch
return {
name
for name, nodes in pipeline.consumes.items()
if any(node.flow not in idle for node in nodes)
}
def get_node(self, node_id: str) -> Node | None:
entry = self.loaded.get(node_id)
return entry.node if entry else None
+22
View File
@@ -655,6 +655,10 @@ class DashboardStore:
self.root = flows.root / DASHBOARD_DIR
# Read-modify-write of the version counter, same as the flow store.
self._lock = threading.Lock()
#: Published documents as last read, until the flow store commits.
self._published: dict[str, DashboardDef | None] = {}
self._published_at = -1
self._published_lock = threading.Lock()
def _file(self, name: str) -> Path:
return self.root / checked(name) / "dashboard.json"
@@ -719,6 +723,24 @@ class DashboardStore:
update={"has_draft": self.has_draft(name)}
)
def published(self, name: str) -> DashboardDef | None:
"""The published document, or None, as of the store's last commit.
What a device's credential is held against on every request, a camera
fetching frame by frame included, so it is read once per commit rather
than from disk each time. Every write commits. The document is shared:
read it, never change it.
"""
with self._published_lock:
if self._published_at != self.flows.revision:
self._published, self._published_at = {}, self.flows.revision
if name not in self._published:
try:
self._published[name] = self.read(name)
except DashboardNotFound:
self._published[name] = None
return self._published[name]
def write(
self, defn: DashboardDef, base_version: int | None = None
) -> DashboardDef:
+114
View File
@@ -0,0 +1,114 @@
"""Which of a device's ports someone wants right now.
A camera is sent nothing while nobody watches it. A port the device reports as
``on_demand`` has demand while a viewer socket wants bytes for the message it
is wired to — a dashboard tile, the Devices tab's preview, a screen through the
portal — or a node of a running flow reads that message.
Every socket says what it wants here, in one registry, and the result is
published on the bus as a ``device_demand`` event whenever a device's set
changes. Each device socket picks out its own and tells the device, so nothing
here has to know which sockets belong to which device.
"""
from __future__ import annotations
import threading
import time
from collections.abc import Hashable
from typing import Any
from fluksio.flow.events import event_bus
class Demand:
"""The media names every live socket wants, and what that asks of devices.
One per process, as the sockets are. ``bind`` hands it the controller,
whose device store says what is wired and whose graph says what a flow
reads.
"""
def __init__(self) -> None:
self._lock = threading.Lock()
self._wanted: dict[Hashable, frozenset[str]] = {}
#: The last set published per device, so only a change is.
self._sent: dict[str, tuple[str, ...]] = {}
self._refresh_lock = threading.Lock()
self._controller: Any = None
def bind(self, controller: Any) -> None:
self._controller = controller
self.refresh()
def want(self, viewer: Hashable, names: set[str]) -> None:
"""What one socket wants bytes for now, replacing what it said before."""
with self._lock:
if names:
self._wanted[viewer] = frozenset(names)
else:
self._wanted.pop(viewer, None)
self.refresh()
def drop(self, viewer: Hashable) -> None:
"""A socket closed, and wants nothing any more."""
self.want(viewer, set())
def viewed(self) -> set[str]:
with self._lock:
return set().union(*self._wanted.values())
def ports(self, device_id: str) -> list[str]:
"""The device's ports that have demand, sorted."""
return list(self._compute().get(device_id, ()))
def refresh(self) -> None:
"""Work demand out again and publish what changed.
Called whenever its inputs move: a socket's wanted set, and a rebuild,
a started or stopped flow or a new report, which change what is wired
and what reads it.
"""
with self._refresh_lock:
found = self._compute()
for device_id in sorted(set(found) | set(self._sent)):
ports = found.get(device_id, ())
if ports == self._sent.get(device_id, ()):
continue
if ports:
self._sent[device_id] = ports
else:
self._sent.pop(device_id, None)
event_bus.publish(
{
"type": "device_demand",
"device": device_id,
"ports": list(ports),
"ts": time.time(),
}
)
def _compute(self) -> dict[str, tuple[str, ...]]:
controller = self._controller
devices = getattr(controller, "devices", None)
if devices is None:
return {}
wanted = self.viewed() | controller.consumed_messages()
found: dict[str, tuple[str, ...]] = {}
for device_id in devices.wiring():
reported = devices.reported_ports(device_id) or {}
ports = tuple(
sorted(
binding.port
for binding in devices.bound(device_id)
if binding.direction == "in"
and binding.message in wanted
and reported.get(binding.port, {}).get("on_demand") is True
)
)
if ports:
found[device_id] = ports
return found
demand = Demand()
+2 -8
View File
@@ -45,7 +45,7 @@ from pydantic import (
model_validator,
)
from fluksio.flow.dashboards import DashboardDef, DashboardNotFound, DashboardStore
from fluksio.flow.dashboards import DashboardDef, DashboardStore
from fluksio.flow.events import event_bus
from fluksio.flow.messages import (
DType,
@@ -985,13 +985,7 @@ def dashboards_for(
"""
if device is None:
return []
found: list[DashboardDef] = []
for name in device.dashboards:
try:
found.append(store.read(name))
except DashboardNotFound:
continue
return found
return [defn for name in device.dashboards if (defn := store.published(name))]
def messages_of(defn: DashboardDef) -> set[str]:
+15
View File
@@ -19,6 +19,8 @@ logger = logging.getLogger(__name__)
QUEUE_SIZE = 256
LOG_HISTORY = 400
#: How many of a message's last artifact digests are remembered.
RECENT_DIGESTS = 8
class EventBus:
@@ -42,6 +44,11 @@ class EventBus:
# zero and draw a busy graph as idle. A session count, not a metric —
# the rollups in `metrics.py` are what survives a restart.
self.emits: dict[str, int] = {}
#: The last few artifacts each message pointed at, newest last. A
#: screen fetching a camera frame by frame asks for one the message
#: may already have moved past, and a device's credential reaches
#: these as well as the current one.
self.digests: dict[str, deque[str]] = {}
def bind(self, loop: asyncio.AbstractEventLoop) -> None:
"""Attach the bus to the running event loop (called once at startup)."""
@@ -59,6 +66,14 @@ class EventBus:
# increment, which is a count nobody is going to miss.
node = str(event.get("node") or "")
self.emits[node] = self.emits.get(node, 0) + 1
elif kind == "message_value":
value = event.get("value")
if isinstance(value, dict) and isinstance(value.get("digest"), str):
name = str(event.get("name") or "")
recent = self.digests.get(name)
if recent is None:
recent = self.digests[name] = deque(maxlen=RECENT_DIGESTS)
recent.append(value["digest"])
loop = self._loop
if loop is None or not self._subscribers:
return
+3 -6
View File
@@ -306,7 +306,7 @@ def _from_run(
).all()
for row in rows:
if output in (row.name, row.name.rsplit(".", 1)[-1]):
if artifacts is not None and artifacts.path(row.digest) is None:
if artifacts is not None and not artifacts.has(row.digest):
raise RunRejected(
f"Parameter '{key}': run '{run_id}' made '{output}', but its "
"bytes are gone from this instance's store"
@@ -337,7 +337,7 @@ def _from_digest(
f"Parameter '{key}': no run has produced '{digest}', so there is "
"nothing here under that digest"
)
if artifacts is not None and artifacts.path(digest) is None:
if artifacts is not None and not artifacts.has(digest):
raise RunRejected(
f"Parameter '{key}': '{digest}' is known but its bytes are gone "
"from this instance's store"
@@ -829,10 +829,7 @@ class RunCache:
def _unrestorable(self, outputs: dict[str, Any]) -> bool:
return any(
is_reference(value)
and (
self.artifacts is None
or self.artifacts.path(str(value["digest"])) is None
)
and (self.artifacts is None or not self.artifacts.has(str(value["digest"])))
for value in outputs.values()
)