Cameras on devices (openpanel C1): frames in memory, a binary uplink, demand
- Volatile frames are held in memory with a byte-bounded queue instead of the /dev/shm ring scanned on every put; workers keep writing to the directory, which is trimmed on a timer. A volatile upload no longer spools to disk. ARTIFACT_VOLATILE_BYTES=0 turns the ring off. - A device sends camera frames as binary messages on its socket, held and published in one step and credited to its device node; refused frames are logged and dropped. Its snapshot carries demand and the uplink (binary, or http through the portal). - One demand registry for every socket's wanted media, the tunnel's included, plus what running flows read, mapped onto on-demand ports and published as device_demand; at most 8 media names per socket. - Per-message deflate off in serve and dev; a device may fetch the last 8 digests of a message; published dashboards cached per store revision; a revoked device socket is closed with 1008. - Media tiles want bytes only while on screen; the Devices tab previews a wired camera; the device node warns when a flow reads an on-demand port.
This commit is contained in:
@@ -19,7 +19,9 @@ import hashlib
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Iterable, Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -52,9 +54,10 @@ class ArtifactStore:
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
#: A ring of frames held in memory, for media a flow only shows live.
|
||||
#: Set by the engine at startup; every lookup below falls through to
|
||||
#: it, which is what lets a volatile reference be an ordinary one.
|
||||
#: Frames held in memory, for media a flow only shows live. Set by the
|
||||
#: engine at startup unless the ring is turned off; every lookup below
|
||||
#: falls through to it, which is what lets a volatile reference be an
|
||||
#: ordinary one.
|
||||
self.volatile: VolatileStore | None = None
|
||||
|
||||
def _path(self, digest: str) -> Path:
|
||||
@@ -79,7 +82,8 @@ class ArtifactStore:
|
||||
``volatile`` puts it in the ring instead, where it is held in memory
|
||||
and falls out once newer frames need the room. The reference is the
|
||||
same shape either way — what differs is how long the bytes last, and a
|
||||
caller that wants one kept returns it from a run.
|
||||
caller that wants one kept returns it from a run. Without a ring it
|
||||
lands here like anything else.
|
||||
"""
|
||||
if volatile and self.volatile is not None:
|
||||
return self.volatile.put(chunks, name=name, media_type=media_type)
|
||||
@@ -128,20 +132,26 @@ class ArtifactStore:
|
||||
)
|
||||
|
||||
def path(self, digest: str) -> Path | None:
|
||||
"""Where the bytes are, or None if this store does not have them.
|
||||
"""Where the stored bytes are, or None if this store does not have them.
|
||||
|
||||
The ring is looked in second, so everything that resolves a digest —
|
||||
serving one over HTTP, checking a run input still exists, a panel's
|
||||
scope — reaches a live frame without knowing there are two stores.
|
||||
Only the durable store: a live frame has no file. ``has`` and ``read``
|
||||
are what reach both.
|
||||
"""
|
||||
if not valid_digest(digest):
|
||||
return None
|
||||
target = self._path(digest)
|
||||
if target.exists():
|
||||
return target
|
||||
if self.volatile is not None:
|
||||
return self.volatile.path(digest)
|
||||
return None
|
||||
return target if target.exists() else None
|
||||
|
||||
def has(self, digest: str) -> bool:
|
||||
"""Whether the bytes are here, stored or held in the ring.
|
||||
|
||||
Everything that only asks whether a digest resolves — a run input
|
||||
that has to still exist, a cached result — reaches a live frame
|
||||
without knowing there are two stores.
|
||||
"""
|
||||
if self.path(digest) is not None:
|
||||
return True
|
||||
return self.volatile is not None and self.volatile.has(digest)
|
||||
|
||||
def adopt(self, digest: str) -> bool:
|
||||
"""Copy a volatile artifact into this store, so it outlives the ring.
|
||||
@@ -154,10 +164,10 @@ class ArtifactStore:
|
||||
return False
|
||||
if self._path(digest).exists():
|
||||
return True
|
||||
source = self.volatile.path(digest)
|
||||
if source is None:
|
||||
data = self.volatile.get(digest)
|
||||
if data is None:
|
||||
return False
|
||||
self.put_file(source)
|
||||
self.put([data])
|
||||
return True
|
||||
|
||||
def delete(self, digest: str) -> bool:
|
||||
@@ -176,7 +186,11 @@ class ArtifactStore:
|
||||
def read(self, digest: str) -> Iterator[bytes]:
|
||||
target = self.path(digest)
|
||||
if target is None:
|
||||
raise FileNotFoundError(digest)
|
||||
data = self.volatile.get(digest) if self.volatile is not None else None
|
||||
if data is None:
|
||||
raise FileNotFoundError(digest)
|
||||
yield data
|
||||
return
|
||||
with target.open("rb") as handle:
|
||||
while chunk := handle.read(CHUNK):
|
||||
yield chunk
|
||||
@@ -211,44 +225,98 @@ class ArtifactStore:
|
||||
return removed
|
||||
|
||||
|
||||
class VolatileStore(ArtifactStore):
|
||||
"""A bounded ring of artifacts, held wherever memory is cheaper than disk.
|
||||
class VolatileStore:
|
||||
"""Live frames, held in memory and bounded by size.
|
||||
|
||||
A camera publishing ten frames a second is ten files a second, and on the
|
||||
wall panel this is built for that disk is an SD card. So the frames go to a
|
||||
memory-backed directory instead and the oldest fall out once the ring is
|
||||
full: nothing sweeps it, because a frame nobody kept is not worth a pass
|
||||
over the store to find.
|
||||
A camera publishing five frames a second is five artifacts a second that
|
||||
nobody keeps, so they never reach a disk: each one is held here in the
|
||||
order it came and the oldest fall out once the bound is reached. Nothing
|
||||
sweeps it, and a ``put`` costs the same however full it is.
|
||||
|
||||
It is an ``ArtifactStore``, digest layout and all, which is what lets a
|
||||
frame be an ordinary reference — the dtype check, the panel's scope, a
|
||||
node opening one with ``load_artifact`` and the widget fetching one all
|
||||
work on it unchanged.
|
||||
``root`` is a memory-backed directory beside it for the frames a worker in
|
||||
this container writes itself (``FLUKSIO_ARTIFACT_VOLATILE_DIR``), since a
|
||||
worker is another process. Read from there once and held here after, so
|
||||
a frame pushed to every open screen is read from the file once. The
|
||||
engine trims that directory on a timer.
|
||||
|
||||
A frame is an ordinary reference all the same: the dtype check, the
|
||||
device's scope, a node opening one with ``load_artifact`` and the widget
|
||||
fetching one all work on it unchanged, through ``ArtifactStore``.
|
||||
"""
|
||||
|
||||
def __init__(self, root: Path, limit_bytes: int) -> None:
|
||||
super().__init__(root)
|
||||
self.root = root
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
self.limit_bytes = limit_bytes
|
||||
self._frames: OrderedDict[str, bytes] = OrderedDict()
|
||||
self._size = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def put(
|
||||
self,
|
||||
chunks: Iterable[bytes],
|
||||
name: str = "",
|
||||
media_type: str = "",
|
||||
volatile: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
reference = super().put(chunks, name=name, media_type=media_type)
|
||||
self.trim()
|
||||
return reference
|
||||
"""Hold a frame and return the reference to it."""
|
||||
data = b"".join(chunks)
|
||||
digest = DIGEST_PREFIX + hashlib.sha256(data).hexdigest()
|
||||
self._hold(digest, data)
|
||||
return {
|
||||
"digest": digest,
|
||||
"size": len(data),
|
||||
"media_type": media_type or "application/octet-stream",
|
||||
"name": name,
|
||||
}
|
||||
|
||||
def _hold(self, digest: str, data: bytes) -> None:
|
||||
with self._lock:
|
||||
if digest in self._frames:
|
||||
# The same bytes again are the newest frame again.
|
||||
self._frames.move_to_end(digest)
|
||||
return
|
||||
self._frames[digest] = data
|
||||
self._size += len(data)
|
||||
# The one just held stays, even past the bound: its reference is
|
||||
# about to be published.
|
||||
while self._size > self.limit_bytes and len(self._frames) > 1:
|
||||
_, dropped = self._frames.popitem(last=False)
|
||||
self._size -= len(dropped)
|
||||
|
||||
def _file(self, digest: str) -> Path:
|
||||
body = digest[len(DIGEST_PREFIX) :]
|
||||
return self.root / body[:2] / body
|
||||
|
||||
def get(self, digest: str) -> bytes | None:
|
||||
"""The frame's bytes, or None once it has fallen out."""
|
||||
with self._lock:
|
||||
data = self._frames.get(digest)
|
||||
if data is not None or not valid_digest(digest):
|
||||
return data
|
||||
try:
|
||||
data = self._file(digest).read_bytes()
|
||||
except OSError:
|
||||
return None
|
||||
self._hold(digest, data)
|
||||
return data
|
||||
|
||||
def has(self, digest: str) -> bool:
|
||||
with self._lock:
|
||||
if digest in self._frames:
|
||||
return True
|
||||
return valid_digest(digest) and self._file(digest).is_file()
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
"""Bytes held in memory."""
|
||||
return self._size
|
||||
|
||||
def trim(self) -> int:
|
||||
"""Drop the oldest until the ring is inside its bound.
|
||||
"""Drop the oldest of the workers' frames until the directory is
|
||||
inside the bound. Returns how many went.
|
||||
|
||||
Also called on a timer, because a worker in this container writes here
|
||||
itself and the engine never sees that ``put``.
|
||||
|
||||
# ponytail: a scandir per trim. An in-memory index if a ring of
|
||||
# thousands of frames ever shows up in a profile.
|
||||
A scan of the directory, so it runs on a timer and never per frame:
|
||||
only frames a worker in this container wrote are there.
|
||||
"""
|
||||
if self.limit_bytes <= 0:
|
||||
return 0
|
||||
|
||||
@@ -35,6 +35,7 @@ from fluksio_worker.worker_main import load_function
|
||||
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.flow.alerts import AlertManager
|
||||
from fluksio.flow.demand import demand
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.executor import ExecutionService
|
||||
from fluksio.flow.messages import MessageSpec, flow_of, qualify
|
||||
@@ -853,6 +854,9 @@ class FlowController:
|
||||
)
|
||||
finally:
|
||||
self._lock.release()
|
||||
# A device created, deleted or reporting changes whose ports send on
|
||||
# demand.
|
||||
demand.refresh()
|
||||
self._publish(
|
||||
{
|
||||
"type": "pipeline_rebuilt",
|
||||
@@ -868,9 +872,13 @@ class FlowController:
|
||||
An empty dashboard name is what a device refetches its own document
|
||||
on, and what its socket rescopes on. The first build only takes note:
|
||||
a device connecting later reads the document as it is.
|
||||
|
||||
What a running flow reads may have moved with it, so demand on the
|
||||
devices' ports is worked out again too.
|
||||
"""
|
||||
if self.devices is None:
|
||||
return
|
||||
demand.refresh()
|
||||
wiring = self.devices.wiring()
|
||||
known, self._device_wiring = self._device_wiring, wiring
|
||||
if known is not None and known != wiring:
|
||||
@@ -1488,6 +1496,23 @@ class FlowController:
|
||||
def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]:
|
||||
return self.pipeline.values(flow) if self.pipeline else {}
|
||||
|
||||
def consumed_messages(self) -> set[str]:
|
||||
"""The messages a node of a running flow reads.
|
||||
|
||||
What keeps a device's on-demand port sending whether or not anyone is
|
||||
watching: a node that reads a camera wants every frame. A stopped or
|
||||
batch flow reads nothing until it runs, and a device node is passive.
|
||||
"""
|
||||
pipeline = self.pipeline
|
||||
if pipeline is None:
|
||||
return set()
|
||||
idle = self.disabled | self.batch
|
||||
return {
|
||||
name
|
||||
for name, nodes in pipeline.consumes.items()
|
||||
if any(node.flow not in idle for node in nodes)
|
||||
}
|
||||
|
||||
def get_node(self, node_id: str) -> Node | None:
|
||||
entry = self.loaded.get(node_id)
|
||||
return entry.node if entry else None
|
||||
|
||||
@@ -655,6 +655,10 @@ class DashboardStore:
|
||||
self.root = flows.root / DASHBOARD_DIR
|
||||
# Read-modify-write of the version counter, same as the flow store.
|
||||
self._lock = threading.Lock()
|
||||
#: Published documents as last read, until the flow store commits.
|
||||
self._published: dict[str, DashboardDef | None] = {}
|
||||
self._published_at = -1
|
||||
self._published_lock = threading.Lock()
|
||||
|
||||
def _file(self, name: str) -> Path:
|
||||
return self.root / checked(name) / "dashboard.json"
|
||||
@@ -719,6 +723,24 @@ class DashboardStore:
|
||||
update={"has_draft": self.has_draft(name)}
|
||||
)
|
||||
|
||||
def published(self, name: str) -> DashboardDef | None:
|
||||
"""The published document, or None, as of the store's last commit.
|
||||
|
||||
What a device's credential is held against on every request, a camera
|
||||
fetching frame by frame included, so it is read once per commit rather
|
||||
than from disk each time. Every write commits. The document is shared:
|
||||
read it, never change it.
|
||||
"""
|
||||
with self._published_lock:
|
||||
if self._published_at != self.flows.revision:
|
||||
self._published, self._published_at = {}, self.flows.revision
|
||||
if name not in self._published:
|
||||
try:
|
||||
self._published[name] = self.read(name)
|
||||
except DashboardNotFound:
|
||||
self._published[name] = None
|
||||
return self._published[name]
|
||||
|
||||
def write(
|
||||
self, defn: DashboardDef, base_version: int | None = None
|
||||
) -> DashboardDef:
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Which of a device's ports someone wants right now.
|
||||
|
||||
A camera is sent nothing while nobody watches it. A port the device reports as
|
||||
``on_demand`` has demand while a viewer socket wants bytes for the message it
|
||||
is wired to — a dashboard tile, the Devices tab's preview, a screen through the
|
||||
portal — or a node of a running flow reads that message.
|
||||
|
||||
Every socket says what it wants here, in one registry, and the result is
|
||||
published on the bus as a ``device_demand`` event whenever a device's set
|
||||
changes. Each device socket picks out its own and tells the device, so nothing
|
||||
here has to know which sockets belong to which device.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Hashable
|
||||
from typing import Any
|
||||
|
||||
from fluksio.flow.events import event_bus
|
||||
|
||||
|
||||
class Demand:
|
||||
"""The media names every live socket wants, and what that asks of devices.
|
||||
|
||||
One per process, as the sockets are. ``bind`` hands it the controller,
|
||||
whose device store says what is wired and whose graph says what a flow
|
||||
reads.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._wanted: dict[Hashable, frozenset[str]] = {}
|
||||
#: The last set published per device, so only a change is.
|
||||
self._sent: dict[str, tuple[str, ...]] = {}
|
||||
self._refresh_lock = threading.Lock()
|
||||
self._controller: Any = None
|
||||
|
||||
def bind(self, controller: Any) -> None:
|
||||
self._controller = controller
|
||||
self.refresh()
|
||||
|
||||
def want(self, viewer: Hashable, names: set[str]) -> None:
|
||||
"""What one socket wants bytes for now, replacing what it said before."""
|
||||
with self._lock:
|
||||
if names:
|
||||
self._wanted[viewer] = frozenset(names)
|
||||
else:
|
||||
self._wanted.pop(viewer, None)
|
||||
self.refresh()
|
||||
|
||||
def drop(self, viewer: Hashable) -> None:
|
||||
"""A socket closed, and wants nothing any more."""
|
||||
self.want(viewer, set())
|
||||
|
||||
def viewed(self) -> set[str]:
|
||||
with self._lock:
|
||||
return set().union(*self._wanted.values())
|
||||
|
||||
def ports(self, device_id: str) -> list[str]:
|
||||
"""The device's ports that have demand, sorted."""
|
||||
return list(self._compute().get(device_id, ()))
|
||||
|
||||
def refresh(self) -> None:
|
||||
"""Work demand out again and publish what changed.
|
||||
|
||||
Called whenever its inputs move: a socket's wanted set, and a rebuild,
|
||||
a started or stopped flow or a new report, which change what is wired
|
||||
and what reads it.
|
||||
"""
|
||||
with self._refresh_lock:
|
||||
found = self._compute()
|
||||
for device_id in sorted(set(found) | set(self._sent)):
|
||||
ports = found.get(device_id, ())
|
||||
if ports == self._sent.get(device_id, ()):
|
||||
continue
|
||||
if ports:
|
||||
self._sent[device_id] = ports
|
||||
else:
|
||||
self._sent.pop(device_id, None)
|
||||
event_bus.publish(
|
||||
{
|
||||
"type": "device_demand",
|
||||
"device": device_id,
|
||||
"ports": list(ports),
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
|
||||
def _compute(self) -> dict[str, tuple[str, ...]]:
|
||||
controller = self._controller
|
||||
devices = getattr(controller, "devices", None)
|
||||
if devices is None:
|
||||
return {}
|
||||
wanted = self.viewed() | controller.consumed_messages()
|
||||
found: dict[str, tuple[str, ...]] = {}
|
||||
for device_id in devices.wiring():
|
||||
reported = devices.reported_ports(device_id) or {}
|
||||
ports = tuple(
|
||||
sorted(
|
||||
binding.port
|
||||
for binding in devices.bound(device_id)
|
||||
if binding.direction == "in"
|
||||
and binding.message in wanted
|
||||
and reported.get(binding.port, {}).get("on_demand") is True
|
||||
)
|
||||
)
|
||||
if ports:
|
||||
found[device_id] = ports
|
||||
return found
|
||||
|
||||
|
||||
demand = Demand()
|
||||
@@ -45,7 +45,7 @@ from pydantic import (
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from fluksio.flow.dashboards import DashboardDef, DashboardNotFound, DashboardStore
|
||||
from fluksio.flow.dashboards import DashboardDef, DashboardStore
|
||||
from fluksio.flow.events import event_bus
|
||||
from fluksio.flow.messages import (
|
||||
DType,
|
||||
@@ -985,13 +985,7 @@ def dashboards_for(
|
||||
"""
|
||||
if device is None:
|
||||
return []
|
||||
found: list[DashboardDef] = []
|
||||
for name in device.dashboards:
|
||||
try:
|
||||
found.append(store.read(name))
|
||||
except DashboardNotFound:
|
||||
continue
|
||||
return found
|
||||
return [defn for name in device.dashboards if (defn := store.published(name))]
|
||||
|
||||
|
||||
def messages_of(defn: DashboardDef) -> set[str]:
|
||||
|
||||
@@ -19,6 +19,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
QUEUE_SIZE = 256
|
||||
LOG_HISTORY = 400
|
||||
#: How many of a message's last artifact digests are remembered.
|
||||
RECENT_DIGESTS = 8
|
||||
|
||||
|
||||
class EventBus:
|
||||
@@ -42,6 +44,11 @@ class EventBus:
|
||||
# zero and draw a busy graph as idle. A session count, not a metric —
|
||||
# the rollups in `metrics.py` are what survives a restart.
|
||||
self.emits: dict[str, int] = {}
|
||||
#: The last few artifacts each message pointed at, newest last. A
|
||||
#: screen fetching a camera frame by frame asks for one the message
|
||||
#: may already have moved past, and a device's credential reaches
|
||||
#: these as well as the current one.
|
||||
self.digests: dict[str, deque[str]] = {}
|
||||
|
||||
def bind(self, loop: asyncio.AbstractEventLoop) -> None:
|
||||
"""Attach the bus to the running event loop (called once at startup)."""
|
||||
@@ -59,6 +66,14 @@ class EventBus:
|
||||
# increment, which is a count nobody is going to miss.
|
||||
node = str(event.get("node") or "")
|
||||
self.emits[node] = self.emits.get(node, 0) + 1
|
||||
elif kind == "message_value":
|
||||
value = event.get("value")
|
||||
if isinstance(value, dict) and isinstance(value.get("digest"), str):
|
||||
name = str(event.get("name") or "")
|
||||
recent = self.digests.get(name)
|
||||
if recent is None:
|
||||
recent = self.digests[name] = deque(maxlen=RECENT_DIGESTS)
|
||||
recent.append(value["digest"])
|
||||
loop = self._loop
|
||||
if loop is None or not self._subscribers:
|
||||
return
|
||||
|
||||
@@ -306,7 +306,7 @@ def _from_run(
|
||||
).all()
|
||||
for row in rows:
|
||||
if output in (row.name, row.name.rsplit(".", 1)[-1]):
|
||||
if artifacts is not None and artifacts.path(row.digest) is None:
|
||||
if artifacts is not None and not artifacts.has(row.digest):
|
||||
raise RunRejected(
|
||||
f"Parameter '{key}': run '{run_id}' made '{output}', but its "
|
||||
"bytes are gone from this instance's store"
|
||||
@@ -337,7 +337,7 @@ def _from_digest(
|
||||
f"Parameter '{key}': no run has produced '{digest}', so there is "
|
||||
"nothing here under that digest"
|
||||
)
|
||||
if artifacts is not None and artifacts.path(digest) is None:
|
||||
if artifacts is not None and not artifacts.has(digest):
|
||||
raise RunRejected(
|
||||
f"Parameter '{key}': '{digest}' is known but its bytes are gone "
|
||||
"from this instance's store"
|
||||
@@ -829,10 +829,7 @@ class RunCache:
|
||||
def _unrestorable(self, outputs: dict[str, Any]) -> bool:
|
||||
return any(
|
||||
is_reference(value)
|
||||
and (
|
||||
self.artifacts is None
|
||||
or self.artifacts.path(str(value["digest"])) is None
|
||||
)
|
||||
and (self.artifacts is None or not self.artifacts.has(str(value["digest"])))
|
||||
for value in outputs.values()
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user