Cameras on devices (openpanel C1): frames in memory, a binary uplink, demand

- Volatile frames are held in memory with a byte-bounded queue instead of
  the /dev/shm ring scanned on every put; workers keep writing to the
  directory, which is trimmed on a timer. A volatile upload no longer spools
  to disk. ARTIFACT_VOLATILE_BYTES=0 turns the ring off.
- A device sends camera frames as binary messages on its socket, held and
  published in one step and credited to its device node; refused frames are
  logged and dropped. Its snapshot carries demand and the uplink (binary, or
  http through the portal).
- One demand registry for every socket's wanted media, the tunnel's
  included, plus what running flows read, mapped onto on-demand ports and
  published as device_demand; at most 8 media names per socket.
- Per-message deflate off in serve and dev; a device may fetch the last 8
  digests of a message; published dashboards cached per store revision; a
  revoked device socket is closed with 1008.
- Media tiles want bytes only while on screen; the Devices tab previews a
  wired camera; the device node warns when a flow reads an on-demand port.
This commit is contained in:
Melvin Strobl
2026-10-16 20:43:14 +02:00
parent d79203a5f7
commit ecc4bee4bb
29 changed files with 1425 additions and 220 deletions
+247 -33
View File
@@ -3,6 +3,7 @@
import asyncio
import logging
import os
import re
import time
from pathlib import Path
from typing import Any
@@ -32,12 +33,23 @@ from fluksio.api.deps import (
get_current_user,
user_from_token,
)
from fluksio.api.routes.messages import port_source
from fluksio.api.socket_auth import TICKET_TTL_S, issue_ticket, socket_token
from fluksio.core.config import settings
from fluksio.core.db import engine
from fluksio.flow.artifacts import is_reference
from fluksio.flow.controller import NODE_TYPES, FlowController, follow_header
from fluksio.flow.dashboards import DashboardStore
from fluksio.flow.devices import DeviceStore, messages_for, presence
from fluksio.flow.demand import demand
from fluksio.flow.devices import (
DeviceDef,
DeviceStore,
DeviceUnavailable,
dashboards_for,
messages_for,
messages_of,
presence,
)
from fluksio.flow.events import event_bus
from fluksio.flow.messages import flow_of, qualify
from fluksio.flow.pipeline import ValidationIssue
@@ -1017,7 +1029,10 @@ def read_message_history(
def snapshot_payload(
controller: FlowController, only: set[str] | None = None
controller: FlowController,
only: set[str] | None = None,
device: str = "",
uplink: str = "",
) -> dict[str, Any]:
"""Everything a client needs to catch up, sent the moment it connects.
@@ -1032,12 +1047,16 @@ def snapshot_payload(
gets: the values its own dashboards draw, and none of the rest — no node
status, no logs, no shape of the graph. A panel renders none of that, and
a screen may be hanging somewhere nobody here can see.
A device's socket also says which of its ports have ``demand`` now, and
which ``uplink`` its frames take: ``binary`` on its own socket, ``http``
through the portal, whose tunnel carries text only.
"""
if only is not None:
# The same keys, emptied rather than dropped: a screen already hanging
# runs whatever bundle it was paired with, and the shape of this
# message is what that bundle reads.
return {
payload: dict[str, Any] = {
"type": "snapshot",
"values": {k: v for k, v in controller.values().items() if k in only},
"nodes": [],
@@ -1047,6 +1066,10 @@ def snapshot_payload(
"emits": {},
"queued": [],
}
if device:
payload["demand"] = demand.ports(device)
payload["uplink"] = uplink
return payload
return {
"type": "snapshot",
"values": controller.values(),
@@ -1059,6 +1082,11 @@ def snapshot_payload(
}
def demand_frame(ports: list[str]) -> str:
"""What a device is told when its ports with demand change."""
return orjson.dumps({"type": "demand", "ports": ports}).decode()
def device_of(token: str, app: Any) -> str:
"""The device this credential stands for, or "" for a person or none."""
try:
@@ -1067,12 +1095,30 @@ def device_of(token: str, app: Any) -> str:
return ""
def revoked(token: str, app: Any) -> bool:
"""Whether a device's credential has stopped standing since its handshake.
Unpaired or deleted. One that cannot be checked right now has not: a
device closed with 1008 checks whether it is still paired.
"""
try:
decode_token(token, app=app)
except InvalidTokenError:
return True
except DeviceUnavailable:
return False
return False
def panel_scope(token: str, app: Any) -> set[str] | None:
"""The messages this credential is bounded to, or None if it is a person's.
The socket is the one authenticated surface the route check cannot reach,
a handshake has no route to judge, so a device is bounded by what it is
sent instead of by what it asks for.
Its own camera's frames are left out unless a dashboard it shows draws
them: it sent them, and a camera's values come several a second.
"""
device_id = device_of(token, app)
if not device_id:
@@ -1081,7 +1127,18 @@ def panel_scope(token: str, app: Any) -> set[str] | None:
devices: DeviceStore | None = getattr(app.state, "device_store", None)
if store is None or devices is None:
return set()
return messages_for(devices.find(device_id), store, devices)
device = devices.find(device_id)
found = messages_for(device, store, devices)
if device is not None:
shown = {
name for defn in dashboards_for(device, store) for name in messages_of(defn)
}
found -= {
b.message
for b in devices.bound(device.id)
if b.direction == "in" and b.dtype == "image"
} - shown
return found
def event_for_panel(event: dict[str, Any], only: set[str]) -> bool:
@@ -1103,9 +1160,9 @@ def event_for_panel(event: dict[str, Any], only: set[str]) -> bool:
MAX_FRAME_EVENTS = 64
#: How many message names one socket may ask for bytes on. A screen draws a
#: handful of tiles; this is only here so a client cannot ask for the whole
#: namespace and be served every frame of it.
MAX_MEDIA_NAMES = 32
#: handful of tiles, and every live one keeps a camera sending, so a phone
#: scrolled past several cameras does not pull them all. The rest are refused.
MAX_MEDIA_NAMES = 8
def wanted_names(frame: str, only: set[str] | None) -> set[str] | None:
@@ -1126,7 +1183,17 @@ def wanted_names(frame: str, only: set[str] | None) -> set[str] | None:
names = payload.get("names")
if not isinstance(names, list):
return set()
asked = {str(name) for name in names[:MAX_MEDIA_NAMES]}
asked = {str(name) for name in names}
if len(asked) > MAX_MEDIA_NAMES:
kept = list(dict.fromkeys(str(name) for name in names))[:MAX_MEDIA_NAMES]
logger.warning(
"A socket asked for live media on %d messages; only %d are served, "
"the first it named: %s",
len(asked),
MAX_MEDIA_NAMES,
", ".join(kept),
)
asked = set(kept)
return asked if only is None else asked & only
@@ -1140,10 +1207,10 @@ def media_frames(
the socket that already carries the event closes that, and only for the
frames a client said it was drawing.
Only what the ring holds: the durable store is what a fetch is for, and a
checkpoint has no business being pushed at anyone. The newest value per
name wins, so a client that fell behind is not handed a backlog of frames
it would only draw over.
Only what the ring holds, out of memory: the durable store is what a fetch
is for, and a checkpoint has no business being pushed at anyone. The
newest value per name wins, so a client that fell behind is not handed a
backlog of frames it would only draw over.
"""
if not wanted or store is None or getattr(store, "volatile", None) is None:
return []
@@ -1161,12 +1228,8 @@ def media_frames(
for name, event in newest.items():
value = event["value"]
digest = str(value.get("digest") or "")
path = store.volatile.path(digest)
if path is None:
continue
try:
payload = path.read_bytes()
except OSError:
payload = store.volatile.get(digest)
if payload is None:
continue
header = orjson.dumps(
{
@@ -1183,6 +1246,91 @@ def media_frames(
return frames
#: What a device may say a frame is.
_IMAGE_TYPE = re.compile(r"^image/[\w.+-]+$")
def parse_frame(data: bytes) -> tuple[dict[str, Any], bytes] | None:
"""Split a binary frame into its JSON header and the bytes after it.
The layout media frames go out in: four bytes of header length, big
endian, the header, then the bytes. None for anything else.
"""
if len(data) < 4:
return None
length = int.from_bytes(data[:4], "big")
if length == 0 or 4 + length > len(data):
return None
try:
header = orjson.loads(data[4 : 4 + length])
except orjson.JSONDecodeError:
return None
if not isinstance(header, dict):
return None
return header, data[4 + length :]
def ingest_frame(app: Any, device: DeviceDef, data: bytes) -> str:
"""Store a frame a device sent on its socket and publish it.
Returns why it was refused, or "" once it is published.
The same frame over HTTP is two requests, each authenticated: the upload,
then the publish. Here the socket was authenticated once, at its
handshake, and the frame is held and published in one step. Only a
message one of the device's own ``in`` ports is wired to, as an ``image``,
and the value is credited to the device node holding that port, as an
HTTP publish is.
"""
parsed = parse_frame(data)
if parsed is None or parsed[0].get("type") != "frame":
return "not a frame"
header, payload = parsed
name = str(header.get("name") or "")
devices: DeviceStore | None = getattr(app.state, "device_store", None)
store: Any = getattr(app.state, "artifact_store", None)
controller: FlowController | None = getattr(app.state, "flow_controller", None)
if devices is None or store is None or controller is None:
return "the engine is not running"
binding = next(
(
b
for b in devices.bound(device.id)
if b.direction == "in" and b.message == name
),
None,
)
if binding is None:
return f"{name!r} is not a message one of its ports publishes"
if binding.dtype != "image":
return f"its port {binding.port!r} is wired as {binding.dtype}, not image"
media_type = str(header.get("media_type") or "")
if not _IMAGE_TYPE.match(media_type):
return f"{media_type!r} is not an image type"
cap = settings.MAX_ARTIFACT_BYTES
if cap and len(payload) > cap:
return f"a frame may be at most {cap} bytes"
reference = store.put(
[payload], name=binding.port, media_type=media_type, volatile=True
)
try:
controller.publish_message(name, reference, port_source(devices, device, name))
except KeyError:
return f"no flow declares {name!r}"
except TypeError as exc:
return str(exc)
return ""
async def _receive(websocket: WebSocket) -> str | bytes:
"""The client's next frame, text or binary."""
message = await websocket.receive()
if message["type"] == "websocket.disconnect":
raise WebSocketDisconnect(message.get("code", 1000), message.get("reason"))
data = message.get("bytes")
return data if data is not None else str(message.get("text") or "")
async def _send(websocket: WebSocket, events: list[dict[str, Any]]) -> None:
"""One event, or a batch of them under `events`.
@@ -1216,6 +1364,11 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
A browser brings a ticket from `POST /flows/ws-ticket`; anything that can
set headers sends `Authorization: Bearer`. `?token=` is still read for one
release, for clients from before either.
A client says which media it is drawing (``{"type": "media", "names":
[...]}``) and is pushed those frames. A device also sends its camera's
frames here, as binary frames in the layout it is pushed them in, and is
told when its on-demand ports have demand.
"""
token = socket_token(websocket.headers, ticket, token)
@@ -1228,10 +1381,10 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
only = panel_scope(token, websocket.app) if user is not None else None
# A device is online for as long as this socket is open.
device = device_of(token, websocket.app) if only is not None else ""
return user, only, device
return user, only, device, _device_def(websocket.app, device)
try:
user, only, device = await run_in_threadpool(_authenticate)
user, only, device, device_def = await run_in_threadpool(_authenticate)
except Exception as exc:
# Not a refusal: the database or the device store could not say. A
# device takes 1008, or a 403 before accepting, as its credential
@@ -1246,6 +1399,10 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
# Nothing until a client says it is drawing something: bytes are what this
# socket cannot afford to send speculatively.
wanted: set[str] = set()
# This socket's entry in the registry every socket's wants are kept in.
viewer = object()
# Each reason a frame was refused for, logged once per socket.
refused: set[str] = set()
await websocket.accept()
@@ -1254,17 +1411,27 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
)
store = getattr(websocket.app.state, "artifact_store", None)
async def send_snapshot() -> None:
if controller is not None:
payload = await run_in_threadpool(snapshot_payload, controller, only)
await websocket.send_text(orjson.dumps(payload).decode())
async def send_snapshot() -> list[str]:
"""Send the snapshot, and return the demand it told a device of."""
if controller is None:
return []
payload = await run_in_threadpool(
snapshot_payload, controller, only, device, "binary"
)
await websocket.send_text(orjson.dumps(payload).decode())
return list(payload.get("demand", []))
await send_snapshot()
told = await send_snapshot()
async with event_bus.subscribe() as queue:
receiver = asyncio.create_task(websocket.receive_text())
receiver = asyncio.create_task(_receive(websocket))
if device:
presence.opened(device)
# Demand that moved between the snapshot and the subscription
# would otherwise wait for the next change.
ports = demand.ports(device)
if ports != told:
await websocket.send_text(demand_frame(ports))
try:
while True:
sender = asyncio.create_task(queue.get())
@@ -1284,14 +1451,35 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
# it every live update from then on.
failure = receiver.exception()
if failure is not None:
# The client is gone, and `receive_text` raises for
# good from here on. Reading it again would spin this
# loop at full tilt against a dead socket.
# The client is gone, and `receive` raises for good
# from here on. Reading it again would spin this loop
# at full tilt against a dead socket.
raise failure
asked = wanted_names(receiver.result(), only)
if asked is not None:
wanted = asked
receiver = asyncio.create_task(websocket.receive_text())
frame = receiver.result()
if isinstance(frame, bytes):
# A camera's frame. Refused ones are dropped and the
# socket stays open: one bad frame is not a reason to
# take the device's live values away.
reason = (
await run_in_threadpool(
ingest_frame, websocket.app, device_def, frame
)
if device_def is not None
else "only a device sends frames"
)
if reason and reason not in refused:
refused.add(reason)
logger.warning(
"Dropped a frame from %s: %s",
device or "a person's socket",
reason,
)
else:
asked = wanted_names(frame, only)
if asked is not None:
wanted = asked
demand.want(viewer, wanted)
receiver = asyncio.create_task(_receive(websocket))
if sender not in done:
continue
# `sender` is done from here: `FIRST_COMPLETED` returns with at
@@ -1309,7 +1497,22 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
out: list[dict[str, Any]] = []
for event in batch:
if event.get("type") == "device_demand":
# For the device it names, and for nobody else.
if device and event.get("device") == device:
await websocket.send_text(
demand_frame(list(event.get("ports") or []))
)
continue
if only is not None and event.get("type") == "dashboard_changed":
# An unpaired or deleted device is closed with 1008,
# which sends it to check its credential: otherwise
# its frames would still be taken here.
if device and await run_in_threadpool(
revoked, token, websocket.app
):
await websocket.close(code=1008)
return
# The scope was resolved once, at the handshake. A
# panel pointed at another dashboard would otherwise
# fetch the new document and then draw tiles nothing
@@ -1318,7 +1521,9 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
# token — and that would widen this socket to
# everything on the bus.
only = panel_scope(token, websocket.app) or set()
device_def = _device_def(websocket.app, device)
wanted &= only
demand.want(viewer, wanted)
if out:
await _send(websocket, out)
out = []
@@ -1339,5 +1544,14 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
pass
finally:
receiver.cancel()
demand.drop(viewer)
if device:
presence.closed(device)
def _device_def(app: Any, device_id: str) -> DeviceDef | None:
"""The device a socket stands for, as its frames are credited to it."""
devices: DeviceStore | None = getattr(app.state, "device_store", None)
if not device_id or devices is None:
return None
return devices.find(device_id)