Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cameras on devices (openpanel C1): frames in memory, a binary uplink, demand

Browse Source 
- Volatile frames are held in memory with a byte-bounded queue instead of
  the /dev/shm ring scanned on every put; workers keep writing to the
  directory, which is trimmed on a timer. A volatile upload no longer spools
  to disk. ARTIFACT_VOLATILE_BYTES=0 turns the ring off.
- A device sends camera frames as binary messages on its socket, held and
  published in one step and credited to its device node; refused frames are
  logged and dropped. Its snapshot carries demand and the uplink (binary, or
  http through the portal).
- One demand registry for every socket's wanted media, the tunnel's
  included, plus what running flows read, mapped onto on-demand ports and
  published as device_demand; at most 8 media names per socket.
- Per-message deflate off in serve and dev; a device may fetch the last 8
  digests of a message; published dashboards cached per store revision; a
  revoked device socket is closed with 1008.
- Media tiles want bytes only while on screen; the Devices tab previews a
  wired camera; the device node warns when a flow reads an on-demand port.
This commit is contained in:
Melvin Strobl
2026-10-16 20:43:14 +02:00
parent d79203a5f7
commit ecc4bee4bb
29 changed files with 1425 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/fluksio/api/deps.py
+10 -5
View File
@@ -20,6 +20,7 @@ from fluksio.flow.artifacts import is_reference
from fluksio.flow.controller import FlowController
from fluksio.flow.dashboards import DashboardDef, DashboardStore
from fluksio.flow.devices import DeviceDef, DeviceStore, DeviceUnavailable, presence
from fluksio.flow.events import event_bus
from fluksio.flow.workers import PythonWorkerPool
from fluksio.models import OAuthClient, TokenPayload, User, WorkerToken
@@ -165,11 +166,13 @@ def _device_writable(device: DeviceDef, request: Request) -> set[str]:
def _device_digests(device: DeviceDef, request: Request) -> set[str]:
"""The artifacts this panel's messages are pointing at right now.
"""The artifacts this panel's messages are pointing at, now or just before.
A media tile fetches the bytes its message names, so the messages already
bounding the panel bound this too — one step further along, through
whatever those messages currently hold.
whatever those messages currently hold. And through the last few they
held: a screen through the portal fetches a camera frame by frame, and by
the time its fetch arrives the camera may have published the next one.
"""
controller: FlowController | None = getattr(
request.app.state, "flow_controller", None
@@ -183,6 +186,8 @@ def _device_digests(device: DeviceDef, request: Request) -> set[str]:
for value in controller.state.get_present(sorted(names)).values():
if is_reference(value):
found.add(str(value["digest"]))
for name in names:
found.update(event_bus.digests.get(name, ()))
return found
@@ -215,9 +220,9 @@ def _device_may(payload: dict[str, Any], request: Request) -> None:
The device is left on ``request.state`` for the route, which credits a
value published on one of its ports to the node holding that port.
# ponytail: the device file and the published dashboards are re-read per
# request (the nonces are cached by the file's stat). Cache them behind
# the store's revision if this shows up in a profile.
The device file is read per request, which is what makes deleting a
device revoke it at once. The nonces are cached by the file's stat and
the published dashboards until the flow store's next commit.
"""
device = _device_for(payload, request.app)
request.state.device = device
backend/fluksio/api/routes/artifacts.py
+19 -2
View File
@@ -101,7 +101,9 @@ async def put_artifact(
``volatile`` puts it in the ring instead of the store — a frame a screen is
watching now, which the oldest of falls out of memory rather than being
kept. A worker on another host publishing a camera sends this.
kept. A worker on another host or a device publishing a camera sends this.
It is read into memory, where the ring holds it, and never touches the
disk.
"""
store = _store(request)
cap = settings.MAX_ARTIFACT_BYTES
@@ -110,6 +112,17 @@ async def put_artifact(
raise HTTPException(
status_code=413, detail=f"An artifact may be at most {cap} bytes"
)
ring = store.volatile if volatile else None
if ring is not None:
body = bytearray()
async for chunk in request.stream():
body += chunk
if cap and len(body) > cap:
raise HTTPException(
status_code=413,
detail=f"An artifact may be at most {cap} bytes",
)
return await run_in_threadpool(ring.put, [bytes(body)], name, media_type)
handle = tempfile.NamedTemporaryFile(dir=store.root, delete=False)
written = 0
try:
@@ -146,7 +159,9 @@ def get_artifact(
"""
store = _store(request)
path = store.path(digest)
if path is None:
# A live frame is held in memory, not on disk.
frame = store.volatile.get(digest) if path is None and store.volatile else None
if path is None and frame is None:
raise HTTPException(status_code=404, detail="No such artifact")
if not _MEDIA_TYPE.match(media_type):
media_type = "application/octet-stream"
@@ -158,6 +173,8 @@ def get_artifact(
# beside the session, for whoever follows a link to it.
headers["Content-Security-Policy"] = "sandbox"
headers["X-Content-Type-Options"] = "nosniff"
if path is None:
return Response(frame, media_type=media_type, headers=headers)
return FileResponse(path, media_type=media_type, headers=headers)
backend/fluksio/api/routes/flows.py
+247 -33
View File
@@ -3,6 +3,7 @@
import asyncio
import logging
import os
import re
import time
from pathlib import Path
from typing import Any
@@ -32,12 +33,23 @@ from fluksio.api.deps import (
get_current_user,
user_from_token,
)
from fluksio.api.routes.messages import port_source
from fluksio.api.socket_auth import TICKET_TTL_S, issue_ticket, socket_token
from fluksio.core.config import settings
from fluksio.core.db import engine
from fluksio.flow.artifacts import is_reference
from fluksio.flow.controller import NODE_TYPES, FlowController, follow_header
from fluksio.flow.dashboards import DashboardStore
from fluksio.flow.devices import DeviceStore, messages_for, presence
from fluksio.flow.demand import demand
from fluksio.flow.devices import (
DeviceDef,
DeviceStore,
DeviceUnavailable,
dashboards_for,
messages_for,
messages_of,
presence,
)
from fluksio.flow.events import event_bus
from fluksio.flow.messages import flow_of, qualify
from fluksio.flow.pipeline import ValidationIssue
@@ -1017,7 +1029,10 @@ def read_message_history(
def snapshot_payload(
controller: FlowController, only: set[str] | None = None
controller: FlowController,
only: set[str] | None = None,
device: str = "",
uplink: str = "",
) -> dict[str, Any]:
"""Everything a client needs to catch up, sent the moment it connects.
@@ -1032,12 +1047,16 @@ def snapshot_payload(
gets: the values its own dashboards draw, and none of the rest — no node
status, no logs, no shape of the graph. A panel renders none of that, and
a screen may be hanging somewhere nobody here can see.
A device's socket also says which of its ports have ``demand`` now, and
which ``uplink`` its frames take: ``binary`` on its own socket, ``http``
through the portal, whose tunnel carries text only.
"""
if only is not None:
# The same keys, emptied rather than dropped: a screen already hanging
# runs whatever bundle it was paired with, and the shape of this
# message is what that bundle reads.
return {
payload: dict[str, Any] = {
"type": "snapshot",
"values": {k: v for k, v in controller.values().items() if k in only},
"nodes": [],
@@ -1047,6 +1066,10 @@ def snapshot_payload(
"emits": {},
"queued": [],
}
if device:
payload["demand"] = demand.ports(device)
payload["uplink"] = uplink
return payload
return {
"type": "snapshot",
"values": controller.values(),
@@ -1059,6 +1082,11 @@ def snapshot_payload(
}
def demand_frame(ports: list[str]) -> str:
"""What a device is told when its ports with demand change."""
return orjson.dumps({"type": "demand", "ports": ports}).decode()
def device_of(token: str, app: Any) -> str:
"""The device this credential stands for, or "" for a person or none."""
try:
@@ -1067,12 +1095,30 @@ def device_of(token: str, app: Any) -> str:
return ""
def revoked(token: str, app: Any) -> bool:
"""Whether a device's credential has stopped standing since its handshake.
Unpaired or deleted. One that cannot be checked right now has not: a
device closed with 1008 checks whether it is still paired.
"""
try:
decode_token(token, app=app)
except InvalidTokenError:
return True
except DeviceUnavailable:
return False
return False
def panel_scope(token: str, app: Any) -> set[str] | None:
"""The messages this credential is bounded to, or None if it is a person's.
The socket is the one authenticated surface the route check cannot reach,
a handshake has no route to judge, so a device is bounded by what it is
sent instead of by what it asks for.
Its own camera's frames are left out unless a dashboard it shows draws
them: it sent them, and a camera's values come several a second.
"""
device_id = device_of(token, app)
if not device_id:
@@ -1081,7 +1127,18 @@ def panel_scope(token: str, app: Any) -> set[str] | None:
devices: DeviceStore | None = getattr(app.state, "device_store", None)
if store is None or devices is None:
return set()
return messages_for(devices.find(device_id), store, devices)
device = devices.find(device_id)
found = messages_for(device, store, devices)
if device is not None:
shown = {
name for defn in dashboards_for(device, store) for name in messages_of(defn)
}
found -= {
b.message
for b in devices.bound(device.id)
if b.direction == "in" and b.dtype == "image"
} - shown
return found
def event_for_panel(event: dict[str, Any], only: set[str]) -> bool:
@@ -1103,9 +1160,9 @@ def event_for_panel(event: dict[str, Any], only: set[str]) -> bool:
MAX_FRAME_EVENTS = 64
#: How many message names one socket may ask for bytes on. A screen draws a
#: handful of tiles; this is only here so a client cannot ask for the whole
#: namespace and be served every frame of it.
MAX_MEDIA_NAMES = 32
#: handful of tiles, and every live one keeps a camera sending, so a phone
#: scrolled past several cameras does not pull them all. The rest are refused.
MAX_MEDIA_NAMES = 8
def wanted_names(frame: str, only: set[str] | None) -> set[str] | None:
@@ -1126,7 +1183,17 @@ def wanted_names(frame: str, only: set[str] | None) -> set[str] | None:
names = payload.get("names")
if not isinstance(names, list):
return set()
asked = {str(name) for name in names[:MAX_MEDIA_NAMES]}
asked = {str(name) for name in names}
if len(asked) > MAX_MEDIA_NAMES:
kept = list(dict.fromkeys(str(name) for name in names))[:MAX_MEDIA_NAMES]
logger.warning(
"A socket asked for live media on %d messages; only %d are served, "
"the first it named: %s",
len(asked),
MAX_MEDIA_NAMES,
", ".join(kept),
)
asked = set(kept)
return asked if only is None else asked & only
@@ -1140,10 +1207,10 @@ def media_frames(
the socket that already carries the event closes that, and only for the
frames a client said it was drawing.
Only what the ring holds: the durable store is what a fetch is for, and a
checkpoint has no business being pushed at anyone. The newest value per
name wins, so a client that fell behind is not handed a backlog of frames
it would only draw over.
Only what the ring holds, out of memory: the durable store is what a fetch
is for, and a checkpoint has no business being pushed at anyone. The
newest value per name wins, so a client that fell behind is not handed a
backlog of frames it would only draw over.
"""
if not wanted or store is None or getattr(store, "volatile", None) is None:
return []
@@ -1161,12 +1228,8 @@ def media_frames(
for name, event in newest.items():
value = event["value"]
digest = str(value.get("digest") or "")
path = store.volatile.path(digest)
if path is None:
continue
try:
payload = path.read_bytes()
except OSError:
payload = store.volatile.get(digest)
if payload is None:
continue
header = orjson.dumps(
{
@@ -1183,6 +1246,91 @@ def media_frames(
return frames
#: What a device may say a frame is.
_IMAGE_TYPE = re.compile(r"^image/[\w.+-]+$")
def parse_frame(data: bytes) -> tuple[dict[str, Any], bytes] | None:
"""Split a binary frame into its JSON header and the bytes after it.
The layout media frames go out in: four bytes of header length, big
endian, the header, then the bytes. None for anything else.
"""
if len(data) < 4:
return None
length = int.from_bytes(data[:4], "big")
if length == 0 or 4 + length > len(data):
return None
try:
header = orjson.loads(data[4 : 4 + length])
except orjson.JSONDecodeError:
return None
if not isinstance(header, dict):
return None
return header, data[4 + length :]
def ingest_frame(app: Any, device: DeviceDef, data: bytes) -> str:
"""Store a frame a device sent on its socket and publish it.
Returns why it was refused, or "" once it is published.
The same frame over HTTP is two requests, each authenticated: the upload,
then the publish. Here the socket was authenticated once, at its
handshake, and the frame is held and published in one step. Only a
message one of the device's own ``in`` ports is wired to, as an ``image``,
and the value is credited to the device node holding that port, as an
HTTP publish is.
"""
parsed = parse_frame(data)
if parsed is None or parsed[0].get("type") != "frame":
return "not a frame"
header, payload = parsed
name = str(header.get("name") or "")
devices: DeviceStore | None = getattr(app.state, "device_store", None)
store: Any = getattr(app.state, "artifact_store", None)
controller: FlowController | None = getattr(app.state, "flow_controller", None)
if devices is None or store is None or controller is None:
return "the engine is not running"
binding = next(
(
b
for b in devices.bound(device.id)
if b.direction == "in" and b.message == name
),
None,
)
if binding is None:
return f"{name!r} is not a message one of its ports publishes"
if binding.dtype != "image":
return f"its port {binding.port!r} is wired as {binding.dtype}, not image"
media_type = str(header.get("media_type") or "")
if not _IMAGE_TYPE.match(media_type):
return f"{media_type!r} is not an image type"
cap = settings.MAX_ARTIFACT_BYTES
if cap and len(payload) > cap:
return f"a frame may be at most {cap} bytes"
reference = store.put(
[payload], name=binding.port, media_type=media_type, volatile=True
)
try:
controller.publish_message(name, reference, port_source(devices, device, name))
except KeyError:
return f"no flow declares {name!r}"
except TypeError as exc:
return str(exc)
return ""
async def _receive(websocket: WebSocket) -> str | bytes:
"""The client's next frame, text or binary."""
message = await websocket.receive()
if message["type"] == "websocket.disconnect":
raise WebSocketDisconnect(message.get("code", 1000), message.get("reason"))
data = message.get("bytes")
return data if data is not None else str(message.get("text") or "")
async def _send(websocket: WebSocket, events: list[dict[str, Any]]) -> None:
"""One event, or a batch of them under `events`.
@@ -1216,6 +1364,11 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
A browser brings a ticket from `POST /flows/ws-ticket`; anything that can
set headers sends `Authorization: Bearer`. `?token=` is still read for one
release, for clients from before either.
A client says which media it is drawing (``{"type": "media", "names":
[...]}``) and is pushed those frames. A device also sends its camera's
frames here, as binary frames in the layout it is pushed them in, and is
told when its on-demand ports have demand.
"""
token = socket_token(websocket.headers, ticket, token)
@@ -1228,10 +1381,10 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
only = panel_scope(token, websocket.app) if user is not None else None
# A device is online for as long as this socket is open.
device = device_of(token, websocket.app) if only is not None else ""
return user, only, device
return user, only, device, _device_def(websocket.app, device)
try:
user, only, device = await run_in_threadpool(_authenticate)
user, only, device, device_def = await run_in_threadpool(_authenticate)
except Exception as exc:
# Not a refusal: the database or the device store could not say. A
# device takes 1008, or a 403 before accepting, as its credential
@@ -1246,6 +1399,10 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
# Nothing until a client says it is drawing something: bytes are what this
# socket cannot afford to send speculatively.
wanted: set[str] = set()
# This socket's entry in the registry every socket's wants are kept in.
viewer = object()
# Each reason a frame was refused for, logged once per socket.
refused: set[str] = set()
await websocket.accept()
@@ -1254,17 +1411,27 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
)
store = getattr(websocket.app.state, "artifact_store", None)
async def send_snapshot() -> None:
if controller is not None:
payload = await run_in_threadpool(snapshot_payload, controller, only)
await websocket.send_text(orjson.dumps(payload).decode())
async def send_snapshot() -> list[str]:
"""Send the snapshot, and return the demand it told a device of."""
if controller is None:
return []
payload = await run_in_threadpool(
snapshot_payload, controller, only, device, "binary"
)
await websocket.send_text(orjson.dumps(payload).decode())
return list(payload.get("demand", []))
await send_snapshot()
told = await send_snapshot()
async with event_bus.subscribe() as queue:
receiver = asyncio.create_task(websocket.receive_text())
receiver = asyncio.create_task(_receive(websocket))
if device:
presence.opened(device)
# Demand that moved between the snapshot and the subscription
# would otherwise wait for the next change.
ports = demand.ports(device)
if ports != told:
await websocket.send_text(demand_frame(ports))
try:
while True:
sender = asyncio.create_task(queue.get())
@@ -1284,14 +1451,35 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
# it every live update from then on.
failure = receiver.exception()
if failure is not None:
# The client is gone, and `receive_text` raises for
# good from here on. Reading it again would spin this
# loop at full tilt against a dead socket.
# The client is gone, and `receive` raises for good
# from here on. Reading it again would spin this loop
# at full tilt against a dead socket.
raise failure
asked = wanted_names(receiver.result(), only)
if asked is not None:
wanted = asked
receiver = asyncio.create_task(websocket.receive_text())
frame = receiver.result()
if isinstance(frame, bytes):
# A camera's frame. Refused ones are dropped and the
# socket stays open: one bad frame is not a reason to
# take the device's live values away.
reason = (
await run_in_threadpool(
ingest_frame, websocket.app, device_def, frame
)
if device_def is not None
else "only a device sends frames"
)
if reason and reason not in refused:
refused.add(reason)
logger.warning(
"Dropped a frame from %s: %s",
device or "a person's socket",
reason,
)
else:
asked = wanted_names(frame, only)
if asked is not None:
wanted = asked
demand.want(viewer, wanted)
receiver = asyncio.create_task(_receive(websocket))
if sender not in done:
continue
# `sender` is done from here: `FIRST_COMPLETED` returns with at
@@ -1309,7 +1497,22 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
out: list[dict[str, Any]] = []
for event in batch:
if event.get("type") == "device_demand":
# For the device it names, and for nobody else.
if device and event.get("device") == device:
await websocket.send_text(
demand_frame(list(event.get("ports") or []))
)
continue
if only is not None and event.get("type") == "dashboard_changed":
# An unpaired or deleted device is closed with 1008,
# which sends it to check its credential: otherwise
# its frames would still be taken here.
if device and await run_in_threadpool(
revoked, token, websocket.app
):
await websocket.close(code=1008)
return
# The scope was resolved once, at the handshake. A
# panel pointed at another dashboard would otherwise
# fetch the new document and then draw tiles nothing
@@ -1318,7 +1521,9 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
# token — and that would widen this socket to
# everything on the bus.
only = panel_scope(token, websocket.app) or set()
device_def = _device_def(websocket.app, device)
wanted &= only
demand.want(viewer, wanted)
if out:
await _send(websocket, out)
out = []
@@ -1339,5 +1544,14 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
pass
finally:
receiver.cancel()
demand.drop(viewer)
if device:
presence.closed(device)
def _device_def(app: Any, device_id: str) -> DeviceDef | None:
"""The device a socket stands for, as its frames are credited to it."""
devices: DeviceStore | None = getattr(app.state, "device_store", None)
if not device_id or devices is None:
return None
return devices.find(device_id)
backend/fluksio/api/routes/messages.py
+13 -6
View File
@@ -72,17 +72,16 @@ async def read_messages(controller: FlowControllerDep) -> Any:
return MessagesPublic(data=infos, count=len(infos))
def _port_source(request: Request, name: str) -> ValueSource | None:
def port_source(
devices: DeviceStore, device: DeviceDef, name: str
) -> ValueSource | None:
"""The device node a device's own publish on a bound ``in`` port is from.
Said by the server rather than by the device: the node is what the canvas
draws the value leaving from, and the device does not know which node in
which flow holds its port.
which flow holds its port. Also what a frame on the device's socket is
credited to.
"""
device: DeviceDef | None = getattr(request.state, "device", None)
devices: DeviceStore | None = getattr(request.app.state, "device_store", None)
if device is None or devices is None:
return None
for bound in devices.bound(device.id):
if bound.direction == "in" and bound.message == name:
return ValueSource(
@@ -94,6 +93,14 @@ def _port_source(request: Request, name: str) -> ValueSource | None:
return None
def _port_source(request: Request, name: str) -> ValueSource | None:
device: DeviceDef | None = getattr(request.state, "device", None)
devices: DeviceStore | None = getattr(request.app.state, "device_store", None)
if device is None or devices is None:
return None
return port_source(devices, device, name)
@router.post("/{name}", response_model=MessageValue)
async def publish_message(
name: str, body: PublishRequest, controller: FlowControllerDep, request: Request