Cameras on devices (openpanel C1): frames in memory, a binary uplink, demand
- Volatile frames are held in memory with a byte-bounded queue instead of the /dev/shm ring scanned on every put; workers keep writing to the directory, which is trimmed on a timer. A volatile upload no longer spools to disk. ARTIFACT_VOLATILE_BYTES=0 turns the ring off. - A device sends camera frames as binary messages on its socket, held and published in one step and credited to its device node; refused frames are logged and dropped. Its snapshot carries demand and the uplink (binary, or http through the portal). - One demand registry for every socket's wanted media, the tunnel's included, plus what running flows read, mapped onto on-demand ports and published as device_demand; at most 8 media names per socket. - Per-message deflate off in serve and dev; a device may fetch the last 8 digests of a message; published dashboards cached per store revision; a revoked device socket is closed with 1008. - Media tiles want bytes only while on screen; the Devices tab previews a wired camera; the device node warns when a flow reads an on-demand port.
This commit is contained in:
@@ -19,7 +19,9 @@ import hashlib
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Iterable, Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -52,9 +54,10 @@ class ArtifactStore:
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
#: A ring of frames held in memory, for media a flow only shows live.
|
||||
#: Set by the engine at startup; every lookup below falls through to
|
||||
#: it, which is what lets a volatile reference be an ordinary one.
|
||||
#: Frames held in memory, for media a flow only shows live. Set by the
|
||||
#: engine at startup unless the ring is turned off; every lookup below
|
||||
#: falls through to it, which is what lets a volatile reference be an
|
||||
#: ordinary one.
|
||||
self.volatile: VolatileStore | None = None
|
||||
|
||||
def _path(self, digest: str) -> Path:
|
||||
@@ -79,7 +82,8 @@ class ArtifactStore:
|
||||
``volatile`` puts it in the ring instead, where it is held in memory
|
||||
and falls out once newer frames need the room. The reference is the
|
||||
same shape either way — what differs is how long the bytes last, and a
|
||||
caller that wants one kept returns it from a run.
|
||||
caller that wants one kept returns it from a run. Without a ring it
|
||||
lands here like anything else.
|
||||
"""
|
||||
if volatile and self.volatile is not None:
|
||||
return self.volatile.put(chunks, name=name, media_type=media_type)
|
||||
@@ -128,20 +132,26 @@ class ArtifactStore:
|
||||
)
|
||||
|
||||
def path(self, digest: str) -> Path | None:
|
||||
"""Where the bytes are, or None if this store does not have them.
|
||||
"""Where the stored bytes are, or None if this store does not have them.
|
||||
|
||||
The ring is looked in second, so everything that resolves a digest —
|
||||
serving one over HTTP, checking a run input still exists, a panel's
|
||||
scope — reaches a live frame without knowing there are two stores.
|
||||
Only the durable store: a live frame has no file. ``has`` and ``read``
|
||||
are what reach both.
|
||||
"""
|
||||
if not valid_digest(digest):
|
||||
return None
|
||||
target = self._path(digest)
|
||||
if target.exists():
|
||||
return target
|
||||
if self.volatile is not None:
|
||||
return self.volatile.path(digest)
|
||||
return None
|
||||
return target if target.exists() else None
|
||||
|
||||
def has(self, digest: str) -> bool:
|
||||
"""Whether the bytes are here, stored or held in the ring.
|
||||
|
||||
Everything that only asks whether a digest resolves — a run input
|
||||
that has to still exist, a cached result — reaches a live frame
|
||||
without knowing there are two stores.
|
||||
"""
|
||||
if self.path(digest) is not None:
|
||||
return True
|
||||
return self.volatile is not None and self.volatile.has(digest)
|
||||
|
||||
def adopt(self, digest: str) -> bool:
|
||||
"""Copy a volatile artifact into this store, so it outlives the ring.
|
||||
@@ -154,10 +164,10 @@ class ArtifactStore:
|
||||
return False
|
||||
if self._path(digest).exists():
|
||||
return True
|
||||
source = self.volatile.path(digest)
|
||||
if source is None:
|
||||
data = self.volatile.get(digest)
|
||||
if data is None:
|
||||
return False
|
||||
self.put_file(source)
|
||||
self.put([data])
|
||||
return True
|
||||
|
||||
def delete(self, digest: str) -> bool:
|
||||
@@ -176,7 +186,11 @@ class ArtifactStore:
|
||||
def read(self, digest: str) -> Iterator[bytes]:
|
||||
target = self.path(digest)
|
||||
if target is None:
|
||||
raise FileNotFoundError(digest)
|
||||
data = self.volatile.get(digest) if self.volatile is not None else None
|
||||
if data is None:
|
||||
raise FileNotFoundError(digest)
|
||||
yield data
|
||||
return
|
||||
with target.open("rb") as handle:
|
||||
while chunk := handle.read(CHUNK):
|
||||
yield chunk
|
||||
@@ -211,44 +225,98 @@ class ArtifactStore:
|
||||
return removed
|
||||
|
||||
|
||||
class VolatileStore(ArtifactStore):
|
||||
"""A bounded ring of artifacts, held wherever memory is cheaper than disk.
|
||||
class VolatileStore:
|
||||
"""Live frames, held in memory and bounded by size.
|
||||
|
||||
A camera publishing ten frames a second is ten files a second, and on the
|
||||
wall panel this is built for that disk is an SD card. So the frames go to a
|
||||
memory-backed directory instead and the oldest fall out once the ring is
|
||||
full: nothing sweeps it, because a frame nobody kept is not worth a pass
|
||||
over the store to find.
|
||||
A camera publishing five frames a second is five artifacts a second that
|
||||
nobody keeps, so they never reach a disk: each one is held here in the
|
||||
order it came and the oldest fall out once the bound is reached. Nothing
|
||||
sweeps it, and a ``put`` costs the same however full it is.
|
||||
|
||||
It is an ``ArtifactStore``, digest layout and all, which is what lets a
|
||||
frame be an ordinary reference — the dtype check, the panel's scope, a
|
||||
node opening one with ``load_artifact`` and the widget fetching one all
|
||||
work on it unchanged.
|
||||
``root`` is a memory-backed directory beside it for the frames a worker in
|
||||
this container writes itself (``FLUKSIO_ARTIFACT_VOLATILE_DIR``), since a
|
||||
worker is another process. Read from there once and held here after, so
|
||||
a frame pushed to every open screen is read from the file once. The
|
||||
engine trims that directory on a timer.
|
||||
|
||||
A frame is an ordinary reference all the same: the dtype check, the
|
||||
device's scope, a node opening one with ``load_artifact`` and the widget
|
||||
fetching one all work on it unchanged, through ``ArtifactStore``.
|
||||
"""
|
||||
|
||||
def __init__(self, root: Path, limit_bytes: int) -> None:
|
||||
super().__init__(root)
|
||||
self.root = root
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
self.limit_bytes = limit_bytes
|
||||
self._frames: OrderedDict[str, bytes] = OrderedDict()
|
||||
self._size = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def put(
|
||||
self,
|
||||
chunks: Iterable[bytes],
|
||||
name: str = "",
|
||||
media_type: str = "",
|
||||
volatile: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
reference = super().put(chunks, name=name, media_type=media_type)
|
||||
self.trim()
|
||||
return reference
|
||||
"""Hold a frame and return the reference to it."""
|
||||
data = b"".join(chunks)
|
||||
digest = DIGEST_PREFIX + hashlib.sha256(data).hexdigest()
|
||||
self._hold(digest, data)
|
||||
return {
|
||||
"digest": digest,
|
||||
"size": len(data),
|
||||
"media_type": media_type or "application/octet-stream",
|
||||
"name": name,
|
||||
}
|
||||
|
||||
def _hold(self, digest: str, data: bytes) -> None:
|
||||
with self._lock:
|
||||
if digest in self._frames:
|
||||
# The same bytes again are the newest frame again.
|
||||
self._frames.move_to_end(digest)
|
||||
return
|
||||
self._frames[digest] = data
|
||||
self._size += len(data)
|
||||
# The one just held stays, even past the bound: its reference is
|
||||
# about to be published.
|
||||
while self._size > self.limit_bytes and len(self._frames) > 1:
|
||||
_, dropped = self._frames.popitem(last=False)
|
||||
self._size -= len(dropped)
|
||||
|
||||
def _file(self, digest: str) -> Path:
|
||||
body = digest[len(DIGEST_PREFIX) :]
|
||||
return self.root / body[:2] / body
|
||||
|
||||
def get(self, digest: str) -> bytes | None:
|
||||
"""The frame's bytes, or None once it has fallen out."""
|
||||
with self._lock:
|
||||
data = self._frames.get(digest)
|
||||
if data is not None or not valid_digest(digest):
|
||||
return data
|
||||
try:
|
||||
data = self._file(digest).read_bytes()
|
||||
except OSError:
|
||||
return None
|
||||
self._hold(digest, data)
|
||||
return data
|
||||
|
||||
def has(self, digest: str) -> bool:
|
||||
with self._lock:
|
||||
if digest in self._frames:
|
||||
return True
|
||||
return valid_digest(digest) and self._file(digest).is_file()
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
"""Bytes held in memory."""
|
||||
return self._size
|
||||
|
||||
def trim(self) -> int:
|
||||
"""Drop the oldest until the ring is inside its bound.
|
||||
"""Drop the oldest of the workers' frames until the directory is
|
||||
inside the bound. Returns how many went.
|
||||
|
||||
Also called on a timer, because a worker in this container writes here
|
||||
itself and the engine never sees that ``put``.
|
||||
|
||||
# ponytail: a scandir per trim. An in-memory index if a ring of
|
||||
# thousands of frames ever shows up in a profile.
|
||||
A scan of the directory, so it runs on a timer and never per frame:
|
||||
only frames a worker in this container wrote are there.
|
||||
"""
|
||||
if self.limit_bytes <= 0:
|
||||
return 0
|
||||
|
||||
Reference in New Issue
Block a user