Push a frame instead of storing and fetching it

The rate the media dtypes could carry was one frame every second or two: each
was a file on the data volume, an event on the socket, and a request back for
the bytes. This closes both halves of that, and they are one feature.

`save_artifact(..., volatile=True)` writes to a `VolatileStore` — the same
content-addressed store, in `/dev/shm`, bounded by size with the oldest falling
out (`ARTIFACT_VOLATILE_BYTES`, 48 MB under the container's raised `shm_size`).
Nothing sweeps it: a frame nobody kept is not worth walking the store to find.
`ArtifactStore.path` falls through to it, which is what lets a volatile frame be
an ordinary reference everywhere else — the dtype check, a panel's digest scope,
`load_artifact` in a node, and the widget's own fetch all work on one unchanged.
`adopt` copies one into the store when a run records it, so "returned media is
kept, emitted media is not" stays true.

The bytes then go down the flows websocket as a length-prefixed binary frame,
sent just ahead of the `message_value` naming them, so a tile has the frame when
it hears the value moved. Nothing is pushed unasked: a client names the messages
it is drawing (`{"type":"media","names":[…]}`), a panel's list is intersected
with the scope it already had, and only the newest frame per name in a batch is
sent — a client that fell behind is not handed frames it would draw over. The
tunnel relays text only, so a screen reached through a portal falls back to
fetching, which is why the rate table now has two rows.

Around the edges: the remote worker's fetch cache is bounded at last
(`FLUKSIO_ARTIFACT_CACHE_BYTES`), since content addressing means nothing in it
ever expires and a media stream fills it with chunks nothing asks for twice; a
port carrying an image draws the frame in the node panel rather than only
saying `image/png · frame.png · 1.79kB`; and an edge chip says that much instead
of a line of hash. The media screenshot stops waiting for `networkidle` — a
camera is a socket that never goes quiet, which is the point of it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YC4u66vjzW54fnHu5Juhh9
This commit is contained in:
2026-09-02 10:15:14 +02:00
co-authored by Claude Opus 5
parent 518231aa39
commit d471614e6a
29 changed files with 1101 additions and 147 deletions
+4 -1
View File
@@ -61,7 +61,10 @@ def emit(**ports: Any) -> None:
def save_artifact( def save_artifact(
source: Any, name: str = "", media_type: str = "application/octet-stream" source: Any,
name: str = "",
media_type: str = "application/octet-stream",
volatile: bool = False,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Put bytes in the artifact store and return a reference to them.""" """Put bytes in the artifact store and return a reference to them."""
raise RuntimeError(_OUTSIDE.format(name="save_artifact", verb="save to")) raise RuntimeError(_OUTSIDE.format(name="save_artifact", verb="save to"))
+6 -1
View File
@@ -80,6 +80,7 @@ async def put_artifact(
request: Request, request: Request,
name: str = Query(default=""), name: str = Query(default=""),
media_type: str = Query(default=""), media_type: str = Query(default=""),
volatile: bool = Query(default=False),
) -> Any: ) -> Any:
"""Store the request body and answer with the reference to it. """Store the request body and answer with the reference to it.
@@ -87,6 +88,10 @@ async def put_artifact(
legitimate a body here as a checkpoint, and neither should have to fit in legitimate a body here as a checkpoint, and neither should have to fit in
memory twice. Capped, because nothing else here was: any account, and any memory twice. Capped, because nothing else here was: any account, and any
worker credential, could otherwise fill the data volume. worker credential, could otherwise fill the data volume.
``volatile`` puts it in the ring instead of the store — a frame a screen is
watching now, which the oldest of falls out of memory rather than being
kept. A worker on another host publishing a camera sends this.
""" """
store = _store(request) store = _store(request)
cap = settings.MAX_ARTIFACT_BYTES cap = settings.MAX_ARTIFACT_BYTES
@@ -112,7 +117,7 @@ async def put_artifact(
# as long as the upload lasts. # as long as the upload lasts.
await run_in_threadpool(handle.write, chunk) await run_in_threadpool(handle.write, chunk)
return await run_in_threadpool( return await run_in_threadpool(
store.put_file, Path(handle.name), media_type, name store.put_file, Path(handle.name), media_type, name, volatile
) )
finally: finally:
Path(handle.name).unlink(missing_ok=True) Path(handle.name).unlink(missing_ok=True)
+96 -1
View File
@@ -28,6 +28,7 @@ from fluksio.api.deps import (
user_from_token, user_from_token,
) )
from fluksio.core.db import engine from fluksio.core.db import engine
from fluksio.flow.artifacts import is_reference
from fluksio.flow.controller import FlowController from fluksio.flow.controller import FlowController
from fluksio.flow.dashboards import DashboardStore from fluksio.flow.dashboards import DashboardStore
from fluksio.flow.events import event_bus from fluksio.flow.events import event_bus
@@ -931,6 +932,88 @@ def event_for_panel(event: dict[str, Any], only: set[str]) -> bool:
# should not be handed the whole queue in one message. # should not be handed the whole queue in one message.
MAX_FRAME_EVENTS = 64 MAX_FRAME_EVENTS = 64
#: How many message names one socket may ask for bytes on. A screen draws a
#: handful of tiles; this is only here so a client cannot ask for the whole
#: namespace and be served every frame of it.
MAX_MEDIA_NAMES = 32
def wanted_names(
frame: str, only: set[str] | None, previous: set[str]
) -> set[str] | None:
"""What this client is asking to be sent bytes for, or None if it said
something else.
A client asks by name — the tiles it is drawing — and is answered only for
the names a panel credential would have been allowed anyway. Bytes are the
expensive thing on this socket, so nothing is pushed until something says
it is looking at it.
"""
try:
payload = orjson.loads(frame)
except orjson.JSONDecodeError:
return None
if not isinstance(payload, dict) or payload.get("type") != "media":
return None
names = payload.get("names")
if not isinstance(names, list):
return set()
asked = {str(name) for name in names[:MAX_MEDIA_NAMES]}
return asked if only is None else asked & only
def media_frames(
events: list[dict[str, Any]], wanted: set[str], store: Any
) -> list[bytes]:
"""The bytes behind the media values in this batch, one frame each.
Referenced-then-fetched costs a round trip per frame, which is what keeps
the message plane at a glance rather than a view. Pushing the bytes down
the socket that already carries the event closes that, and only for the
frames a client said it was drawing.
Only what the ring holds: the durable store is what a fetch is for, and a
checkpoint has no business being pushed at anyone. The newest value per
name wins, so a client that fell behind is not handed a backlog of frames
it would only draw over.
"""
if not wanted or store is None or getattr(store, "volatile", None) is None:
return []
newest: dict[str, dict[str, Any]] = {}
for event in events:
if event.get("type") != "message_value":
continue
name = str(event.get("name") or "")
if name not in wanted or not is_reference(event.get("value")):
continue
newest[name] = event
frames: list[bytes] = []
for name, event in newest.items():
value = event["value"]
digest = str(value.get("digest") or "")
path = store.volatile.path(digest)
if path is None:
continue
try:
payload = path.read_bytes()
except OSError:
continue
header = orjson.dumps(
{
"type": "media",
"name": name,
"digest": digest,
"media_type": str(value.get("media_type") or ""),
"ts": event.get("ts"),
}
)
# Length-prefixed, so one frame carries both halves and the reader
# never has to guess where the JSON stops.
frames.append(len(header).to_bytes(4, "big") + header + payload)
return frames
async def _send(websocket: WebSocket, events: list[dict[str, Any]]) -> None: async def _send(websocket: WebSocket, events: list[dict[str, Any]]) -> None:
"""One event, or a batch of them under `events`. """One event, or a batch of them under `events`.
@@ -962,12 +1045,16 @@ async def flow_events(websocket: WebSocket, token: str = "") -> None:
await websocket.close(code=1008) await websocket.close(code=1008)
return return
only = panel_scope(token, websocket.app) only = panel_scope(token, websocket.app)
# Nothing until a client says it is drawing something: bytes are what this
# socket cannot afford to send speculatively.
wanted: set[str] = set()
await websocket.accept() await websocket.accept()
controller: FlowController | None = getattr( controller: FlowController | None = getattr(
websocket.app.state, "flow_controller", None websocket.app.state, "flow_controller", None
) )
store = getattr(websocket.app.state, "artifact_store", None)
async def send_snapshot() -> None: async def send_snapshot() -> None:
if controller is not None: if controller is not None:
@@ -989,7 +1076,10 @@ async def flow_events(websocket: WebSocket, token: str = "") -> None:
# stream — a keepalive, or anything else it decides to # stream — a keepalive, or anything else it decides to
# say, used to be read as the client going away and cost # say, used to be read as the client going away and cost
# it every live update from then on. # it every live update from then on.
receiver.exception() if receiver.exception() is None:
asked = wanted_names(receiver.result(), only, wanted)
if asked is not None:
wanted = asked
receiver = asyncio.create_task(websocket.receive_text()) receiver = asyncio.create_task(websocket.receive_text())
if sender not in done: if sender not in done:
continue continue
@@ -1017,6 +1107,7 @@ async def flow_events(websocket: WebSocket, token: str = "") -> None:
# token — and that would widen this socket to # token — and that would widen this socket to
# everything on the bus. # everything on the bus.
only = panel_scope(token, websocket.app) or set() only = panel_scope(token, websocket.app) or set()
wanted &= only
if out: if out:
await _send(websocket, out) await _send(websocket, out)
out = [] out = []
@@ -1025,6 +1116,10 @@ async def flow_events(websocket: WebSocket, token: str = "") -> None:
continue continue
out.append(event) out.append(event)
if out: if out:
# The bytes first: a tile that has the frame when the
# value lands draws it in one pass rather than two.
for frame in media_frames(out, wanted, store):
await websocket.send_bytes(frame)
await _send(websocket, out) await _send(websocket, out)
except (WebSocketDisconnect, RuntimeError): except (WebSocketDisconnect, RuntimeError):
# A peer that goes away mid-send takes the RuntimeError route # A peer that goes away mid-send takes the RuntimeError route
+11
View File
@@ -150,6 +150,17 @@ class Settings(BaseSettings):
# Storing bytes and recording the reference are two steps; this is the # Storing bytes and recording the reference are two steps; this is the
# window between them. # window between them.
ARTIFACT_GC_GRACE_S: int = 3600 ARTIFACT_GC_GRACE_S: int = 3600
#: Where frames a flow only shows live are held: memory rather than the
#: data volume, so a camera at ten frames a second is not writing to an SD
#: card. Empty works it out — a directory under `/dev/shm` named for this
#: data directory, so two instances on one host do not trim each other —
#: and falls back to the temporary directory where there is no `/dev/shm`.
ARTIFACT_VOLATILE_DIR: Path | None = None
#: How much the volatile ring holds before the oldest frames fall out of
#: it; 0 turns it off, and a volatile save then lands in the durable store
#: like any other. Under Docker's default 64 MB `/dev/shm`: raise
#: `shm_size` with it.
ARTIFACT_VOLATILE_BYTES: int = 48 * 1024 * 1024
# Without a Redis host the engine keeps its state in memory. # Without a Redis host the engine keeps its state in memory.
REDIS_HOST: str | None = None REDIS_HOST: str | None = None
REDIS_PORT: int = 6379 REDIS_PORT: int = 6379
+118 -4
View File
@@ -52,6 +52,10 @@ class ArtifactStore:
def __init__(self, root: Path) -> None: def __init__(self, root: Path) -> None:
self.root = root self.root = root
self.root.mkdir(parents=True, exist_ok=True) self.root.mkdir(parents=True, exist_ok=True)
#: A ring of frames held in memory, for media a flow only shows live.
#: Set by the engine at startup; every lookup below falls through to
#: it, which is what lets a volatile reference be an ordinary one.
self.volatile: VolatileStore | None = None
def _path(self, digest: str) -> Path: def _path(self, digest: str) -> Path:
body = digest[len(DIGEST_PREFIX) :] body = digest[len(DIGEST_PREFIX) :]
@@ -60,14 +64,25 @@ class ArtifactStore:
return self.root / body[:2] / body return self.root / body[:2] / body
def put( def put(
self, chunks: Iterable[bytes], name: str = "", media_type: str = "" self,
chunks: Iterable[bytes],
name: str = "",
media_type: str = "",
volatile: bool = False,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Store a stream and return the reference to it. """Store a stream and return the reference to it.
Written to a temporary file first and moved into place once the digest Written to a temporary file first and moved into place once the digest
is known, so a half-written artifact never has a name anyone can find. is known, so a half-written artifact never has a name anyone can find.
A file already there is left alone: identical content is identical. A file already there is left alone: identical content is identical.
``volatile`` puts it in the ring instead, where it is held in memory
and falls out once newer frames need the room. The reference is the
same shape either way — what differs is how long the bytes last, and a
caller that wants one kept returns it from a run.
""" """
if volatile and self.volatile is not None:
return self.volatile.put(chunks, name=name, media_type=media_type)
digester = hashlib.sha256() digester = hashlib.sha256()
size = 0 size = 0
handle = tempfile.NamedTemporaryFile(dir=self.root, delete=False) handle = tempfile.NamedTemporaryFile(dir=self.root, delete=False)
@@ -98,21 +113,52 @@ class ArtifactStore:
} }
def put_file( def put_file(
self, path: Path, media_type: str = "", name: str = "" self,
path: Path,
media_type: str = "",
name: str = "",
volatile: bool = False,
) -> dict[str, Any]: ) -> dict[str, Any]:
with path.open("rb") as handle: with path.open("rb") as handle:
return self.put( return self.put(
iter(lambda: handle.read(CHUNK), b""), iter(lambda: handle.read(CHUNK), b""),
name=name or path.name, name=name or path.name,
media_type=media_type, media_type=media_type,
volatile=volatile,
) )
def path(self, digest: str) -> Path | None: def path(self, digest: str) -> Path | None:
"""Where the bytes are, or None if this store does not have them.""" """Where the bytes are, or None if this store does not have them.
The ring is looked in second, so everything that resolves a digest —
serving one over HTTP, checking a run input still exists, a panel's
scope — reaches a live frame without knowing there are two stores.
"""
if not valid_digest(digest): if not valid_digest(digest):
return None return None
target = self._path(digest) target = self._path(digest)
return target if target.exists() else None if target.exists():
return target
if self.volatile is not None:
return self.volatile.path(digest)
return None
def adopt(self, digest: str) -> bool:
"""Copy a volatile artifact into this store, so it outlives the ring.
What makes "emitted media is not kept, returned media is" true: a run
recording a reference calls this, and the frame stops being one the
next few seconds can evict.
"""
if self.volatile is None or not valid_digest(digest):
return False
if self._path(digest).exists():
return True
source = self.volatile.path(digest)
if source is None:
return False
self.put_file(source)
return True
def read(self, digest: str) -> Iterator[bytes]: def read(self, digest: str) -> Iterator[bytes]:
target = self.path(digest) target = self.path(digest)
@@ -150,3 +196,71 @@ class ArtifactStore:
except OSError: except OSError:
logger.warning("Could not remove artifact %s", entry.name) logger.warning("Could not remove artifact %s", entry.name)
return removed return removed
class VolatileStore(ArtifactStore):
"""A bounded ring of artifacts, held wherever memory is cheaper than disk.
A camera publishing ten frames a second is ten files a second, and on the
wall panel this is built for that disk is an SD card. So the frames go to a
memory-backed directory instead and the oldest fall out once the ring is
full: nothing sweeps it, because a frame nobody kept is not worth a pass
over the store to find.
It is an ``ArtifactStore``, digest layout and all, which is what lets a
frame be an ordinary reference — the dtype check, the panel's scope, a
node opening one with ``load_artifact`` and the widget fetching one all
work on it unchanged.
"""
def __init__(self, root: Path, limit_bytes: int) -> None:
super().__init__(root)
self.limit_bytes = limit_bytes
def put(
self,
chunks: Iterable[bytes],
name: str = "",
media_type: str = "",
volatile: bool = False,
) -> dict[str, Any]:
reference = super().put(chunks, name=name, media_type=media_type)
self.trim()
return reference
def trim(self) -> int:
"""Drop the oldest until the ring is inside its bound.
Also called on a timer, because a worker in this container writes here
itself and the engine never sees that ``put``.
# ponytail: a scandir per trim. An in-memory index if a ring of
# thousands of frames ever shows up in a profile.
"""
if self.limit_bytes <= 0:
return 0
entries: list[tuple[float, int, Path]] = []
total = 0
for entry in self.root.glob("*/*"):
try:
stat = entry.stat()
except OSError:
continue
if not entry.is_file():
continue
entries.append((stat.st_mtime, stat.st_size, entry))
total += stat.st_size
if total <= self.limit_bytes:
return 0
removed = 0
for _mtime, size, entry in sorted(entries):
if total <= self.limit_bytes:
break
try:
entry.unlink()
except OSError:
continue
total -= size
removed += 1
return removed
+9 -1
View File
@@ -141,6 +141,7 @@ class ConnectorNode(Node):
data: bytes, data: bytes,
name: str = "", name: str = "",
media_type: str = "application/octet-stream", media_type: str = "application/octet-stream",
volatile: bool = False,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Store bytes and return the reference to publish on a media port. """Store bytes and return the reference to publish on a media port.
@@ -149,6 +150,11 @@ class ConnectorNode(Node):
store and the reference names them, which is what an ``image``, store and the reference names them, which is what an ``image``,
``audio`` or ``video`` port carries. ``audio`` or ``video`` port carries.
``volatile`` is what a camera publishes with: the bytes go to a ring
held in memory rather than to the data volume, are pushed to whatever
screen is watching, and last seconds. Use it for a frame; leave it off
for a recording somebody asked to keep.
Only available once the node has started — the store belongs to the Only available once the node has started — the store belongs to the
engine, and is handed over then. engine, and is handed over then.
""" """
@@ -156,7 +162,9 @@ class ConnectorNode(Node):
raise RuntimeError( raise RuntimeError(
"no artifact store: a connector can only save bytes once it has started" "no artifact store: a connector can only save bytes once it has started"
) )
return self._artifacts.put([data], name=name, media_type=media_type) return self._artifacts.put(
[data], name=name, media_type=media_type, volatile=volatile
)
async def start(self, app: FastAPI | None = None) -> None: async def start(self, app: FastAPI | None = None) -> None:
self._artifacts = getattr(app.state, "artifact_store", None) if app else None self._artifacts = getattr(app.state, "artifact_store", None) if app else None
+5
View File
@@ -1313,6 +1313,11 @@ class RunService:
with Session(db_engine) as session: with Session(db_engine) as session:
session.merge(row) session.merge(row)
for message, ref in outcome.artifacts.items(): for message, ref in outcome.artifacts.items():
# A run recording a reference is what "returned media is
# kept" means: copied out of the volatile ring, or the row
# would outlive the bytes it names by a few seconds.
if self._artifacts is not None:
self._artifacts.adopt(str(ref.get("digest") or ""))
session.merge( session.merge(
RunArtifact( RunArtifact(
run_id=run_id, run_id=run_id,
+49 -2
View File
@@ -1,16 +1,19 @@
import asyncio import asyncio
import contextlib import contextlib
import hashlib
import inspect import inspect
import logging import logging
import tempfile
from collections.abc import AsyncIterator, Callable, Coroutine from collections.abc import AsyncIterator, Callable, Coroutine
from contextlib import AbstractAsyncContextManager, asynccontextmanager from contextlib import AbstractAsyncContextManager, asynccontextmanager
from pathlib import Path
from typing import Any from typing import Any
from fastapi import FastAPI, Request from fastapi import FastAPI, Request
from fastapi.concurrency import run_in_threadpool from fastapi.concurrency import run_in_threadpool
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from fastapi.routing import APIRoute from fastapi.routing import APIRoute
from fluksio_worker.worker_main import ARTIFACT_DIR_ENV from fluksio_worker.worker_main import ARTIFACT_DIR_ENV, ARTIFACT_VOLATILE_DIR_ENV
from starlette.middleware.cors import CORSMiddleware from starlette.middleware.cors import CORSMiddleware
from fluksio import __version__ from fluksio import __version__
@@ -23,7 +26,7 @@ from fluksio.core.db import engine as db_engine
from fluksio.core.db import prepare from fluksio.core.db import prepare
from fluksio.flow import logs, modules from fluksio.flow import logs, modules
from fluksio.flow.alerts import AlertManager from fluksio.flow.alerts import AlertManager
from fluksio.flow.artifacts import ArtifactStore from fluksio.flow.artifacts import ArtifactStore, VolatileStore
from fluksio.flow.controller import FlowController, RebuildBusy from fluksio.flow.controller import FlowController, RebuildBusy
from fluksio.flow.dashboards import DashboardStore from fluksio.flow.dashboards import DashboardStore
from fluksio.flow.events import event_bus from fluksio.flow.events import event_bus
@@ -113,6 +116,43 @@ async def _sweep_artifacts(store: ArtifactStore, controller: FlowController) ->
logger.exception("Artifact sweep failed") logger.exception("Artifact sweep failed")
#: How often the ring is measured. Seconds rather than the sweep's hour: it is
#: bounded by size and the frames arriving are what push the old ones out.
VOLATILE_TRIM_S = 5.0
async def _trim_volatile(ring: VolatileStore) -> None:
"""Hold the volatile ring inside its bound.
``put`` trims what it wrote, but a worker in this container writes to the
ring itself and the engine never sees that one — so the bound needs
something of its own watching it.
"""
if ring.limit_bytes <= 0:
return
while True:
await asyncio.sleep(VOLATILE_TRIM_S)
try:
await run_in_threadpool(ring.trim)
except Exception:
logger.exception("Could not trim the volatile artifact ring")
def _volatile_root() -> Path:
"""Where the ring goes: memory if this machine has some to lend.
Named for the data directory rather than fixed, so two engines on one host
have a ring each instead of quietly evicting each other's frames.
"""
configured = settings.ARTIFACT_VOLATILE_DIR
if configured is not None:
return configured
tag = hashlib.sha256(str(settings.DATA_DIR.resolve()).encode()).hexdigest()[:8]
shm = Path("/dev/shm")
parent = shm if shm.is_dir() else Path(tempfile.gettempdir())
return parent / f"fluksio-volatile-{tag}"
def _mcp_sessions() -> AbstractAsyncContextManager[None]: def _mcp_sessions() -> AbstractAsyncContextManager[None]:
"""The MCP session manager's run scope, or nothing when MCP is off.""" """The MCP session manager's run scope, or nothing when MCP is off."""
if not settings.MCP_ENABLED: if not settings.MCP_ENABLED:
@@ -205,6 +245,11 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
# Beside the flows rather than in them: an artifact is what a run produced, # Beside the flows rather than in them: an artifact is what a run produced,
# not something anyone wrote, so it has no business in the git repository. # not something anyone wrote, so it has no business in the git repository.
artifacts = ArtifactStore(settings.FLOWS_DIR.parent / "artifacts") artifacts = ArtifactStore(settings.FLOWS_DIR.parent / "artifacts")
# Frames a flow only shows live, held in memory and bounded by size.
# The store falls through to it, so a volatile reference is an
# ordinary one everywhere but in how long its bytes last.
volatile = VolatileStore(_volatile_root(), settings.ARTIFACT_VOLATILE_BYTES)
artifacts.volatile = volatile
app.state.artifact_store = artifacts app.state.artifact_store = artifacts
accountant = ResourceAccountant( accountant = ResourceAccountant(
cpus=settings.FLOW_CPUS, gpus=settings.FLOW_GPUS cpus=settings.FLOW_CPUS, gpus=settings.FLOW_GPUS
@@ -223,6 +268,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
# is given a URL instead. Node code calls the same two functions. # is given a URL instead. Node code calls the same two functions.
env={ env={
ARTIFACT_DIR_ENV: str(artifacts.root), ARTIFACT_DIR_ENV: str(artifacts.root),
ARTIFACT_VOLATILE_DIR_ENV: str(volatile.root),
# Every slot can be busy at once, so a worker left to size its own # Every slot can be busy at once, so a worker left to size its own
# thread pool to the machine means as many processes as there are # thread pool to the machine means as many processes as there are
# slots, each believing it has the whole of it. A node that says # slots, each believing it has the whole of it. A node that says
@@ -292,6 +338,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
_background(alerts.run(), "alert-manager") _background(alerts.run(), "alert-manager")
_background(MetricsCollector(event_bus).run(), "metrics-collector") _background(MetricsCollector(event_bus).run(), "metrics-collector")
_background(_sweep_artifacts(artifacts, controller), "artifact-gc") _background(_sweep_artifacts(artifacts, controller), "artifact-gc")
_background(_trim_volatile(volatile), "artifact-ring")
await controller.start() await controller.start()
started.append(controller.stop) started.append(controller.stop)
run_service.start() run_service.start()
+80
View File
@@ -838,3 +838,83 @@ def test_a_panel_fetches_only_the_media_its_tiles_are_showing(
).status_code ).status_code
== 403 == 403
) )
def test_a_socket_pushes_only_the_frames_it_was_asked_for(tmp_path) -> None:
"""Bytes are the one thing this socket cannot send speculatively.
A frame a second per subscriber is affordable; every frame to every open
editor tab is not. So nothing goes until a client names what it is drawing,
and a panel credential can only name what it was already allowed to see.
"""
import orjson
from fluksio.api.routes.flows import media_frames, wanted_names
from fluksio.flow.artifacts import ArtifactStore, VolatileStore
store = ArtifactStore(tmp_path / "artifacts")
store.volatile = VolatileStore(tmp_path / "ring", limit_bytes=1_000_000)
frame = store.put([b"\x89PNG..."], name="f.png", media_type="image/png",
volatile=True)
kept = store.put([b"checkpoint"], name="w.pt")
asking = orjson.dumps({"type": "media", "names": ["cam.frame", "other.frame"]})
# A person's socket gets what it asked for; a panel's is intersected with
# what it draws, so naming a message is not a way around the scope.
assert wanted_names(asking.decode(), None, set()) == {"cam.frame", "other.frame"}
assert wanted_names(asking.decode(), {"cam.frame"}, set()) == {"cam.frame"}
# Anything else on this socket leaves the set alone.
assert wanted_names('{"type":"ping"}', None, set()) is None
assert wanted_names("not json", None, set()) is None
events = [
{"type": "message_value", "name": "cam.frame", "value": frame, "ts": 1.0},
{"type": "message_value", "name": "other.frame", "value": frame, "ts": 1.0},
{"type": "message_value", "name": "cam.model", "value": kept, "ts": 1.0},
{"type": "node_log", "name": "cam.frame", "value": frame},
]
frames = media_frames(events, {"cam.frame", "cam.model"}, store)
# One frame: the ring's, for the name that asked. The durable checkpoint is
# what a fetch is for, and a log is not a value.
assert len(frames) == 1
length = int.from_bytes(frames[0][:4], "big")
header = orjson.loads(frames[0][4 : 4 + length])
assert header == {
"type": "media",
"name": "cam.frame",
"digest": frame["digest"],
"media_type": "image/png",
"ts": 1.0,
}
assert frames[0][4 + length :] == b"\x89PNG..."
# Nothing asked for, nothing sent.
assert media_frames(events, set(), store) == []
def test_only_the_newest_frame_of_a_batch_is_pushed(tmp_path) -> None:
"""A client that fell behind is not handed frames it would only draw over."""
import orjson
from fluksio.api.routes.flows import media_frames
from fluksio.flow.artifacts import ArtifactStore, VolatileStore
store = ArtifactStore(tmp_path / "artifacts")
store.volatile = VolatileStore(tmp_path / "ring", limit_bytes=1_000_000)
first = store.put([b"one"], media_type="image/png", volatile=True)
second = store.put([b"two"], media_type="image/png", volatile=True)
frames = media_frames(
[
{"type": "message_value", "name": "cam.frame", "value": first, "ts": 1.0},
{"type": "message_value", "name": "cam.frame", "value": second, "ts": 2.0},
],
{"cam.frame"},
store,
)
assert len(frames) == 1
length = int.from_bytes(frames[0][:4], "big")
assert orjson.loads(frames[0][4 : 4 + length])["digest"] == second["digest"]
assert frames[0][4 + length :] == b"two"
+81
View File
@@ -730,3 +730,84 @@ def test_cancelling_reaches_a_node_running_in_a_child(pool):
assert pool.cancel("demo.slow"), "the parent must find a child's node" assert pool.cancel("demo.slow"), "the parent must find a child's node"
thread.join(timeout=10) thread.join(timeout=10)
assert failed assert failed
def test_a_node_saves_a_frame_to_the_ring(tmp_path):
"""A camera writes to memory, and the next node reads it from there.
The point of the ring being an ordinary store: node code says one word
more, and everything downstream the dtype, ``load_artifact``, the widget
carries on not knowing there are two of them.
"""
from fluksio_worker.worker_main import ARTIFACT_VOLATILE_DIR_ENV
from fluksio.flow.artifacts import VolatileStore
store = ArtifactStore(tmp_path / "artifacts")
store.volatile = VolatileStore(tmp_path / "ring", limit_bytes=1_000_000)
pool = PythonWorkerPool(
python=sys.executable,
size=1,
env={
ARTIFACT_DIR_ENV: str(store.root),
ARTIFACT_VOLATILE_DIR_ENV: str(store.volatile.root),
},
)
pool.start()
try:
ref = pool.run(
"demo",
"frame",
"import fluksio\n"
"def process():\n"
" return {'frame': fluksio.save_artifact(\n"
" b'\\x89PNG' + b'p' * 64, 'f.png',\n"
" media_type='image/png', volatile=True)}\n",
{},
"demo.frame",
timeout=10,
)["frame"]
assert MessageSpec(name="frame", dtype=DType.IMAGE).check(ref) is None
# In the ring, and nowhere near the data volume.
assert store.volatile.path(ref["digest"]) is not None
assert not (store.root / ref["digest"][7:9]).exists()
read = pool.run(
"demo",
"read",
"import fluksio\n"
"def process(frame):\n"
" with open(fluksio.load_artifact(frame), 'rb') as f:\n"
" return {'size': len(f.read())}\n",
{"frame": ref},
"demo.read",
timeout=10,
)
assert read == {"size": 68}
finally:
pool.stop()
def test_the_fetch_cache_is_bounded(tmp_path):
"""A worker downloading a media stream fills its cache with chunks nothing
will ask for twice, and content addressing means nothing ever expires."""
import os
from fluksio_worker.worker_main import ARTIFACT_CACHE_BYTES_ENV, _trim_cache
cache = tmp_path / "cache"
cache.mkdir()
for index in range(5):
entry = cache / f"chunk{index}"
entry.write_bytes(b"c" * 1000)
os.utime(entry, (index, index))
os.environ[ARTIFACT_CACHE_BYTES_ENV] = "3000"
try:
_trim_cache(str(cache))
finally:
del os.environ[ARTIFACT_CACHE_BYTES_ENV]
left = sorted(entry.name for entry in cache.iterdir())
assert left == ["chunk2", "chunk3", "chunk4"]
+74 -1
View File
@@ -3,10 +3,13 @@ import time
from datetime import UTC, datetime from datetime import UTC, datetime
import pytest import pytest
from fastapi.testclient import TestClient
from sqlmodel import Session, col, select from sqlmodel import Session, col, select
from fluksio.core.config import settings
from fluksio.core.db import engine as db_engine from fluksio.core.db import engine as db_engine
from fluksio.flow.artifacts import ArtifactStore from fluksio.flow.artifacts import ArtifactStore, VolatileStore
from fluksio.flow.runs import new_run_id, sweep_artifacts from fluksio.flow.runs import new_run_id, sweep_artifacts
from fluksio.flow.state import MemoryState from fluksio.flow.state import MemoryState
from fluksio.models import Run, RunArtifact from fluksio.models import Run, RunArtifact
@@ -108,3 +111,73 @@ def test_a_streamed_chunk_falls_out_once_the_message_moves_on(store):
assert sweep_artifacts(store, state, grace_s=5) == 1 assert sweep_artifacts(store, state, grace_s=5) == 1
assert store.path(first["digest"]) is None assert store.path(first["digest"]) is None
assert store.path(second["digest"]) is not None assert store.path(second["digest"]) is not None
def test_a_ring_drops_the_oldest_once_it_is_full(tmp_path):
"""What makes a camera affordable: the room is fixed, not the history."""
ring = VolatileStore(tmp_path / "ring", limit_bytes=3000)
refs = [ring.put([bytes([i]) * 1000], name=f"{i}.bin") for i in range(5)]
held = [ref for ref in refs if ring.path(ref["digest"]) is not None]
assert len(held) == 3
# The newest three, in order: eviction is by age, not by chance.
assert held == refs[2:]
def test_a_volatile_reference_resolves_like_any_other(tmp_path):
"""A frame is an ordinary reference, which is what keeps the rest honest.
Everything that resolves a digest serving one, checking a run's input,
a panel's scope — goes through ``path``, so the ring has to answer there.
"""
store = ArtifactStore(tmp_path / "artifacts")
store.volatile = VolatileStore(tmp_path / "ring", limit_bytes=1_000_000)
ref = store.put([b"frame"], name="f.png", media_type="image/png", volatile=True)
assert store.path(ref["digest"]) is not None
# In the ring, and not on the volume the sweep is about.
assert not (store.root / ref["digest"][7:9]).exists()
def test_a_recorded_frame_is_copied_out_of_the_ring(tmp_path):
"""Emitted media is not kept; returned media is."""
store = ArtifactStore(tmp_path / "artifacts")
store.volatile = VolatileStore(tmp_path / "ring", limit_bytes=1_000_000)
ref = store.put([b"kept"], name="f.png", volatile=True)
assert store.adopt(ref["digest"]) is True
store.volatile.collect(set())
assert store.path(ref["digest"]) is not None
assert store.adopt("sha256:" + "0" * 64) is False
def test_an_upload_may_ask_for_the_ring(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
"""A camera on another host publishes over HTTP and wants the ring too."""
store = client.app.state.artifact_store
previous = store.volatile
store.volatile = VolatileStore(
store.root.parent / "test-ring", limit_bytes=1_000_000
)
try:
url = f"{settings.API_V1_STR}/artifacts"
answer = client.put(
f"{url}?name=f.png&media_type=image%2Fpng&volatile=1",
headers=superuser_token_headers,
content=b"\x89PNG-frame",
)
assert answer.status_code == 200, answer.text
digest = answer.json()["digest"]
assert store.volatile.path(digest) is not None
assert not (store.root / digest[7:9]).exists()
# And it is served back like anything else, which is what the widget
# falls back on when the socket did not push it.
served = client.get(f"{url}/{digest}", headers=superuser_token_headers)
assert served.status_code == 200
assert served.content == b"\x89PNG-frame"
finally:
store.volatile = previous
+5
View File
@@ -141,6 +141,11 @@ services:
volumes: volumes:
- app-flow-data:/data - app-flow-data:/data
# `/dev/shm`, where frames a flow only shows live are held. Docker's
# default is 64 MB, which the ring's own bound sits just under; raise both
# together (ARTIFACT_VOLATILE_BYTES) for more cameras or bigger frames.
shm_size: 128mb
# Deep health: fails when the event loop is wedged or Redis is gone, not # Deep health: fails when the event loop is wedged or Redis is gone, not
# just when the process is dead. Autoheal restarts on unhealthy. # just when the process is dead. Autoheal restarts on unhealthy.
healthcheck: healthcheck:
+12 -2
View File
@@ -137,7 +137,8 @@ def process(speech): # an `audio` port
def process(camera_url): def process(camera_url):
for index, jpeg in enumerate(grab(camera_url)): # a generator for index, jpeg in enumerate(grab(camera_url)): # a generator
frame = fluksio.save_artifact( frame = fluksio.save_artifact(
jpeg, f"frame-{index:05d}.jpg", media_type="image/jpeg" jpeg, f"frame-{index:05d}.jpg",
media_type="image/jpeg", volatile=True,
) )
frame["meta"] = {"seq": index} frame["meta"] = {"seq": index}
yield {"frame": frame} # an `image` stream port yield {"frame": frame} # an `image` stream port
@@ -148,12 +149,21 @@ has to match, so a node declaring `audio` never receives a video by accident.
See [Payload types](../reference/payload-types.md#image-audio-video) for what See [Payload types](../reference/payload-types.md#image-audio-video) for what
each carries and what rates are realistic. each carries and what rates are realistic.
`volatile=True` is for a frame rather than a result. The bytes go to a ring in
memory instead of the data volume, and the engine pushes them down the
websocket to whichever screens are drawing that message — so a camera runs at
ten frames a second without writing anything to disk. They last as long as it
takes newer frames to need the room. Leave it off for a clip somebody asked to
keep.
!!! warning "Emitted media is not kept; returned media is" !!! warning "Emitted media is not kept; returned media is"
Only what a node *returns* is recorded against its run. Frames yielded Only what a node *returns* is recorded against its run. Frames yielded
along the way are replaced in state by the next one, and the artifact sweep along the way are replaced in state by the next one, and the artifact sweep
removes bytes nothing refers to any more, which is what stops a camera removes bytes nothing refers to any more, which is what stops a camera
filling the disk. If a particular frame matters, return it. filling the disk. If a particular frame matters, return it: a volatile one
is copied out of the ring when the run records it, so returning it is also
what makes it outlive the next few seconds.
## Printing ## Printing
+8
View File
@@ -141,6 +141,14 @@ filesystem writes to it directly; one that does not fetches and uploads over
HTTP, using the artifact endpoint beside the socket it already has. Either way HTTP, using the artifact endpoint beside the socket it already has. Either way
your node code is the same two calls. your node code is the same two calls.
A fetch is cached on the worker by digest, since content addressing means an
entry is never stale. Nothing expires on its own, so the cache is bounded by
size and the oldest fall out: `FLUKSIO_ARTIFACT_CACHE` says where it lives
(default a directory in the temporary directory) and
`FLUKSIO_ARTIFACT_CACHE_BYTES` how much it holds (default 1 GiB). Worth raising
where a worker reads the same large inputs repeatedly, and worth leaving alone
where it reads a media stream — those are chunks nothing asks for twice.
## Seeing what is attached ## Seeing what is attached
```sh ```sh
+12 -9
View File
@@ -105,21 +105,24 @@ so an answer to a different question is ignored.
A media widget draws what its message points at: a picture, a clip with A media widget draws what its message points at: a picture, a clip with
controls, a video. Media does not travel as a message; a reference to it does. controls, a video. Media does not travel as a message; a reference to it does.
The tile fetches the bytes behind whichever reference the message holds, The tile takes the bytes behind whichever reference the message holds, and
and redraws when a new one arrives. redraws when a new one arrives.
**Crop or fit** decides how a picture fills the tile. **Play as it arrives** **Crop or fit** decides how a picture fills the tile. **Play as it arrives**
starts a clip by itself, though a browser only plays sound once somebody has starts a clip by itself, though a browser only plays sound once somebody has
touched the page, so a screen nobody has tapped stays silent. touched the page, so a screen nobody has tapped stays silent.
Rate is the thing to get right. A frame every second or two is a glance at a Rate is the thing to get right, and it is decided by the node publishing rather
door, and works; through the portal, make that every few seconds. Live video is than by the tile. A frame the engine is holding in memory is pushed down the
not something to push through the message plane at all. Put the camera's own same websocket that carries the value, so ten a second is a real view on the
address in **Live stream** and the browser plays it from source, leaving the local network; a stored one is fetched, a round trip each, which suits a glance
messages to carry the occasional still that a flow can actually react to. at a door every second or two. Through the portal everything is fetched, so
make it every few seconds there. Above that, put the camera's own address in
**Live stream** and the browser plays it from source, leaving the messages to
carry the occasional still that a flow can actually react to.
Panels see media the same way, and only their own: a screen may fetch the bytes Panels see media the same way, and only their own: a screen is sent, and may
its own tiles are showing and nothing else. fetch, the bytes its own tiles are showing and nothing else.
## Player tiles ## Player tiles
+12
View File
@@ -131,6 +131,9 @@ warning into a refusal to start.
| `OBS_RETENTION_DAYS` | `30` | how long metrics, events and run records are kept | | `OBS_RETENTION_DAYS` | `30` | how long metrics, events and run records are kept |
| `ARTIFACT_GC_INTERVAL_S` | `3600` | how often artifact bytes nothing refers to are swept away; 0 never sweeps | | `ARTIFACT_GC_INTERVAL_S` | `3600` | how often artifact bytes nothing refers to are swept away; 0 never sweeps |
| `ARTIFACT_GC_GRACE_S` | `3600` | how long a freshly written artifact is spared, whatever refers to it | | `ARTIFACT_GC_GRACE_S` | `3600` | how long a freshly written artifact is spared, whatever refers to it |
| `MAX_ARTIFACT_BYTES` | `2147483648` | the largest body `PUT /artifacts` will take; 0 removes the limit |
| `ARTIFACT_VOLATILE_DIR` | worked out | where frames a flow only shows live are held; empty picks a directory under `/dev/shm` named for the data directory, and falls back to the temporary directory |
| `ARTIFACT_VOLATILE_BYTES` | `50331648` | how much that ring holds before the oldest frames fall out; 0 turns it off and volatile saves land in the store |
The three concurrency limits are also flags on `fluksio serve` The three concurrency limits are also flags on `fluksio serve`
(`--max-workers`, `--max-cascades`, `--max-runs`), as is the card count, (`--max-workers`, `--max-cascades`, `--max-runs`), as is the card count,
@@ -146,6 +149,15 @@ sweep is what keeps a flow streaming media from filling the disk. It stands
aside entirely while a run is in flight, since a node may store a checkpoint aside entirely while a run is in flight, since a node may store a checkpoint
long before it returns the reference to it. long before it returns the reference to it.
Frames saved with `volatile=True` skip all of that. They go to a ring in memory
instead of the volume, the oldest falling out once the newest need the room,
and the engine pushes them down the websocket to whichever screens are drawing
them — which is what a camera at ten frames a second needs and the store cannot
give it. A frame a run *records* is copied into the store on the way, so
returned media is kept and emitted media is not. Under Docker the ring lives in
the container's `/dev/shm`, whose default is 64 MB: raise `shm_size` alongside
`ARTIFACT_VOLATILE_BYTES`.
A node that declares nothing is not accounted against `FLOW_CPUS`; it runs on A node that declares nothing is not accounted against `FLOW_CPUS`; it runs on
the shared pool and is given `FLOW_CPUS / FLOW_MAX_WORKERS` as a thread cap, so the shared pool and is given `FLOW_CPUS / FLOW_MAX_WORKERS` as a thread cap, so
several at once cannot each size themselves to the whole machine. Setting several at once cannot each size themselves to the whole machine. Setting
+13 -3
View File
@@ -177,7 +177,9 @@ connector publishes a reference to it instead:
async def poll(self) -> dict[str, Any] | None: async def poll(self) -> dict[str, Any] | None:
jpeg = await asyncio.to_thread(self._grab) jpeg = await asyncio.to_thread(self._grab)
return { return {
"frame": self.save_artifact(jpeg, "frame.jpg", media_type="image/jpeg") "frame": self.save_artifact(
jpeg, "frame.jpg", media_type="image/jpeg", volatile=True
)
} }
``` ```
@@ -186,10 +188,18 @@ async def poll(self) -> dict[str, Any] | None:
works once the node has started, since the store is the engine's and is handed works once the node has started, since the store is the engine's and is handed
over then. over then.
`volatile=True` is what a camera publishes with. The frame goes to a ring in
memory rather than the data volume and is pushed down the websocket to
whichever screens are drawing it, so a wall panel sees ten frames a second and
the SD card under it is never written to. Frames last until newer ones need the
room; leave the flag off for a reading somebody asked to keep.
Each reading is a new artifact, which the poll loop publishes because its Each reading is a new artifact, which the poll loop publishes because its
digest differs from the last. Set `poll_interval` to what somebody actually digest differs from the last. Set `poll_interval` to what somebody actually
wants to look at: a frame every second or two is a glance, and live video wants to look at. Through a portal the bytes are fetched rather than pushed, so
belongs on the camera's own stream rather than in the graph. a remote panel wants a frame every second or two; higher rates than that are
for the local network, and full-rate video still belongs on the camera's own
stream.
## Lifecycle ## Lifecycle
+16 -8
View File
@@ -117,19 +117,27 @@ sequence numbers are for whoever consumes the media.
Bytes still never travel as a message. A camera publishes one reference per Bytes still never travel as a message. A camera publishes one reference per
frame and a microphone one per chunk, which makes a media stream an ordinary frame and a microphone one per chunk, which makes a media stream an ordinary
[streaming port](../concepts/flows.md#streaming-ports), and each frame an [streaming port](../concepts/flows.md#streaming-ports). What that costs
artifact. What that costs is worth knowing before pointing a camera at it: depends on how the bytes reach the screen:
| Rate | Where it works | | Rate | Where it works |
|---|---| |---|---|
| A clip a second (speech) | anywhere, including through the portal | | A clip a second (speech) | anywhere, including through the portal |
| A frame every second or two (a glance at a door) | locally; through the portal, every few seconds | | Ten frames a second (a camera worth watching) | on the local network, with `volatile=True` |
| Live video, 1530 fps | not here — see below | | A frame every second or two (a glance at a door) | anywhere, including through the portal |
| Full-rate video, 30 fps and up | not here — see below |
Real-time video is not a message-plane problem: every frame would be an The difference is one flag. A frame saved with
artifact, an event and a fetch. Point a media widget's **stream URL** at [`volatile=True`](../code/nodes.md#media) is held in memory and pushed down the
whatever the camera already serves and the browser plays it from source; the websocket in front of the value naming it, so a screen draws it without asking
messages then carry the occasional still, and the flow reacts to those. for anything. A stored one is fetched instead: a round trip per frame, which is
a glance rather than a view. Through a portal every frame is fetched, so a
remote panel is in the second row whatever the flag says.
Above that, video is not a message-plane problem — every frame would still be
an event. Point a media widget's **stream URL** at whatever the camera already
serves and the browser plays it from source; the messages then carry the
occasional still, and the flow reacts to those.
### `json` ### `json`
+9 -4
View File
@@ -199,13 +199,18 @@ async function captureDashboards(page, dir) {
* *
* Skipped unless the media example is seeded (root `make seed-example-media`), * Skipped unless the media example is seeded (root `make seed-example-media`),
* since it is the one shot that needs a source of frames. The bytes arrive as * since it is the one shot that needs a source of frames. The bytes arrive as
* a blob the tile fetches them with the session's credential, which no `img` * a blob pushed down the socket, or fetched with the session's credential,
* could carry on its own so a `blob:` source is the proof the whole path ran * which no `img` could carry on its own so a `blob:` source is the proof the
* rather than that a picture is merely present. * whole path ran rather than that a picture is merely present.
*
* The one page that cannot wait for `networkidle`: a camera publishing several
* frames a second is a socket that never goes quiet, which is the point of it.
* The blob source below is a stronger wait anyway it says a frame arrived,
* where idleness only ever said the page stopped asking.
*/ */
async function captureMedia(page, dir) { async function captureMedia(page, dir) {
const answer = await page.goto(`${APP_URL}/view/camera`, { const answer = await page.goto(`${APP_URL}/view/camera`, {
waitUntil: "networkidle", waitUntil: "domcontentloaded",
}) })
if (!answer?.ok()) return if (!answer?.ok()) return
@@ -1,82 +1,17 @@
import { useEffect, useState } from "react" import { isRef, useArtifactUrl } from "@/lib/media"
import { OpenAPI } from "@/client"
import { apiToken } from "@/lib/portal"
import { cn } from "@/lib/utils" import { cn } from "@/lib/utils"
import { useBoundValue } from "./dataContext" import { useBoundValue } from "./dataContext"
import { config, text } from "./ui/core/config" import { config, text } from "./ui/core/config"
import type { WidgetProps } from "./widgets" import type { WidgetProps } from "./widgets"
/** An artifact reference, as a message carries one. */
type MediaRef = {
digest?: string
media_type?: string
name?: string
size?: number
}
const isRef = (value: unknown): value is MediaRef =>
typeof value === "object" &&
value !== null &&
typeof (value as MediaRef).digest === "string" &&
(value as MediaRef).digest!.startsWith("sha256:")
/**
* A local URL for an artifact's bytes, refreshed whenever the digest changes.
*
* Not the endpoint itself: `/artifacts/{digest}` takes a bearer token, and no
* `<img>` or `<audio>` can carry a header. So the bytes come through fetch and
* are handed to the element as an object URL which also means playback never
* asks the server for a range, since the blob is already here.
*
* The URL is revoked when it is replaced, or the tab would hold every frame a
* camera has ever sent for as long as the page is open.
*/
function useArtifactUrl(ref: MediaRef | null): string {
const digest = ref?.digest ?? ""
const mediaType = ref?.media_type ?? ""
const [url, setUrl] = useState("")
useEffect(() => {
if (!digest) {
setUrl("")
return
}
let live = true
let made = ""
const token = apiToken()
const query = mediaType
? `?media_type=${encodeURIComponent(mediaType)}`
: ""
fetch(`${OpenAPI.BASE}/api/v1/artifacts/${digest}${query}`, {
headers: token ? { Authorization: `Bearer ${token}` } : {},
})
.then((answer) => (answer.ok ? answer.blob() : Promise.reject(answer)))
.then((blob) => {
if (!live) return
made = URL.createObjectURL(blob)
setUrl(made)
})
.catch(() => {
if (live) setUrl("")
})
return () => {
live = false
if (made) URL.revokeObjectURL(made)
}
}, [digest, mediaType])
return url
}
/** /**
* What a message's bytes look like: a frame, a clip, a segment. * What a message's bytes look like: a frame, a clip, a segment.
* *
* Media never travels as a message the reference does, and the bytes are * Media never travels as a message the reference does. The bytes come down
* fetched from the artifact store. What that means for a wall panel is a * the socket in front of it where the engine is holding the frame in memory,
* refresh per published frame, which suits a camera glancing every few seconds * and are fetched from the artifact store otherwise; either way this tile only
* rather than a live view; for that, point `stream_url` at whatever the camera * asks for what it is drawing. A camera serving its own stream is still played
* already serves and the browser plays it directly. * from source: point `stream_url` at it and the browser does the work.
*/ */
export function MediaWidget({ widget }: WidgetProps) { export function MediaWidget({ widget }: WidgetProps) {
const cfg = config(widget) const cfg = config(widget)
@@ -85,7 +20,7 @@ export function MediaWidget({ widget }: WidgetProps) {
const live = useBoundValue(message || undefined) const live = useBoundValue(message || undefined)
const value = live?.value const value = live?.value
const ref = isRef(value) ? value : null const ref = isRef(value) ? value : null
const url = useArtifactUrl(ref) const url = useArtifactUrl(ref, message || undefined)
const kind = (ref?.media_type ?? text(cfg.dtype)).split("/")[0] const kind = (ref?.media_type ?? text(cfg.dtype)).split("/")[0]
const fit = text(cfg.fit) === "contain" ? "object-contain" : "object-cover" const fit = text(cfg.fit) === "contain" ? "object-contain" : "object-cover"
@@ -159,7 +159,12 @@ export function EdgeInspector({
Nothing has come through yet. Run the flow to see a value here. Nothing has come through yet. Run the flow to see a value here.
</p> </p>
) : scalar === null ? ( ) : scalar === null ? (
<ValuePreview value={live.value} className="mt-2" defaultOpen /> <ValuePreview
value={live.value}
name={edge.message}
className="mt-2"
defaultOpen
/>
) : null} ) : null}
</PopoverContent> </PopoverContent>
</Popover> </Popover>
@@ -8,6 +8,7 @@ import {
} from "@xyflow/react" } from "@xyflow/react"
import { memo, useEffect, useRef, useState } from "react" import { memo, useEffect, useRef, useState } from "react"
import { describeArtifact, isRef } from "@/lib/media"
import { duration } from "@/lib/motion" import { duration } from "@/lib/motion"
import { cn } from "@/lib/utils" import { cn } from "@/lib/utils"
import type { FlowEdgeData } from "./deriveEdges" import type { FlowEdgeData } from "./deriveEdges"
@@ -66,6 +67,9 @@ function formatValue(value: unknown): string {
return Number.isInteger(value) ? String(value) : value.toFixed(2) return Number.isInteger(value) ? String(value) : value.toFixed(2)
} }
if (typeof value === "string") return value if (typeof value === "string") return value
// A media reference serialised whole is a line of hash: what belongs in a
// chip this size is what kind of bytes crossed the edge.
if (isRef(value)) return describeArtifact(value as Record<string, unknown>)
return JSON.stringify(value) ?? "" return JSON.stringify(value) ?? ""
} }
@@ -119,7 +119,13 @@ export function MessageSparkline({
</p> </p>
) )
} }
return <ValuePreview value={live.value} dtype={dtype} /> return (
<ValuePreview
value={live.value}
name={qualify(flow, message)}
dtype={dtype}
/>
)
} }
// The value is live here, so the dot on the newest reading is earned. The // The value is live here, so the dot on the newest reading is earned. The
+32 -21
View File
@@ -3,7 +3,8 @@ import { useState } from "react"
import type { DType } from "@/client" import type { DType } from "@/client"
import { Marquee } from "@/components/Common/Marquee" import { Marquee } from "@/components/Common/Marquee"
import { cn, si } from "@/lib/utils" import { describeArtifact, isRef, useArtifactUrl } from "@/lib/media"
import { cn } from "@/lib/utils"
/** /**
* How much of a structured value is worth unfolding in a side panel. * How much of a structured value is worth unfolding in a side panel.
@@ -19,11 +20,6 @@ function count(n: number, one: string, many = `${one}s`): string {
return `${n} ${n === 1 ? one : many}` return `${n} ${n === 1 ? one : many}`
} }
/** Whether this is an artifact reference rather than data of its own. */
function isArtifact(value: Record<string, unknown>): boolean {
return typeof value.digest === "string" && value.digest.startsWith("sha256:")
}
/** /**
* What a structured value *is*, in the space a value would have taken. * What a structured value *is*, in the space a value would have taken.
* *
@@ -38,21 +34,7 @@ function summarize(value: object, dtype?: DType): string {
} }
const record = value as Record<string, unknown> const record = value as Record<string, unknown>
if (isArtifact(record)) { if (isRef(record)) return describeArtifact(record)
const name =
typeof record.name === "string" && record.name ? record.name : ""
const size = typeof record.size === "number" ? `${si(record.size)}B` : ""
// The media type when there is one: what kind of bytes these are is the
// first thing worth knowing about a frame or a clip, and it is what the
// port's own type was declared against.
const media =
typeof record.media_type === "string" &&
record.media_type &&
record.media_type !== "application/octet-stream"
? record.media_type
: "artifact"
return [media, name, size].filter(Boolean).join(" · ")
}
if (Array.isArray(record.lines)) { if (Array.isArray(record.lines)) {
const points = record.lines.reduce( const points = record.lines.reduce(
@@ -77,13 +59,39 @@ function summarize(value: object, dtype?: DType): string {
* which is what keeps a panel four hundred pixels wide from being pushed open * which is what keeps a panel four hundred pixels wide from being pushed open
* by one checkpoint reference. * by one checkpoint reference.
*/ */
/**
* The frame itself, where the value is one.
*
* A port carrying an image says `image/png · frame.png · 48kB`, which is the
* right answer while wiring and the wrong one while pointing a camera. Drawn
* small: this is a side panel, and the tile is where a frame is looked at.
*/
function Thumbnail({ value, name }: { value: object; name?: string }) {
const url = useArtifactUrl(value, name)
if (!url) return null
return (
<img
src={url}
alt={String((value as { name?: string }).name ?? "frame")}
className="mt-1 max-h-24 w-full rounded-sm object-contain"
data-testid="value-preview-thumbnail"
/>
)
}
export function ValuePreview({ export function ValuePreview({
value, value,
name,
dtype, dtype,
defaultOpen = false, defaultOpen = false,
className, className,
}: { }: {
value: unknown value: unknown
/**
* The message this value is on. Only needed to draw a live frame: it is what
* asks the engine to push the bytes for it.
*/
name?: string
dtype?: DType dtype?: DType
/** Start unfolded, where there is room for it — an inspector, not a row. */ /** Start unfolded, where there is room for it — an inspector, not a row. */
defaultOpen?: boolean defaultOpen?: boolean
@@ -91,6 +99,8 @@ export function ValuePreview({
}) { }) {
const [open, setOpen] = useState(defaultOpen) const [open, setOpen] = useState(defaultOpen)
const structured = value !== null && typeof value === "object" const structured = value !== null && typeof value === "object"
const image =
structured && isRef(value) && (value.media_type ?? "").startsWith("image/")
if (!structured) { if (!structured) {
return ( return (
@@ -118,6 +128,7 @@ export function ValuePreview({
/> />
<Marquee text={summarize(value, dtype)} className="flex-1 font-mono" /> <Marquee text={summarize(value, dtype)} className="flex-1 font-mono" />
</button> </button>
{image ? <Thumbnail value={value as object} name={name} /> : null}
{open ? ( {open ? (
<pre <pre
className={cn( className={cn(
+83
View File
@@ -83,6 +83,26 @@ const listeners = new Map<string, Set<Listener>>()
let connected = false let connected = false
const connectionListeners = new Set<Listener>() const connectionListeners = new Set<Listener>()
/**
* Object URLs for frames the engine pushed, keyed by digest.
*
* Small on purpose: a camera's frames are worth exactly as long as the next
* one takes to arrive, and a tab that held every one of them would grow
* without bound. The oldest is revoked when the room runs out.
*/
const BYTES_LIMIT = 8
const bytes = new Map<string, string>()
/**
* Which messages this page is drawing bytes for, and how many tiles each.
*
* Refcounted because two tiles may show one camera and the first to unmount
* must not stop the second's frames. The socket sends the key set whenever it
* changes; nothing is pushed for a name nobody is looking at.
*/
const wanted = new Map<string, number>()
const wantedListeners = new Set<Listener>()
function notify(key: string) { function notify(key: string) {
for (const listener of listeners.get(key) ?? []) listener() for (const listener of listeners.get(key) ?? []) listener()
} }
@@ -247,6 +267,53 @@ export const liveStore = {
isConnected() { isConnected() {
return connected return connected
}, },
/**
* Take a frame the socket pushed, as an object URL the tile can draw.
*
* Content addressing means a digest already here is the same bytes, so the
* blob is dropped rather than replacing an identical one.
*/
setBytes(digest: string, blob: Blob) {
if (bytes.has(digest)) return
bytes.set(digest, URL.createObjectURL(blob))
while (bytes.size > BYTES_LIMIT) {
const oldest = bytes.keys().next().value
if (oldest === undefined) break
const url = bytes.get(oldest)
bytes.delete(oldest)
if (url) URL.revokeObjectURL(url)
notify(`bytes:${oldest}`)
}
notify(`bytes:${digest}`)
},
getBytes(digest: string) {
return bytes.get(digest)
},
/** Ask for this message's frames while the caller is drawing it. */
wantBytes(name: string) {
wanted.set(name, (wanted.get(name) ?? 0) + 1)
if (wanted.get(name) === 1)
for (const listener of wantedListeners) listener()
return () => {
const left = (wanted.get(name) ?? 1) - 1
if (left > 0) {
wanted.set(name, left)
return
}
wanted.delete(name)
for (const listener of wantedListeners) listener()
}
},
wantedNames() {
return [...wanted.keys()]
},
/** Told when the set changes, so the socket can say so. */
onWanted(listener: Listener) {
wantedListeners.add(listener)
return () => {
wantedListeners.delete(listener)
}
},
reset() { reset() {
for (const key of values.keys()) notify(`value:${key}`) for (const key of values.keys()) notify(`value:${key}`)
values.clear() values.clear()
@@ -264,6 +331,13 @@ export const liveStore = {
notify("logs") notify("logs")
for (const flow of paused) notify(`paused:${flow}`) for (const flow of paused) notify(`paused:${flow}`)
paused.clear() paused.clear()
for (const [digest, url] of bytes) {
URL.revokeObjectURL(url)
notify(`bytes:${digest}`)
}
bytes.clear()
// `wanted` is deliberately kept: the tiles asking are still mounted, and a
// reconnecting socket has to say what they want all over again.
}, },
} }
@@ -274,6 +348,15 @@ export function useLiveValue(name: string | undefined): LiveValue | undefined {
) )
} }
/** The pushed bytes for a digest, if the socket carried them. */
export function useLiveBytes(digest: string | undefined): string | undefined {
return useSyncExternalStore(
(listener) =>
digest ? subscribeKey(`bytes:${digest}`, listener) : () => {},
() => (digest ? bytes.get(digest) : undefined),
)
}
export function useNodeStatus(nodeId: string): LiveStatus | undefined { export function useNodeStatus(nodeId: string): LiveStatus | undefined {
return useSyncExternalStore( return useSyncExternalStore(
(listener) => subscribeKey(`status:${nodeId}`, listener), (listener) => subscribeKey(`status:${nodeId}`, listener),
@@ -7,6 +7,7 @@ import { dashboardKeys, panelKeys } from "@/components/Dashboard/queries"
import { healthKeys } from "@/components/Health/queries" import { healthKeys } from "@/components/Health/queries"
import { runKeys } from "@/components/Runs/queries" import { runKeys } from "@/components/Runs/queries"
import { connectionStore } from "@/lib/connectionStore" import { connectionStore } from "@/lib/connectionStore"
import { parseMediaFrame } from "@/lib/media"
import { apiToken } from "@/lib/portal" import { apiToken } from "@/lib/portal"
import { type LogLine, liveStore, type ValueSource } from "./liveStore" import { type LogLine, liveStore, type ValueSource } from "./liveStore"
import { flowKeys } from "./queries" import { flowKeys } from "./queries"
@@ -160,9 +161,25 @@ function schedule() {
retry = Math.min(retry * 2, RECONNECT_MAX) retry = Math.min(retry * 2, RECONNECT_MAX)
} }
/**
* Say which messages this page wants frames for.
*
* Bytes are the one thing the socket does not send unasked: a camera is
* hundreds of kilobytes a second and most tabs are drawing no media at all. So
* a Media tile or a thumbnail registers its message, and this tells the engine
* again on every reconnect, since the new socket knows nothing.
*/
function sendWanted() {
if (socket?.readyState !== WebSocket.OPEN) return
socket.send(JSON.stringify({ type: "media", names: liveStore.wantedNames() }))
}
liveStore.onWanted(sendWanted)
function connect() { function connect() {
if (watchers === 0 || socket || timer) return if (watchers === 0 || socket || timer) return
const ws = new WebSocket(socketUrl()) const ws = new WebSocket(socketUrl())
ws.binaryType = "arraybuffer"
socket = ws socket = ws
ws.onopen = () => { ws.onopen = () => {
@@ -185,6 +202,7 @@ function connect() {
]) { ]) {
client?.invalidateQueries({ queryKey }) client?.invalidateQueries({ queryKey })
} }
sendWanted()
} }
const handle = (message: FlowEvent) => { const handle = (message: FlowEvent) => {
@@ -318,6 +336,13 @@ function connect() {
// frame rather than the connection: an exception thrown here escapes into // frame rather than the connection: an exception thrown here escapes into
// `window.onerror` and leaves whatever it had already applied behind. // `window.onerror` and leaves whatever it had already applied behind.
try { try {
if (event.data instanceof ArrayBuffer) {
// Media, sent in front of the value that names it — so the tile has
// the bytes by the time it hears the message changed.
const frame = parseMediaFrame(event.data)
if (frame) liveStore.setBytes(frame.header.digest, frame.bytes)
return
}
const payload = JSON.parse(event.data) const payload = JSON.parse(event.data)
if (payload?.type === "batch") { if (payload?.type === "batch") {
// A cascade publishes a dozen events at once and the engine coalesces // A cascade publishes a dozen events at once and the engine coalesces
+63
View File
@@ -0,0 +1,63 @@
/**
* The framing a pushed frame arrives in, and what a malformed one must do.
*
* Run: `bun src/lib/media.check.ts` (there is no unit runner; the suite in
* `tests/` drives a running stack).
*/
import assert from "node:assert/strict"
import { describeArtifact, isRef, parseMediaFrame } from "./media"
function frame(header: object, payload: Uint8Array): ArrayBuffer {
const encoded = new TextEncoder().encode(JSON.stringify(header))
const out = new Uint8Array(4 + encoded.length + payload.length)
new DataView(out.buffer).setUint32(0, encoded.length)
out.set(encoded, 4)
out.set(payload, 4 + encoded.length)
return out.buffer
}
const bytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47])
const parsed = parseMediaFrame(
frame(
{
type: "media",
name: "cam.frame",
digest: `sha256:${"a".repeat(64)}`,
media_type: "image/png",
ts: 1,
},
bytes,
),
)
assert.ok(parsed)
assert.equal(parsed.header.name, "cam.frame")
assert.equal(parsed.header.media_type, "image/png")
assert.equal(parsed.bytes.size, 4)
assert.equal(parsed.bytes.type, "image/png")
// A frame the client cannot read is dropped, never thrown on: this runs on
// every socket message.
assert.equal(parseMediaFrame(new ArrayBuffer(0)), null)
assert.equal(parseMediaFrame(new ArrayBuffer(2)), null)
assert.equal(parseMediaFrame(frame({ name: "n" }, bytes)), null)
// A length longer than the frame is truncation, not a payload.
const short = new Uint8Array(8)
new DataView(short.buffer).setUint32(0, 999)
assert.equal(parseMediaFrame(short.buffer), null)
assert.ok(isRef({ digest: `sha256:${"b".repeat(64)}` }))
assert.equal(isRef({ digest: "nope" }), false)
assert.equal(isRef(null), false)
assert.equal(
describeArtifact({ media_type: "image/png", name: "f.png", size: 48000 }),
"image/png · f.png · 48kB",
)
// No media type is still worth summarising as something.
assert.equal(
describeArtifact({ media_type: "application/octet-stream", size: 12 }),
"artifact · 12B",
)
console.log("media.check.ts ok")
+152
View File
@@ -0,0 +1,152 @@
/**
* Media references, and the bytes behind them.
*
* A frame never travels as a message the reference does. What the bytes cost
* to get is what limits the rate: fetched, it is a round trip per frame and a
* camera is a glance; pushed down the socket that already carries the value,
* it is a view. Both paths are here, and a caller does not choose between
* them: the hook takes whichever arrived.
*/
import { useEffect, useState } from "react"
import { OpenAPI } from "@/client"
import { liveStore, useLiveBytes } from "@/components/Flow/liveStore"
import { apiToken } from "@/lib/portal"
import { si } from "@/lib/utils"
/** An artifact reference, as a message carries one. */
export type MediaRef = {
digest?: string
media_type?: string
name?: string
size?: number
}
export const isRef = (value: unknown): value is MediaRef =>
typeof value === "object" &&
value !== null &&
typeof (value as MediaRef).digest === "string" &&
(value as MediaRef).digest!.startsWith("sha256:")
/**
* What a reference *is*, in the space a value would have taken: what kind of
* bytes, what they were called, how many of them.
*/
export function describeArtifact(record: Record<string, unknown>): string {
const name = typeof record.name === "string" && record.name ? record.name : ""
const size = typeof record.size === "number" ? `${si(record.size)}B` : ""
// The media type when there is one: what kind of bytes these are is the
// first thing worth knowing about a frame or a clip, and it is what the
// port's own type was declared against.
const media =
typeof record.media_type === "string" &&
record.media_type &&
record.media_type !== "application/octet-stream"
? record.media_type
: "artifact"
return [media, name, size].filter(Boolean).join(" · ")
}
/** The header a pushed frame carries in front of its bytes. */
export type MediaFrame = {
name: string
digest: string
media_type: string
ts?: number
}
/**
* Split a binary socket frame into what it says and what it carries.
*
* Four bytes of header length, the header as JSON, then the bytes. Returns
* null for anything that does not parse, since a frame the client cannot read
* is one to drop rather than one to crash on.
*/
export function parseMediaFrame(
buffer: ArrayBuffer,
): { header: MediaFrame; bytes: Blob } | null {
if (buffer.byteLength < 4) return null
const length = new DataView(buffer).getUint32(0)
if (length <= 0 || 4 + length > buffer.byteLength) return null
try {
const header = JSON.parse(
new TextDecoder().decode(new Uint8Array(buffer, 4, length)),
) as MediaFrame
if (
typeof header?.digest !== "string" ||
typeof header?.name !== "string"
) {
return null
}
return {
header,
bytes: new Blob([new Uint8Array(buffer, 4 + length)], {
type: header.media_type || "application/octet-stream",
}),
}
} catch {
return null
}
}
/**
* A local URL for an artifact's bytes, refreshed whenever the digest changes.
*
* Pushed bytes are used where the socket sent them, which is the fast path and
* costs no request at all. Otherwise they are fetched: `/artifacts/{digest}`
* takes a bearer token and no `<img>` or `<audio>` can carry a header, so the
* bytes come through fetch and are handed to the element as an object URL.
*
* Passing `name` registers interest in that message, which is what tells the
* engine to push its frames at all nothing is sent to a screen that is not
* drawing it.
*/
export function useArtifactUrl(ref: MediaRef | null, name?: string): string {
const digest = ref?.digest ?? ""
const mediaType = ref?.media_type ?? ""
const pushed = useLiveBytes(digest)
const [fetched, setFetched] = useState("")
useEffect(() => {
if (!name) return
return liveStore.wantBytes(name)
}, [name])
useEffect(() => {
// Already here: the socket carried the bytes in front of the value.
if (!digest || pushed) {
setFetched("")
return
}
let live = true
let made = ""
const token = apiToken()
const query = mediaType
? `?media_type=${encodeURIComponent(mediaType)}`
: ""
const request = new AbortController()
fetch(`${OpenAPI.BASE}/api/v1/artifacts/${digest}${query}`, {
headers: token ? { Authorization: `Bearer ${token}` } : {},
signal: request.signal,
})
.then((answer) => (answer.ok ? answer.blob() : Promise.reject(answer)))
.then((blob) => {
if (!live) return
made = URL.createObjectURL(blob)
setFetched(made)
})
.catch(() => {
if (live) setFetched("")
})
return () => {
live = false
// Abandoned rather than merely ignored: a tile showing a camera starts a
// request per frame, and the ones it no longer wants should not still be
// arriving.
request.abort()
if (made) URL.revokeObjectURL(made)
}
}, [digest, mediaType, pushed])
return pushed || fetched
}
+102 -15
View File
@@ -62,9 +62,18 @@ MAX_LOG = 16 * 1024
#: Where the artifact store is, from this worker's point of view: a directory #: Where the artifact store is, from this worker's point of view: a directory
#: when it shares the engine's filesystem, a URL when it does not. #: when it shares the engine's filesystem, a URL when it does not.
ARTIFACT_DIR_ENV = "FLUKSIO_ARTIFACT_DIR" ARTIFACT_DIR_ENV = "FLUKSIO_ARTIFACT_DIR"
#: The ring, for frames a flow only shows live. Same layout as the store above;
#: what differs is that the oldest fall out of it.
ARTIFACT_VOLATILE_DIR_ENV = "FLUKSIO_ARTIFACT_VOLATILE_DIR"
ARTIFACT_URL_ENV = "FLUKSIO_ARTIFACT_URL" ARTIFACT_URL_ENV = "FLUKSIO_ARTIFACT_URL"
ARTIFACT_TOKEN_ENV = "FLUKSIO_ARTIFACT_TOKEN" ARTIFACT_TOKEN_ENV = "FLUKSIO_ARTIFACT_TOKEN"
ARTIFACT_CACHE_ENV = "FLUKSIO_ARTIFACT_CACHE" ARTIFACT_CACHE_ENV = "FLUKSIO_ARTIFACT_CACHE"
#: How much a worker that fetches over HTTP keeps of what it downloaded. The
#: cache is keyed by digest and nothing ever invalidates an entry, so without
#: a bound a node reading a media stream fills the disk with chunks nothing
#: will ask for twice.
ARTIFACT_CACHE_BYTES_ENV = "FLUKSIO_ARTIFACT_CACHE_BYTES"
DEFAULT_CACHE_BYTES = 1024 * 1024 * 1024
ARTIFACT_TIMEOUT_S = 300 ARTIFACT_TIMEOUT_S = 300
#: How much of an artifact is held in memory at a time, matching the engine's #: How much of an artifact is held in memory at a time, matching the engine's
#: own store. Repeated rather than imported: nothing of the engine is here. #: own store. Repeated rather than imported: nothing of the engine is here.
@@ -109,16 +118,27 @@ class _Reporter(ModuleType):
source: Any, source: Any,
name: str = "", name: str = "",
media_type: str = "application/octet-stream", media_type: str = "application/octet-stream",
volatile: bool = False,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Store bytes or a file and return the reference to return onward. """Store bytes or a file and return the reference to return onward.
Return the reference from an ``artifact`` port: a checkpoint is far too Return the reference from an ``artifact`` port: a checkpoint is far too
big to be a message, and the reference is what the next node opens. big to be a message, and the reference is what the next node opens.
``volatile`` is for a frame rather than a result: the bytes go to a
ring held in memory and are pushed to whatever screen is watching,
instead of being written to the data volume. They last seconds. A
volatile reference a node *returns* from a run is kept anyway, so what
this costs is only what nobody asked to keep.
""" """
if isinstance(source, (bytes, bytearray)): if isinstance(source, (bytes, bytearray)):
return _store_bytes(bytes(source), name or "artifact.bin", media_type) return _store_bytes(
bytes(source), name or "artifact.bin", media_type, volatile
)
path = str(source) path = str(source)
return _store_file(path, name or os.path.basename(path), media_type) return _store_file(
path, name or os.path.basename(path), media_type, volatile
)
def load_artifact(self, ref: dict[str, Any]) -> str: def load_artifact(self, ref: dict[str, Any]) -> str:
"""Fetch an artifact and hand back a local path to read it from.""" """Fetch an artifact and hand back a local path to read it from."""
@@ -186,7 +206,22 @@ class _Declared:
return "<fluksio declaration>" return "<fluksio declaration>"
def _store_bytes(data: bytes, name: str, media_type: str) -> dict[str, Any]: def _artifact_dir(volatile: bool) -> str | None:
"""Which store this worker writes to, if it can see one at all.
A volatile write falls back to the durable directory rather than failing:
an engine with its ring turned off still has to take the frame.
"""
if volatile:
ring = os.environ.get(ARTIFACT_VOLATILE_DIR_ENV)
if ring:
return ring
return os.environ.get(ARTIFACT_DIR_ENV)
def _store_bytes(
data: bytes, name: str, media_type: str, volatile: bool = False
) -> dict[str, Any]:
"""Write to the artifact store, whichever end of it this worker can see. """Write to the artifact store, whichever end of it this worker can see.
A worker in the engine's own container writes the file; one on another host A worker in the engine's own container writes the file; one on another host
@@ -196,7 +231,7 @@ def _store_bytes(data: bytes, name: str, media_type: str) -> dict[str, Any]:
is importable here. is importable here.
""" """
digest = "sha256:" + hashlib.sha256(data).hexdigest() digest = "sha256:" + hashlib.sha256(data).hexdigest()
directory = os.environ.get(ARTIFACT_DIR_ENV) directory = _artifact_dir(volatile)
if directory: if directory:
target = os.path.join(directory, digest[7:9], digest[7:]) target = os.path.join(directory, digest[7:9], digest[7:])
if not os.path.exists(target): if not os.path.exists(target):
@@ -208,7 +243,7 @@ def _store_bytes(data: bytes, name: str, media_type: str) -> dict[str, Any]:
out.write(data) out.write(data)
os.replace(temporary, target) os.replace(temporary, target)
else: else:
_put_over_http(data, name, media_type) _put_over_http(data, name, media_type, volatile=volatile)
return { return {
"digest": digest, "digest": digest,
"size": len(data), "size": len(data),
@@ -217,7 +252,9 @@ def _store_bytes(data: bytes, name: str, media_type: str) -> dict[str, Any]:
} }
def _store_file(path: str, name: str, media_type: str) -> dict[str, Any]: def _store_file(
path: str, name: str, media_type: str, volatile: bool = False
) -> dict[str, Any]:
"""The same, for a file already on disk — read a chunk at a time. """The same, for a file already on disk — read a chunk at a time.
A video segment or a checkpoint is as likely to be handed over as a path as A video segment or a checkpoint is as likely to be handed over as a path as
@@ -232,7 +269,7 @@ def _store_file(path: str, name: str, media_type: str) -> dict[str, Any]:
size += len(chunk) size += len(chunk)
digest = "sha256:" + digester.hexdigest() digest = "sha256:" + digester.hexdigest()
directory = os.environ.get(ARTIFACT_DIR_ENV) directory = _artifact_dir(volatile)
if directory: if directory:
target = os.path.join(directory, digest[7:9], digest[7:]) target = os.path.join(directory, digest[7:9], digest[7:])
if not os.path.exists(target): if not os.path.exists(target):
@@ -244,12 +281,16 @@ def _store_file(path: str, name: str, media_type: str) -> dict[str, Any]:
os.replace(temporary, target) os.replace(temporary, target)
else: else:
with open(path, "rb") as handle: with open(path, "rb") as handle:
_put_over_http(handle, name, media_type, size) _put_over_http(handle, name, media_type, size, volatile=volatile)
return {"digest": digest, "size": size, "media_type": media_type, "name": name} return {"digest": digest, "size": size, "media_type": media_type, "name": name}
def _put_over_http( def _put_over_http(
data: Any, name: str, media_type: str, length: int | None = None data: Any,
name: str,
media_type: str,
length: int | None = None,
volatile: bool = False,
) -> None: ) -> None:
base = os.environ.get(ARTIFACT_URL_ENV) base = os.environ.get(ARTIFACT_URL_ENV)
if not base: if not base:
@@ -257,7 +298,10 @@ def _put_over_http(
"this worker has no artifact store — neither " "this worker has no artifact store — neither "
f"{ARTIFACT_DIR_ENV} nor {ARTIFACT_URL_ENV} is set" f"{ARTIFACT_DIR_ENV} nor {ARTIFACT_URL_ENV} is set"
) )
query = urllib.parse.urlencode({"name": name, "media_type": media_type}) fields = {"name": name, "media_type": media_type}
if volatile:
fields["volatile"] = "1"
query = urllib.parse.urlencode(fields)
request = urllib.request.Request( request = urllib.request.Request(
f"{base.rstrip('/')}?{query}", data=data, method="PUT" f"{base.rstrip('/')}?{query}", data=data, method="PUT"
) )
@@ -271,12 +315,17 @@ def _put_over_http(
def _fetch(digest: str) -> str: def _fetch(digest: str) -> str:
"""A local path holding this artifact's bytes, downloading it if needed.""" """A local path holding this artifact's bytes, downloading it if needed."""
directory = os.environ.get(ARTIFACT_DIR_ENV) for variable in (ARTIFACT_DIR_ENV, ARTIFACT_VOLATILE_DIR_ENV):
if directory: directory = os.environ.get(variable)
if not directory:
continue
target = os.path.join(directory, digest[7:9], digest[7:]) target = os.path.join(directory, digest[7:9], digest[7:])
if not os.path.exists(target): if os.path.exists(target):
raise FileNotFoundError(digest) return target
return target if os.environ.get(ARTIFACT_DIR_ENV):
# A worker sharing the engine's filesystem has nowhere else to look:
# either the bytes are in one of the two stores or they have gone.
raise FileNotFoundError(digest)
# Cached by digest: content-addressing means a file once fetched is never # Cached by digest: content-addressing means a file once fetched is never
# stale, so a sweep pulls a shared input across the network once. # stale, so a sweep pulls a shared input across the network once.
@@ -298,9 +347,47 @@ def _fetch(digest: str) -> str:
while chunk := response.read(CHUNK): while chunk := response.read(CHUNK):
out.write(chunk) out.write(chunk)
os.replace(temporary, target) os.replace(temporary, target)
_trim_cache(cache)
return target return target
def _trim_cache(cache: str) -> None:
"""Hold the fetch cache inside its bound, oldest first.
Content addressing means an entry is never stale, so nothing here ever
expires on its own and a node reading a media stream asks for a digest it
will never ask for again. Cheap: one listing per download.
"""
try:
limit = int(os.environ.get(ARTIFACT_CACHE_BYTES_ENV) or DEFAULT_CACHE_BYTES)
except ValueError:
limit = DEFAULT_CACHE_BYTES
if limit <= 0:
return
entries = []
total = 0
with os.scandir(cache) as listing:
for entry in listing:
try:
stat = entry.stat()
except OSError:
continue
if not entry.is_file():
continue
entries.append((stat.st_mtime, stat.st_size, entry.path))
total += stat.st_size
if total <= limit:
return
for _mtime, size, path in sorted(entries):
if total <= limit:
break
try:
os.unlink(path)
except OSError:
continue
total -= size
def _authorize(request: Any) -> None: def _authorize(request: Any) -> None:
token = os.environ.get(ARTIFACT_TOKEN_ENV) token = os.environ.get(ARTIFACT_TOKEN_ENV)
if token: if token: