Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Push a frame instead of storing and fetching it

Browse Source 
The rate the media dtypes could carry was one frame every second or two: each
was a file on the data volume, an event on the socket, and a request back for
the bytes. This closes both halves of that, and they are one feature.

`save_artifact(..., volatile=True)` writes to a `VolatileStore` — the same
content-addressed store, in `/dev/shm`, bounded by size with the oldest falling
out (`ARTIFACT_VOLATILE_BYTES`, 48 MB under the container's raised `shm_size`).
Nothing sweeps it: a frame nobody kept is not worth walking the store to find.
`ArtifactStore.path` falls through to it, which is what lets a volatile frame be
an ordinary reference everywhere else — the dtype check, a panel's digest scope,
`load_artifact` in a node, and the widget's own fetch all work on one unchanged.
`adopt` copies one into the store when a run records it, so "returned media is
kept, emitted media is not" stays true.

The bytes then go down the flows websocket as a length-prefixed binary frame,
sent just ahead of the `message_value` naming them, so a tile has the frame when
it hears the value moved. Nothing is pushed unasked: a client names the messages
it is drawing (`{"type":"media","names":[…]}`), a panel's list is intersected
with the scope it already had, and only the newest frame per name in a batch is
sent — a client that fell behind is not handed frames it would draw over. The
tunnel relays text only, so a screen reached through a portal falls back to
fetching, which is why the rate table now has two rows.

Around the edges: the remote worker's fetch cache is bounded at last
(`FLUKSIO_ARTIFACT_CACHE_BYTES`), since content addressing means nothing in it
ever expires and a media stream fills it with chunks nothing asks for twice; a
port carrying an image draws the frame in the node panel rather than only
saying `image/png · frame.png · 1.79kB`; and an edge chip says that much instead
of a line of hash. The media screenshot stops waiting for `networkidle` — a
camera is a socket that never goes quiet, which is the point of it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YC4u66vjzW54fnHu5Juhh9
This commit is contained in:
stroblme
2026-09-02 10:15:14 +02:00
co-authored by Claude Opus 5
parent 518231aa39
commit d471614e6a
29 changed files with 1101 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
worker/fluksio_worker/worker_main.py
+102 -15
View File
@@ -62,9 +62,18 @@ MAX_LOG = 16 * 1024
#: Where the artifact store is, from this worker's point of view: a directory
#: when it shares the engine's filesystem, a URL when it does not.
ARTIFACT_DIR_ENV = "FLUKSIO_ARTIFACT_DIR"
#: The ring, for frames a flow only shows live. Same layout as the store above;
#: what differs is that the oldest fall out of it.
ARTIFACT_VOLATILE_DIR_ENV = "FLUKSIO_ARTIFACT_VOLATILE_DIR"
ARTIFACT_URL_ENV = "FLUKSIO_ARTIFACT_URL"
ARTIFACT_TOKEN_ENV = "FLUKSIO_ARTIFACT_TOKEN"
ARTIFACT_CACHE_ENV = "FLUKSIO_ARTIFACT_CACHE"
#: How much a worker that fetches over HTTP keeps of what it downloaded. The
#: cache is keyed by digest and nothing ever invalidates an entry, so without
#: a bound a node reading a media stream fills the disk with chunks nothing
#: will ask for twice.
ARTIFACT_CACHE_BYTES_ENV = "FLUKSIO_ARTIFACT_CACHE_BYTES"
DEFAULT_CACHE_BYTES = 1024 * 1024 * 1024
ARTIFACT_TIMEOUT_S = 300
#: How much of an artifact is held in memory at a time, matching the engine's
#: own store. Repeated rather than imported: nothing of the engine is here.
@@ -109,16 +118,27 @@ class _Reporter(ModuleType):
source: Any,
name: str = "",
media_type: str = "application/octet-stream",
volatile: bool = False,
) -> dict[str, Any]:
"""Store bytes or a file and return the reference to return onward.
Return the reference from an ``artifact`` port: a checkpoint is far too
big to be a message, and the reference is what the next node opens.
``volatile`` is for a frame rather than a result: the bytes go to a
ring held in memory and are pushed to whatever screen is watching,
instead of being written to the data volume. They last seconds. A
volatile reference a node *returns* from a run is kept anyway, so what
this costs is only what nobody asked to keep.
"""
if isinstance(source, (bytes, bytearray)):
return _store_bytes(bytes(source), name or "artifact.bin", media_type)
return _store_bytes(
bytes(source), name or "artifact.bin", media_type, volatile
)
path = str(source)
return _store_file(path, name or os.path.basename(path), media_type)
return _store_file(
path, name or os.path.basename(path), media_type, volatile
)
def load_artifact(self, ref: dict[str, Any]) -> str:
"""Fetch an artifact and hand back a local path to read it from."""
@@ -186,7 +206,22 @@ class _Declared:
return "<fluksio declaration>"
def _store_bytes(data: bytes, name: str, media_type: str) -> dict[str, Any]:
def _artifact_dir(volatile: bool) -> str | None:
"""Which store this worker writes to, if it can see one at all.
A volatile write falls back to the durable directory rather than failing:
an engine with its ring turned off still has to take the frame.
"""
if volatile:
ring = os.environ.get(ARTIFACT_VOLATILE_DIR_ENV)
if ring:
return ring
return os.environ.get(ARTIFACT_DIR_ENV)
def _store_bytes(
data: bytes, name: str, media_type: str, volatile: bool = False
) -> dict[str, Any]:
"""Write to the artifact store, whichever end of it this worker can see.
A worker in the engine's own container writes the file; one on another host
@@ -196,7 +231,7 @@ def _store_bytes(data: bytes, name: str, media_type: str) -> dict[str, Any]:
is importable here.
"""
digest = "sha256:" + hashlib.sha256(data).hexdigest()
directory = os.environ.get(ARTIFACT_DIR_ENV)
directory = _artifact_dir(volatile)
if directory:
target = os.path.join(directory, digest[7:9], digest[7:])
if not os.path.exists(target):
@@ -208,7 +243,7 @@ def _store_bytes(data: bytes, name: str, media_type: str) -> dict[str, Any]:
out.write(data)
os.replace(temporary, target)
else:
_put_over_http(data, name, media_type)
_put_over_http(data, name, media_type, volatile=volatile)
return {
"digest": digest,
"size": len(data),
@@ -217,7 +252,9 @@ def _store_bytes(data: bytes, name: str, media_type: str) -> dict[str, Any]:
}
def _store_file(path: str, name: str, media_type: str) -> dict[str, Any]:
def _store_file(
path: str, name: str, media_type: str, volatile: bool = False
) -> dict[str, Any]:
"""The same, for a file already on disk — read a chunk at a time.
A video segment or a checkpoint is as likely to be handed over as a path as
@@ -232,7 +269,7 @@ def _store_file(path: str, name: str, media_type: str) -> dict[str, Any]:
size += len(chunk)
digest = "sha256:" + digester.hexdigest()
directory = os.environ.get(ARTIFACT_DIR_ENV)
directory = _artifact_dir(volatile)
if directory:
target = os.path.join(directory, digest[7:9], digest[7:])
if not os.path.exists(target):
@@ -244,12 +281,16 @@ def _store_file(path: str, name: str, media_type: str) -> dict[str, Any]:
os.replace(temporary, target)
else:
with open(path, "rb") as handle:
_put_over_http(handle, name, media_type, size)
_put_over_http(handle, name, media_type, size, volatile=volatile)
return {"digest": digest, "size": size, "media_type": media_type, "name": name}
def _put_over_http(
data: Any, name: str, media_type: str, length: int | None = None
data: Any,
name: str,
media_type: str,
length: int | None = None,
volatile: bool = False,
) -> None:
base = os.environ.get(ARTIFACT_URL_ENV)
if not base:
@@ -257,7 +298,10 @@ def _put_over_http(
"this worker has no artifact store — neither "
f"{ARTIFACT_DIR_ENV} nor {ARTIFACT_URL_ENV} is set"
)
query = urllib.parse.urlencode({"name": name, "media_type": media_type})
fields = {"name": name, "media_type": media_type}
if volatile:
fields["volatile"] = "1"
query = urllib.parse.urlencode(fields)
request = urllib.request.Request(
f"{base.rstrip('/')}?{query}", data=data, method="PUT"
)
@@ -271,12 +315,17 @@ def _put_over_http(
def _fetch(digest: str) -> str:
"""A local path holding this artifact's bytes, downloading it if needed."""
directory = os.environ.get(ARTIFACT_DIR_ENV)
if directory:
for variable in (ARTIFACT_DIR_ENV, ARTIFACT_VOLATILE_DIR_ENV):
directory = os.environ.get(variable)
if not directory:
continue
target = os.path.join(directory, digest[7:9], digest[7:])
if not os.path.exists(target):
raise FileNotFoundError(digest)
return target
if os.path.exists(target):
return target
if os.environ.get(ARTIFACT_DIR_ENV):
# A worker sharing the engine's filesystem has nowhere else to look:
# either the bytes are in one of the two stores or they have gone.
raise FileNotFoundError(digest)
# Cached by digest: content-addressing means a file once fetched is never
# stale, so a sweep pulls a shared input across the network once.
@@ -298,9 +347,47 @@ def _fetch(digest: str) -> str:
while chunk := response.read(CHUNK):
out.write(chunk)
os.replace(temporary, target)
_trim_cache(cache)
return target
def _trim_cache(cache: str) -> None:
"""Hold the fetch cache inside its bound, oldest first.
Content addressing means an entry is never stale, so nothing here ever
expires on its own — and a node reading a media stream asks for a digest it
will never ask for again. Cheap: one listing per download.
"""
try:
limit = int(os.environ.get(ARTIFACT_CACHE_BYTES_ENV) or DEFAULT_CACHE_BYTES)
except ValueError:
limit = DEFAULT_CACHE_BYTES
if limit <= 0:
return
entries = []
total = 0
with os.scandir(cache) as listing:
for entry in listing:
try:
stat = entry.stat()
except OSError:
continue
if not entry.is_file():
continue
entries.append((stat.st_mtime, stat.st_size, entry.path))
total += stat.st_size
if total <= limit:
return
for _mtime, size, path in sorted(entries):
if total <= limit:
break
try:
os.unlink(path)
except OSError:
continue
total -= size
def _authorize(request: Any) -> None:
token = os.environ.get(ARTIFACT_TOKEN_ENV)
if token: