Push a frame instead of storing and fetching it
The rate the media dtypes could carry was one frame every second or two: each
was a file on the data volume, an event on the socket, and a request back for
the bytes. This closes both halves of that, and they are one feature.
`save_artifact(..., volatile=True)` writes to a `VolatileStore` — the same
content-addressed store, in `/dev/shm`, bounded by size with the oldest falling
out (`ARTIFACT_VOLATILE_BYTES`, 48 MB under the container's raised `shm_size`).
Nothing sweeps it: a frame nobody kept is not worth walking the store to find.
`ArtifactStore.path` falls through to it, which is what lets a volatile frame be
an ordinary reference everywhere else — the dtype check, a panel's digest scope,
`load_artifact` in a node, and the widget's own fetch all work on one unchanged.
`adopt` copies one into the store when a run records it, so "returned media is
kept, emitted media is not" stays true.
The bytes then go down the flows websocket as a length-prefixed binary frame,
sent just ahead of the `message_value` naming them, so a tile has the frame when
it hears the value moved. Nothing is pushed unasked: a client names the messages
it is drawing (`{"type":"media","names":[…]}`), a panel's list is intersected
with the scope it already had, and only the newest frame per name in a batch is
sent — a client that fell behind is not handed frames it would draw over. The
tunnel relays text only, so a screen reached through a portal falls back to
fetching, which is why the rate table now has two rows.
Around the edges: the remote worker's fetch cache is bounded at last
(`FLUKSIO_ARTIFACT_CACHE_BYTES`), since content addressing means nothing in it
ever expires and a media stream fills it with chunks nothing asks for twice; a
port carrying an image draws the frame in the node panel rather than only
saying `image/png · frame.png · 1.79kB`; and an edge chip says that much instead
of a line of hash. The media screenshot stops waiting for `networkidle` — a
camera is a socket that never goes quiet, which is the point of it.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YC4u66vjzW54fnHu5Juhh9
This commit is contained in:
@@ -3,10 +3,13 @@ import time
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import Session, col, select
|
||||
|
||||
from fluksio.core.config import settings
|
||||
|
||||
from fluksio.core.db import engine as db_engine
|
||||
from fluksio.flow.artifacts import ArtifactStore
|
||||
from fluksio.flow.artifacts import ArtifactStore, VolatileStore
|
||||
from fluksio.flow.runs import new_run_id, sweep_artifacts
|
||||
from fluksio.flow.state import MemoryState
|
||||
from fluksio.models import Run, RunArtifact
|
||||
@@ -108,3 +111,73 @@ def test_a_streamed_chunk_falls_out_once_the_message_moves_on(store):
|
||||
assert sweep_artifacts(store, state, grace_s=5) == 1
|
||||
assert store.path(first["digest"]) is None
|
||||
assert store.path(second["digest"]) is not None
|
||||
|
||||
|
||||
def test_a_ring_drops_the_oldest_once_it_is_full(tmp_path):
|
||||
"""What makes a camera affordable: the room is fixed, not the history."""
|
||||
ring = VolatileStore(tmp_path / "ring", limit_bytes=3000)
|
||||
refs = [ring.put([bytes([i]) * 1000], name=f"{i}.bin") for i in range(5)]
|
||||
|
||||
held = [ref for ref in refs if ring.path(ref["digest"]) is not None]
|
||||
assert len(held) == 3
|
||||
# The newest three, in order: eviction is by age, not by chance.
|
||||
assert held == refs[2:]
|
||||
|
||||
|
||||
def test_a_volatile_reference_resolves_like_any_other(tmp_path):
|
||||
"""A frame is an ordinary reference, which is what keeps the rest honest.
|
||||
|
||||
Everything that resolves a digest — serving one, checking a run's input,
|
||||
a panel's scope — goes through ``path``, so the ring has to answer there.
|
||||
"""
|
||||
store = ArtifactStore(tmp_path / "artifacts")
|
||||
store.volatile = VolatileStore(tmp_path / "ring", limit_bytes=1_000_000)
|
||||
|
||||
ref = store.put([b"frame"], name="f.png", media_type="image/png", volatile=True)
|
||||
|
||||
assert store.path(ref["digest"]) is not None
|
||||
# In the ring, and not on the volume the sweep is about.
|
||||
assert not (store.root / ref["digest"][7:9]).exists()
|
||||
|
||||
|
||||
def test_a_recorded_frame_is_copied_out_of_the_ring(tmp_path):
|
||||
"""Emitted media is not kept; returned media is."""
|
||||
store = ArtifactStore(tmp_path / "artifacts")
|
||||
store.volatile = VolatileStore(tmp_path / "ring", limit_bytes=1_000_000)
|
||||
ref = store.put([b"kept"], name="f.png", volatile=True)
|
||||
|
||||
assert store.adopt(ref["digest"]) is True
|
||||
|
||||
store.volatile.collect(set())
|
||||
assert store.path(ref["digest"]) is not None
|
||||
assert store.adopt("sha256:" + "0" * 64) is False
|
||||
|
||||
|
||||
def test_an_upload_may_ask_for_the_ring(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
"""A camera on another host publishes over HTTP and wants the ring too."""
|
||||
store = client.app.state.artifact_store
|
||||
previous = store.volatile
|
||||
store.volatile = VolatileStore(
|
||||
store.root.parent / "test-ring", limit_bytes=1_000_000
|
||||
)
|
||||
try:
|
||||
url = f"{settings.API_V1_STR}/artifacts"
|
||||
answer = client.put(
|
||||
f"{url}?name=f.png&media_type=image%2Fpng&volatile=1",
|
||||
headers=superuser_token_headers,
|
||||
content=b"\x89PNG-frame",
|
||||
)
|
||||
assert answer.status_code == 200, answer.text
|
||||
digest = answer.json()["digest"]
|
||||
|
||||
assert store.volatile.path(digest) is not None
|
||||
assert not (store.root / digest[7:9]).exists()
|
||||
# And it is served back like anything else, which is what the widget
|
||||
# falls back on when the socket did not push it.
|
||||
served = client.get(f"{url}/{digest}", headers=superuser_token_headers)
|
||||
assert served.status_code == 200
|
||||
assert served.content == b"\x89PNG-frame"
|
||||
finally:
|
||||
store.volatile = previous
|
||||
|
||||
Reference in New Issue
Block a user