Media dtypes: image, audio and video as narrowed artifact references

A port may now declare `image`, `audio` or `video`. Each is the artifact
reference the engine already had, narrowed by the `media_type` on it, so a
speech recogniser declares what it eats rather than taking any bytes at all and
finding out. Bytes still never travel as a message and nothing on the wire
stops being JSON: a camera publishes one reference per frame, a microphone one
per chunk, and a reference may carry a `meta` dict nothing here interprets.

Streaming media is therefore an ordinary streaming port — with one change to
what that means. An emission used to journal an item with no payload, so
downstream read whatever was current when the item was claimed; a consumer
slower than its producer saw only the newest chunk and the ones between were
lost. That is right for a training curve and wrong for a second of speech, so
an emission now journals a `kind="emission"` item carrying its values, and the
executor hands them to the nodes reading that message instead of writing them
to state again. The value in state stays the latest, which is what everything
else reads, and the wave is filtered by what actually changed rather than
walking everything reachable. No queue serialization change — the existing
`outputs` field carries it.

Continuous media makes the store's missing GC a real problem, so this closes
it: `sweep_artifacts` runs hourly, keeps every digest a `run_artifact` row
records or a live message holds, spares anything written in the last hour, and
stands aside entirely while a run is in flight, since a node may store a
checkpoint long before it returns the reference to it. That also collects the
orphans a deleted flow has always left behind. `ARTIFACT_GC_INTERVAL_S=0` turns
it off.

Around the edges: `GET /artifacts/{digest}` serves the media type the caller
passes and answers ranged requests, so a browser plays a clip rather than
downloading it; `PUT` spools to disk instead of holding the whole body in
memory, as does `save_artifact` given a path; a Media widget draws whatever its
message points at, and a wall panel may fetch the bytes its own tiles are
showing and nothing else; and a connector gets `save_artifact`, for a device
whose readings are bytes.

What this cannot do is live video: a frame every second or two is a glance, and
the honest answer above that is the camera's own stream, which the widget takes
as a URL and the browser plays from source.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/tests/api/routes/test_panels.py

@@ -764,3 +764,45 @@ def test_a_panels_socket_carries_only_what_it_draws(
 
     # A person's credential is not bounded at all.
     assert panel_scope(superuser_token_headers["Authorization"][7:], client.app) is None
+
+
+def test_a_panel_fetches_only_the_media_its_tiles_are_showing(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    """A media tile needs the bytes, and only the ones its message points at."""
+    _dashboard_with(
+        client,
+        superuser_token_headers,
+        "panel_camera",
+        [
+            {
+                "id": "w_media",
+                "type": "media",
+                "config": {"message": "demo.frame", "dtype": "image"},
+            }
+        ],
+    )
+    _panels(
+        client,
+        superuser_token_headers,
+        {"panels": [{"id": "porch", "dashboards": ["panel_camera"]}]},
+    )
+    panel_headers = _pair(client, superuser_token_headers, "porch")
+
+    store = client.app.state.artifact_store
+    shown = store.put([b"the frame on the wall"], media_type="image/png")
+    elsewhere = store.put([b"an artifact of some other run"])
+    client.app.state.flow_controller.state.update({"demo.frame": shown})
+
+    artifacts = f"{settings.API_V1_STR}/artifacts"
+    assert (
+        client.get(f"{artifacts}/{shown['digest']}", headers=panel_headers).status_code
+        == 200
+    )
+    # Knowing a digest is not being entitled to it: a screen reads what it draws.
+    assert (
+        client.get(
+            f"{artifacts}/{elsewhere['digest']}", headers=panel_headers
+        ).status_code
+        == 403
+    )

backend/tests/flow/test_connector.py

@@ -1,8 +1,12 @@
 """The connector contract: polling, deduplication, health and discovery."""
 
 import asyncio
+from types import SimpleNamespace
 from typing import Any
 
+import pytest
+
+from fluksio.flow.artifacts import ArtifactStore
 from fluksio.flow.connector import CONTRACT_VERSION, ConnectorNode
 from fluksio.flow.controller import NODE_TYPES
 from fluksio.flow.messages import DType, MessageSpec
@@ -44,11 +48,11 @@ def a_sensor(readings: list[Any], **params: Any) -> Sensor:
     return node
 
 
-def run_briefly(node: ConnectorNode, seconds: float = 0.12) -> None:
+def run_briefly(node: ConnectorNode, seconds: float = 0.12, app: Any = None) -> None:
     """Start the poll loop, let it tick a few times, stop it."""
 
     async def cycle() -> None:
-        await node.start()
+        await node.start(app)
         await asyncio.sleep(seconds)
         await node.stop()
 
@@ -188,3 +192,35 @@ def test_a_connector_may_not_take_over_a_built_in_type(monkeypatch):
     )
     assert load_plugins() == []
     assert NODE_TYPES["mqtt"].plugin is None
+
+
+def test_a_connector_publishes_bytes_as_a_media_reference(tmp_path):
+    """A camera's reading is bytes, and bytes never travel as a message."""
+
+    class Camera(ConnectorNode):
+        contract = CONTRACT_VERSION
+
+        async def poll(self) -> dict[str, Any] | None:
+            return {"frame": self.save_artifact(b"\x89PNG...", "f.png", "image/png")}
+
+    node = Camera(
+        provides=[MessageSpec(name="frame", dtype=DType.IMAGE)],
+        params={"poll_interval": 0.01},
+    )
+    node.assign_flow("demo", "camera")
+    pipeline = Pipeline(nodes=[node])
+
+    # Before it starts there is no store to write to, and saying so beats an
+    # AttributeError from inside somebody's connector.
+    with pytest.raises(RuntimeError, match="artifact store"):
+        node.save_artifact(b"x")
+
+    store = ArtifactStore(tmp_path / "artifacts")
+    app = SimpleNamespace(state=SimpleNamespace(artifact_store=store))
+    run_briefly(node, app=app)
+
+    reference = pipeline.state["demo.frame"]
+    # It typechecks against the port it was published on, which is the whole
+    # point of a media dtype.
+    MessageSpec(name="demo.frame", dtype=DType.IMAGE).check(reference)
+    assert store.path(reference["digest"]).read_bytes() == b"\x89PNG..."

backend/tests/flow/test_dashboards.py

@@ -241,6 +241,17 @@ def test_the_structured_widgets_bind_their_shapes():
         WidgetDef(id="f", type="forecast", config={"message": "a.b", "dtype": "json"})
 
 
+def test_a_media_widget_binds_media_and_nothing_else():
+    for dtype in ("image", "audio", "video", "artifact"):
+        WidgetDef(id="m", type="media", config={"message": "cam.frame", "dtype": dtype})
+
+    for dtype in ("float", "record"):
+        with pytest.raises(ValueError):
+            WidgetDef(
+                id="m", type="media", config={"message": "cam.frame", "dtype": dtype}
+            )
+
+
 def test_a_bar_nests_a_second_number():
     WidgetDef(
         id="b",

backend/tests/flow/test_messages.py

@@ -91,6 +91,55 @@ def test_a_list_holds_one_declared_level():
             MessageSpec(name="x", dtype=DType.LIST, item=item)
 
 
+def _ref(media_type: str) -> dict:
+    return {
+        "digest": "sha256:" + "ab" * 32,
+        "size": 12,
+        "media_type": media_type,
+        "name": "chunk",
+    }
+
+
+def test_a_media_port_checks_the_family():
+    spec = MessageSpec(name="speech", dtype=DType.AUDIO)
+    spec.check(_ref("audio/wav"))
+    for wrong in (_ref("video/mp4"), _ref(""), {"a": 1}, "sha256:x"):
+        with pytest.raises(TypeError):
+            spec.check(wrong)
+
+
+def test_a_media_mismatch_names_the_media_type():
+    spec = MessageSpec(name="speech", dtype=DType.AUDIO)
+    with pytest.raises(TypeError, match="video/mp4"):
+        spec.check(_ref("video/mp4"))
+
+
+def test_an_artifact_port_accepts_a_media_reference():
+    # Media narrows artifact, so the wider port still takes it; the reverse is
+    # what the family check refuses.
+    MessageSpec(name="blob", dtype=DType.ARTIFACT).check(_ref("image/png"))
+    with pytest.raises(TypeError):
+        MessageSpec(name="frame", dtype=DType.IMAGE).check(
+            {"digest": "sha256:" + "cd" * 32, "size": 1}
+        )
+
+
+def test_a_media_reference_may_carry_meta():
+    spec = MessageSpec(name="frame", dtype=DType.IMAGE)
+    spec.check({**_ref("image/png"), "meta": {"width": 640, "seq": 3}})
+
+
+def test_a_list_refuses_media_items():
+    for item in (DType.IMAGE, DType.AUDIO, DType.VIDEO):
+        with pytest.raises(ValueError):
+            MessageSpec(name="x", dtype=DType.LIST, item=item)
+
+
+def test_coerce_parses_a_media_reference_from_text():
+    spec = MessageSpec(name="frame", dtype=DType.IMAGE)
+    assert spec.coerce(json.dumps(_ref("image/png"))) == _ref("image/png")
+
+
 def test_coerce_parses_structured_text():
     spec = MessageSpec(name="notice", dtype=DType.RECORD)
     assert spec.coerce('{"title": "Boiler"}') == {"title": "Boiler"}

backend/tests/flow/test_queue.py

@@ -431,3 +431,94 @@ def test_no_more_is_claimed_than_the_pool_can_run():
     finally:
         release.set()
         service.stop()
+
+
+# -----------------------------------------------------------------------------
+# Emissions: values a node publishes while it is still running
+# -----------------------------------------------------------------------------
+
+
+def _emitting_pipeline() -> tuple[Pipeline, Node, MemoryState, list]:
+    """A source with a streaming port, and a consumer that records every value."""
+    seen: list[float] = []
+
+    def consume(chunk, params):
+        seen.append(chunk)
+        return None
+
+    source = Node(
+        f=lambda params: None,
+        provides=[
+            MessageSpec(name="chunk", port="chunk", dtype=DType.FLOAT, stream=True)
+        ],
+        name="source",
+    )
+    consumer = Node(
+        f=consume,
+        requires=[MessageSpec(name="chunk", port="chunk", dtype=DType.FLOAT)],
+        name="consumer",
+    )
+    source.assign_flow("f", "source")
+    consumer.assign_flow("f", "consumer")
+
+    state = MemoryState()
+    queue = MemoryWorkQueue()
+    pipeline = Pipeline(nodes=[source, consumer], state=state, work_queue=queue)
+    return pipeline, source, state, seen
+
+
+def test_every_emitted_chunk_reaches_the_consumer():
+    """A consumer slower than its producer must not skip what it missed.
+
+    Reading state instead would give whichever chunk is newest by the time the
+    item runs — fine for a temperature, lossy for a second of speech.
+    """
+    pipeline, source, state, seen = _emitting_pipeline()
+    service = ExecutionService(pipeline._queue)
+    service.bind(pipeline)
+
+    # Both emitted before either item is claimed, so state has moved on.
+    pipeline.publish_emission(source, {"f.chunk": 1.0})
+    pipeline.publish_emission(source, {"f.chunk": 2.0})
+    assert state["f.chunk"] == 2.0
+
+    for item in pipeline._queue.claim(10, 10):
+        service._run_item(item)
+
+    assert seen == [1.0, 2.0]
+
+
+def test_a_delivered_emission_does_not_write_state_a_second_time():
+    """The value in state stays the newest one, whenever an item is claimed."""
+    pipeline, source, state, seen = _emitting_pipeline()
+    service = ExecutionService(pipeline._queue)
+    service.bind(pipeline)
+
+    pipeline.publish_emission(source, {"f.chunk": 1.0})
+    # What the node returned at the end, after the emission it made on the way.
+    pipeline.apply_outputs(source, {"f.chunk": 9.0})
+
+    for item in pipeline._queue.claim(10, 10):
+        service._run_item(item)
+
+    assert seen[0] == 1.0
+    # Re-applying the carried chunk here is what would undo the final value.
+    assert state["f.chunk"] == 9.0
+
+
+def test_a_throttled_emission_wakes_nothing():
+    pipeline, source, _state, seen = _emitting_pipeline()
+    source.provides["f.chunk"] = MessageSpec(
+        name="f.chunk", port="chunk", dtype=DType.FLOAT, stream=True, interval=60
+    )
+    service = ExecutionService(pipeline._queue)
+    service.bind(pipeline)
+
+    pipeline.publish_emission(source, {"f.chunk": 1.0})
+    pipeline.publish_emission(source, {"f.chunk": 2.0})
+    for item in pipeline._queue.claim(10, 10):
+        service._run_item(item)
+
+    # The first is let through; the second is held by the interval, and a value
+    # nothing published is nothing to wake on.
+    assert seen == [1.0]

backend/tests/flow/test_workers.py

@@ -455,6 +455,78 @@ def test_a_node_saves_and_loads_an_artifact(tmp_path):
         pool.stop()
 
 
+def test_a_node_passes_audio_to_the_next_one(tmp_path):
+    # The media path end to end: one node writes a clip and declares what kind
+    # of bytes it is, the next opens it. Only the reference travels.
+    store = ArtifactStore(tmp_path / "artifacts")
+    pool = PythonWorkerPool(
+        python=sys.executable, size=1, env={ARTIFACT_DIR_ENV: str(store.root)}
+    )
+    pool.start()
+    try:
+        ref = pool.run(
+            "demo",
+            "speak",
+            "import fluksio, io, math, struct, wave\n"
+            "def process():\n"
+            "    buffer = io.BytesIO()\n"
+            "    with wave.open(buffer, 'wb') as out:\n"
+            "        out.setnchannels(1)\n"
+            "        out.setsampwidth(2)\n"
+            "        out.setframerate(8000)\n"
+            "        out.writeframes(b''.join(\n"
+            "            struct.pack('<h', int(16000 * math.sin(i / 8)))\n"
+            "            for i in range(1600)\n"
+            "        ))\n"
+            "    return {'speech': fluksio.save_artifact(\n"
+            "        buffer.getvalue(), 'beep.wav', media_type='audio/wav')}\n",
+            {},
+            "demo.speak",
+            timeout=10,
+        )["speech"]
+
+        assert ref["media_type"] == "audio/wav"
+        assert MessageSpec(name="speech", dtype=DType.AUDIO).check(ref) is None
+        # An audio port takes it; a video one does not.
+        with pytest.raises(TypeError):
+            MessageSpec(name="speech", dtype=DType.VIDEO).check(ref)
+
+        heard = pool.run(
+            "demo",
+            "listen",
+            "import fluksio, wave\n"
+            "def process(speech):\n"
+            "    with wave.open(fluksio.load_artifact(speech), 'rb') as clip:\n"
+            "        return {'seconds': clip.getnframes() / clip.getframerate()}\n",
+            {"speech": ref},
+            "demo.listen",
+            timeout=10,
+        )
+        assert heard == {"seconds": 0.2}
+
+        # A file is stored by streaming it rather than reading it in: a video
+        # segment is as likely to be handed over as a path as as bytes.
+        source = tmp_path / "segment.mp4"
+        source.write_bytes(b"m" * (2 * 1024 * 1024 + 7))
+        from_path = pool.run(
+            "demo",
+            "record",
+            "import fluksio\n"
+            "def process(path):\n"
+            "    return {'clip': fluksio.save_artifact(\n"
+            "        path, media_type='video/mp4')}\n",
+            {"path": str(source)},
+            "demo.record",
+            timeout=10,
+        )["clip"]
+        assert from_path["size"] == source.stat().st_size
+        assert from_path["name"] == "segment.mp4"
+        assert MessageSpec(name="clip", dtype=DType.VIDEO).check(from_path) is None
+        assert store.path(from_path["digest"]).read_bytes() == source.read_bytes()
+    finally:
+        pool.stop()
+
+
 # -----------------------------------------------------------------------------
 # No timeout at all
 #

backend/tests/test_artifacts.py

@@ -0,0 +1,110 @@
+import os
+import time
+from datetime import UTC, datetime
+
+import pytest
+from sqlmodel import Session, col, select
+
+from fluksio.core.db import engine as db_engine
+from fluksio.flow.artifacts import ArtifactStore
+from fluksio.flow.runs import new_run_id, sweep_artifacts
+from fluksio.flow.state import MemoryState
+from fluksio.models import Run, RunArtifact
+
+
+@pytest.fixture
+def store(tmp_path) -> ArtifactStore:
+    return ArtifactStore(tmp_path / "artifacts")
+
+
+def _run_row(status: str, digest: str = "") -> str:
+    """A run, and optionally the artifact it recorded. Returns its id."""
+    run_id = new_run_id()
+    with Session(db_engine) as session:
+        session.add(
+            Run(id=run_id, flow="f", status=status, created_at=datetime.now(UTC))
+        )
+        if digest:
+            session.add(
+                RunArtifact(
+                    run_id=run_id,
+                    name="f.out",
+                    node="n",
+                    digest=digest,
+                    size=3,
+                )
+            )
+        session.commit()
+    return run_id
+
+
+def _forget(run_id: str) -> None:
+    with Session(db_engine) as session:
+        rows = session.exec(
+            select(RunArtifact).where(col(RunArtifact.run_id) == run_id)
+        ).all()
+        for row in rows:
+            session.delete(row)
+        session.delete(session.get(Run, run_id))
+        session.commit()
+
+
+def test_the_grace_window_spares_a_fresh_artifact(store):
+    ref = store.put([b"new"])
+    assert store.collect(set(), grace_s=3600) == 0
+    assert store.path(ref["digest"]) is not None
+    assert store.collect(set(), grace_s=0) == 1
+    assert store.path(ref["digest"]) is None
+
+
+def test_a_sweep_keeps_what_a_run_recorded_and_what_a_message_holds(store):
+    recorded = store.put([b"kept by a run"])
+    held = store.put([b"kept by a message"])
+    nested = store.put([b"kept inside a payload"])
+    orphan = store.put([b"referred to by nothing"])
+
+    run_id = _run_row("ok", recorded["digest"])
+    state = MemoryState()
+    state.update(
+        {
+            "cam.frame": held,
+            # A reference inside a json payload counts as much as a bare one.
+            "cam.report": {"latest": {"clip": nested}},
+            "cam.count": 3,
+        }
+    )
+
+    try:
+        assert sweep_artifacts(store, state) == 1
+    finally:
+        _forget(run_id)
+
+    assert store.path(orphan["digest"]) is None
+    for kept in (recorded, held, nested):
+        assert store.path(kept["digest"]) is not None
+
+
+def test_a_sweep_stands_aside_while_a_run_is_in_flight(store):
+    orphan = store.put([b"mid-run checkpoint"])
+    run_id = _run_row("running")
+    try:
+        assert sweep_artifacts(store, MemoryState()) == 0
+    finally:
+        _forget(run_id)
+    assert store.path(orphan["digest"]) is not None
+
+
+def test_a_streamed_chunk_falls_out_once_the_message_moves_on(store):
+    """What makes a media stream affordable: only the current frame is held."""
+    state = MemoryState()
+    first = store.put([b"frame one"], media_type="image/png")
+    state.update({"cam.frame": first})
+    assert sweep_artifacts(store, state) == 0
+
+    second = store.put([b"frame two"], media_type="image/png")
+    state.update({"cam.frame": second})
+    # Old enough to be swept: the sweep only spares what grace covers.
+    os.utime(store.path(first["digest"]), (time.time() - 10, time.time() - 10))
+    assert sweep_artifacts(store, state, grace_s=5) == 1
+    assert store.path(first["digest"]) is None
+    assert store.path(second["digest"]) is not None