Stage caching for batch runs, and an engine that lives in the command

A code node in a batch run is now fingerprinted by its source, its raw
settings and the values it reads — an artifact input counting as its digest,
which is what the content addressing was always for. A run that finds the key
restores what the earlier one returned and skips the node, recorded as
`cached`. The run history is the cache: `run_node.outputs` beside the
`cache_key` the schema already had, no second store. On for code nodes, never
for the built-in and connector types that have side effects; off per node with
`@node(cache=False)` and per run with `--no-cache`.

Emissions are not replayed on a hit, so a cached training node returns its
result without redrawing its curve. Recorded in NOTEPAD.md with the two other
deliberate limits.

`fluksio run --local` boots the real app in the command's own process and
drives it through its ASGI interface behind the ordinary client, so a run no
longer needs a `serve` terminal beside it — same data directory, same history,
and the cache carries between the two. It always waits, because the engine it
starts lives exactly as long as the command.

Also: `fluksio sweep --param lr=0.1,0.01` for the product of the lists,
`run --follow` for a run's numbers as they arrive, Ctrl-C cancelling a waited
run rather than abandoning it, coloured statuses on a terminal, and `name`
made optional on the metrics endpoint so a follower can ask for every series.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/tests/api/routes/test_runs.py

@@ -0,0 +1,77 @@
+"""The stage cache, from the side that needs a database.
+
+The pipeline half — what a hit restores and what a key is made of — is in
+`tests/flow/test_runs.py`, which runs without one.
+"""
+
+import json
+
+from sqlmodel import Session
+
+from fluksio.core.db import engine as db_engine
+from fluksio.flow.artifacts import ArtifactStore
+from fluksio.flow.pipeline import NodeOutcome
+from fluksio.flow.runs import OUTPUT_CAP, RunCache, _cacheable
+from fluksio.models import RunNode
+
+
+def test_a_run_cache_finds_what_an_earlier_run_recorded(tmp_path):
+    """The run history is the cache; there is no second store to keep."""
+    store = ArtifactStore(tmp_path / "artifacts")
+    reference = store.put([b"payload"], name="data.bin")
+    plain, with_artifact, collected = "k-plain", "k-artifact", "k-collected"
+    with Session(db_engine) as session:
+        session.add(
+            RunNode(
+                run_id="cache-1",
+                node="study.a",
+                status="ok",
+                cache_key=plain,
+                outputs=json.dumps({"study.loss": 1.5}),
+            )
+        )
+        session.add(
+            RunNode(
+                run_id="cache-2",
+                node="study.b",
+                status="ok",
+                cache_key=with_artifact,
+                outputs=json.dumps({"study.data": reference}),
+            )
+        )
+        session.add(
+            RunNode(
+                run_id="cache-3",
+                node="study.c",
+                status="ok",
+                cache_key=collected,
+                outputs=json.dumps(
+                    {"study.data": {**reference, "digest": "sha256:" + "1" * 64}}
+                ),
+            )
+        )
+        session.commit()
+
+    cache = RunCache(store)
+    assert cache.lookup(plain) == (True, {"study.loss": 1.5})
+    assert cache.lookup(with_artifact) == (True, {"study.data": reference})
+    # Its bytes have gone from the store, so the reference names nothing a
+    # restored run could open. That is a miss, not a broken run.
+    assert cache.lookup(collected) == (False, None)
+    assert cache.lookup("never-seen") == (False, None)
+    assert cache.lookup("") == (False, None)
+
+
+def test_what_may_be_stored_as_a_cache_entry():
+    """A row carries a key and its outputs together, or neither."""
+    ok = NodeOutcome(
+        node="study.a", ok=True, cache_key="k", output_values={"study.loss": 1.0}
+    )
+    assert _cacheable(ok) == '{"study.loss":1.0}'
+    # A node that published nothing is still an answer worth reusing.
+    assert _cacheable(ok.model_copy(update={"output_values": None})) == "null"
+    # Not cacheable: it failed, it has no key, or it returned too much.
+    assert _cacheable(ok.model_copy(update={"ok": False})) is None
+    assert _cacheable(ok.model_copy(update={"cache_key": ""})) is None
+    big = {"study.data": "x" * (OUTPUT_CAP + 1)}
+    assert _cacheable(ok.model_copy(update={"output_values": big})) is None

backend/tests/flow/test_runs.py

@@ -10,7 +10,7 @@ import pytest
 
 from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.nodes import Node
-from fluksio.flow.pipeline import NodeOutcome, Pipeline
+from fluksio.flow.pipeline import NodeOutcome, Pipeline, run_cache_key
 from fluksio.flow.runs import (
     MetricSink,
     RunRejected,
@@ -333,3 +333,105 @@ def test_a_runs_seed_fills_an_input_of_that_name():
 def test_a_flow_without_a_seed_input_ignores_the_runs_seed():
     flow = double_flow()
     assert seed_values(flow, {"lr": 1.0}, seed=7) == {"study.lr": 1.0}
+
+
+# -----------------------------------------------------------------------------
+# The stage cache — a node whose inputs have not changed is not run again
+# -----------------------------------------------------------------------------
+
+
+class FakeCache:
+    """A stage cache with no database behind it, and a record of what it was asked."""
+
+    def __init__(self, entries: dict[str, dict | None] | None = None) -> None:
+        self.entries = entries or {}
+        self.asked: list[str] = []
+
+    def lookup(self, key: str):
+        self.asked.append(key)
+        if key in self.entries:
+            return True, self.entries[key]
+        return False, None
+
+
+def counting_node(flow: str = "study") -> tuple[Node, list[int]]:
+    """A node that says how many times it actually ran."""
+    calls: list[int] = []
+
+    def train(lr, params):
+        calls.append(1)
+        return {"loss": lr * 2}
+
+    node = make_node(
+        "train", flow, train, requires=[spec("lr")], provides=[spec("loss")]
+    )
+    node.fingerprint = "fp-train"
+    return node, calls
+
+
+def test_a_cache_hit_restores_the_outputs_without_running_the_node():
+    flow = double_flow()
+    node, calls = counting_node()
+    state = MemoryState()
+    seen: list[NodeOutcome] = []
+    key = run_cache_key("fp-train", {"study.lr": 0.5})
+    cache = FakeCache({key: {"study.loss": 99.0}})
+
+    Pipeline(nodes=[node], state=state, observer=seen.append, run_cache=cache).run(
+        seed_values(flow, {"lr": 0.5})
+    )
+
+    assert calls == []
+    # Restored into this run's own state, which is where everything
+    # downstream of it looks — its namespace holds nothing otherwise.
+    assert collect_result(flow, state) == {"loss": 99.0}
+    assert seen[0].cached and seen[0].cache_key == key
+
+
+def test_a_miss_runs_the_node_and_carries_what_would_be_stored():
+    flow = double_flow()
+    node, calls = counting_node()
+    seen: list[NodeOutcome] = []
+    cache = FakeCache()
+
+    Pipeline(
+        nodes=[node], state=MemoryState(), observer=seen.append, run_cache=cache
+    ).run(seed_values(flow, {"lr": 0.5}))
+
+    assert calls == [1]
+    assert cache.asked == [run_cache_key("fp-train", {"study.lr": 0.5})]
+    assert not seen[0].cached
+    assert seen[0].cache_key and seen[0].output_values == {"study.loss": 1.0}
+
+
+def test_the_key_follows_the_inputs():
+    first = run_cache_key("fp", {"lr": 0.5})
+    assert first != run_cache_key("fp", {"lr": 0.6})
+    assert first != run_cache_key("other", {"lr": 0.5})
+    assert first == run_cache_key("fp", {"lr": 0.5})
+
+
+def test_an_artifact_input_counts_as_its_digest():
+    digest = "sha256:" + "0" * 64
+    # The same bytes under another name, of a size recorded differently, are
+    # the same input — the reference is a handle, the digest is the content.
+    assert run_cache_key(
+        "fp", {"data": {"digest": digest, "name": "a.csv", "size": 3}}
+    ) == run_cache_key("fp", {"data": {"digest": digest, "name": "b.csv", "size": 3}})
+
+
+def test_a_node_with_no_fingerprint_is_never_looked_up():
+    """Built-in and connector nodes, and anything declared `cache=False`."""
+    flow = double_flow()
+    node, calls = counting_node()
+    node.fingerprint = ""
+    seen: list[NodeOutcome] = []
+    cache = FakeCache()
+
+    Pipeline(
+        nodes=[node], state=MemoryState(), observer=seen.append, run_cache=cache
+    ).run(seed_values(flow, {"lr": 0.5}))
+
+    assert calls == [1]
+    assert cache.asked == []
+    assert seen[0].cache_key == ""

backend/tests/flow/test_stage_cache_build.py

@@ -0,0 +1,83 @@
+"""What a node is fingerprinted with, decided where it is built.
+
+The pipeline half — what a hit restores — is in `test_runs.py`. This is the
+link between the two: a node only carries a fingerprint when it is built for a
+run, and only when it is allowed to be cached at all.
+"""
+
+from pathlib import Path
+
+import pytest
+
+from fluksio.flow.controller import FlowController, RunContext
+from fluksio.flow.messages import DType, MessageSpec
+from fluksio.flow.schemas import FlowDef, NodeDef
+from fluksio.flow.state import MemoryState
+from fluksio.flow.store import FlowStore
+
+SOURCE = "def process(reading, factor):\n    return {'scaled': reading * factor}\n"
+
+
+def a_flow(**params: object) -> FlowDef:
+    return FlowDef(
+        name="house",
+        mode="batch",
+        nodes=[
+            NodeDef(
+                id="scale",
+                params=dict(params),
+                requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
+                provides=[MessageSpec(name="scaled", dtype=DType.FLOAT)],
+            )
+        ],
+    )
+
+
+@pytest.fixture
+def store(tmp_path: Path) -> FlowStore:
+    return FlowStore(tmp_path / "flows")
+
+
+def fingerprint_of(store: FlowStore, flow: FlowDef) -> str:
+    store.write_flow(flow)
+    store.write_node_source(flow.name, "scale", SOURCE)
+    pipeline = FlowController(store).build_run_pipeline(
+        store.read_flow(flow.name),
+        state=MemoryState(),
+        run=RunContext(run_id="r-1"),
+    )
+    return pipeline.nodes[0].fingerprint
+
+
+def test_a_settings_change_is_a_different_node(store: FlowStore):
+    first = fingerprint_of(store, a_flow(factor=3))
+    assert first
+    assert fingerprint_of(store, a_flow(factor=4)) != first
+    # And the same flow again is the same node, which is the whole point.
+    assert fingerprint_of(store, a_flow(factor=3)) == first
+
+
+def test_a_source_change_is_a_different_node(store: FlowStore):
+    first = fingerprint_of(store, a_flow(factor=3))
+    store.write_node_source("house", "scale", SOURCE.replace("*", "+"))
+    pipeline = FlowController(store).build_run_pipeline(
+        store.read_flow("house"), state=MemoryState(), run=RunContext(run_id="r-2")
+    )
+    assert pipeline.nodes[0].fingerprint != first
+
+
+def test_a_node_that_opted_out_carries_none(store: FlowStore):
+    flow = a_flow(factor=3)
+    flow.nodes[0].cache = False
+    assert fingerprint_of(store, flow) == ""
+
+
+def test_a_live_pipeline_fingerprints_nothing(store: FlowStore):
+    """Only a run may reuse a result; a cascade is about what just happened."""
+    store.write_flow(a_flow(factor=3))
+    store.write_node_source("house", "scale", SOURCE)
+    controller = FlowController(store)
+    nodes, _loaded, _initial, _inputs = controller._build_flows(
+        [(store.read_flow("house"), False)]
+    )
+    assert nodes[0].fingerprint == ""

backend/tests/test_cli.py

@@ -137,3 +137,121 @@ def test_run_syncs_by_default_and_can_be_told_not_to() -> None:
 
     assert _parser().parse_args(["run", "train"]).no_sync is False
     assert _parser().parse_args(["run", "train", "--no-sync"]).no_sync is True
+
+
+def test_a_sweep_is_the_product_of_the_parameters_given() -> None:
+    """`--param lr=0.1,0.01 --param epochs=1,2` is four runs, typed by the flow."""
+    import pytest
+
+    from fluksio.sdk import SyncError
+    from fluksio.sdk.cli import _grid
+
+    definition = {
+        "inputs": [
+            {"spec": {"name": "lr", "dtype": "float"}},
+            {"spec": {"name": "epochs", "dtype": "int"}},
+        ]
+    }
+
+    grid = _grid(definition, ["lr=0.1,0.01", "epochs=1,2"], seed=7)
+    assert [entry["params"] for entry in grid] == [
+        {"lr": 0.1, "epochs": 1},
+        {"lr": 0.1, "epochs": 2},
+        {"lr": 0.01, "epochs": 1},
+        {"lr": 0.01, "epochs": 2},
+    ]
+    assert all(entry["seed"] == 7 for entry in grid)
+
+    with pytest.raises(SyncError, match="not an input of this flow"):
+        _grid(definition, ["nonesuch=1"], seed=None)
+    with pytest.raises(SyncError, match="name=value"):
+        _grid(definition, ["lr"], seed=None)
+
+
+def test_the_local_engine_is_asked_for_rather_than_guessed() -> None:
+    from fluksio.cli import _parser
+
+    parser = _parser()
+    assert parser.parse_args(["run", "train"]).local is False
+    assert parser.parse_args(["run", "train", "--local"]).local is True
+    assert parser.parse_args(["runs", "--local"]).local is True
+    assert parser.parse_args(["sweep", "train", "--param", "lr=1"]).local is False
+
+
+def test_a_local_run_always_waits(monkeypatch) -> None:
+    """The engine is this process, so a run nobody waits for is thrown away."""
+    from contextlib import contextmanager
+
+    from fluksio.cli import _parser
+    from fluksio.sdk import cli
+
+    submitted: dict[str, object] = {}
+
+    class FakeHandle:
+        id = "run-1"
+        status = "ok"
+        result: dict[str, object] = {}
+
+        def wait(self, timeout: float = 0.0) -> "FakeHandle":
+            submitted["waited"] = True
+            return self
+
+    class FakeClient:
+        def get_flow(self, name: str) -> dict[str, object]:
+            return {"definition": {"inputs": []}}
+
+        def submit(self, flow, params, seed=None, no_cache=False):
+            submitted["flow"] = flow
+            submitted["no_cache"] = no_cache
+            return FakeHandle()
+
+        def run(self, run_id: str) -> dict[str, object]:
+            return {"nodes": [{"status": "cached"}, {"status": "ok"}]}
+
+    @contextmanager
+    def fake_engine():
+        yield FakeClient()
+
+    monkeypatch.setattr(cli, "_engine_client", fake_engine)
+
+    args = _parser().parse_args(["run", "train", "--local", "--no-sync", "--no-cache"])
+    assert cli.cmd_run(args, []) == 0
+    assert submitted == {"flow": "train", "no_cache": True, "waited": True}
+
+
+def test_ctrl_c_while_waiting_cancels_the_run(monkeypatch) -> None:
+    """Interrupting means stop the run, not walk away leaving it going."""
+    from contextlib import contextmanager
+
+    from fluksio.cli import _parser
+    from fluksio.sdk import cli
+
+    cancelled: list[str] = []
+
+    class FakeHandle:
+        id = "run-1"
+        status = "running"
+        result: dict[str, object] = {}
+
+        def wait(self, timeout: float = 0.0):
+            raise KeyboardInterrupt
+
+    class FakeClient:
+        def get_flow(self, name: str) -> dict[str, object]:
+            return {"definition": {"inputs": []}}
+
+        def submit(self, flow, params, seed=None, no_cache=False):
+            return FakeHandle()
+
+        def cancel(self, run_id: str) -> None:
+            cancelled.append(run_id)
+
+    @contextmanager
+    def fake_engine():
+        yield FakeClient()
+
+    monkeypatch.setattr(cli, "_engine_client", fake_engine)
+
+    args = _parser().parse_args(["run", "train", "--local", "--no-sync"])
+    assert cli.cmd_run(args, []) == 130
+    assert cancelled == ["run-1"]