Stage caching for batch runs, and an engine that lives in the command

A code node in a batch run is now fingerprinted by its source, its raw
settings and the values it reads — an artifact input counting as its digest,
which is what the content addressing was always for. A run that finds the key
restores what the earlier one returned and skips the node, recorded as
`cached`. The run history is the cache: `run_node.outputs` beside the
`cache_key` the schema already had, no second store. On for code nodes, never
for the built-in and connector types that have side effects; off per node with
`@node(cache=False)` and per run with `--no-cache`.

Emissions are not replayed on a hit, so a cached training node returns its
result without redrawing its curve. Recorded in NOTEPAD.md with the two other
deliberate limits.

`fluksio run --local` boots the real app in the command's own process and
drives it through its ASGI interface behind the ordinary client, so a run no
longer needs a `serve` terminal beside it — same data directory, same history,
and the cache carries between the two. It always waits, because the engine it
starts lives exactly as long as the command.

Also: `fluksio sweep --param lr=0.1,0.01` for the product of the lists,
`run --follow` for a run's numbers as they arrive, Ctrl-C cancelling a waited
run rather than abandoning it, coloured statuses on a terminal, and `name`
made optional on the metrics endpoint so a follower can ask for every series.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/tests/flow/test_runs.py

@@ -10,7 +10,7 @@ import pytest
 
 from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.nodes import Node
-from fluksio.flow.pipeline import NodeOutcome, Pipeline
+from fluksio.flow.pipeline import NodeOutcome, Pipeline, run_cache_key
 from fluksio.flow.runs import (
     MetricSink,
     RunRejected,
@@ -333,3 +333,105 @@ def test_a_runs_seed_fills_an_input_of_that_name():
 def test_a_flow_without_a_seed_input_ignores_the_runs_seed():
     flow = double_flow()
     assert seed_values(flow, {"lr": 1.0}, seed=7) == {"study.lr": 1.0}
+
+
+# -----------------------------------------------------------------------------
+# The stage cache — a node whose inputs have not changed is not run again
+# -----------------------------------------------------------------------------
+
+
+class FakeCache:
+    """A stage cache with no database behind it, and a record of what it was asked."""
+
+    def __init__(self, entries: dict[str, dict | None] | None = None) -> None:
+        self.entries = entries or {}
+        self.asked: list[str] = []
+
+    def lookup(self, key: str):
+        self.asked.append(key)
+        if key in self.entries:
+            return True, self.entries[key]
+        return False, None
+
+
+def counting_node(flow: str = "study") -> tuple[Node, list[int]]:
+    """A node that says how many times it actually ran."""
+    calls: list[int] = []
+
+    def train(lr, params):
+        calls.append(1)
+        return {"loss": lr * 2}
+
+    node = make_node(
+        "train", flow, train, requires=[spec("lr")], provides=[spec("loss")]
+    )
+    node.fingerprint = "fp-train"
+    return node, calls
+
+
+def test_a_cache_hit_restores_the_outputs_without_running_the_node():
+    flow = double_flow()
+    node, calls = counting_node()
+    state = MemoryState()
+    seen: list[NodeOutcome] = []
+    key = run_cache_key("fp-train", {"study.lr": 0.5})
+    cache = FakeCache({key: {"study.loss": 99.0}})
+
+    Pipeline(nodes=[node], state=state, observer=seen.append, run_cache=cache).run(
+        seed_values(flow, {"lr": 0.5})
+    )
+
+    assert calls == []
+    # Restored into this run's own state, which is where everything
+    # downstream of it looks — its namespace holds nothing otherwise.
+    assert collect_result(flow, state) == {"loss": 99.0}
+    assert seen[0].cached and seen[0].cache_key == key
+
+
+def test_a_miss_runs_the_node_and_carries_what_would_be_stored():
+    flow = double_flow()
+    node, calls = counting_node()
+    seen: list[NodeOutcome] = []
+    cache = FakeCache()
+
+    Pipeline(
+        nodes=[node], state=MemoryState(), observer=seen.append, run_cache=cache
+    ).run(seed_values(flow, {"lr": 0.5}))
+
+    assert calls == [1]
+    assert cache.asked == [run_cache_key("fp-train", {"study.lr": 0.5})]
+    assert not seen[0].cached
+    assert seen[0].cache_key and seen[0].output_values == {"study.loss": 1.0}
+
+
+def test_the_key_follows_the_inputs():
+    first = run_cache_key("fp", {"lr": 0.5})
+    assert first != run_cache_key("fp", {"lr": 0.6})
+    assert first != run_cache_key("other", {"lr": 0.5})
+    assert first == run_cache_key("fp", {"lr": 0.5})
+
+
+def test_an_artifact_input_counts_as_its_digest():
+    digest = "sha256:" + "0" * 64
+    # The same bytes under another name, of a size recorded differently, are
+    # the same input — the reference is a handle, the digest is the content.
+    assert run_cache_key(
+        "fp", {"data": {"digest": digest, "name": "a.csv", "size": 3}}
+    ) == run_cache_key("fp", {"data": {"digest": digest, "name": "b.csv", "size": 3}})
+
+
+def test_a_node_with_no_fingerprint_is_never_looked_up():
+    """Built-in and connector nodes, and anything declared `cache=False`."""
+    flow = double_flow()
+    node, calls = counting_node()
+    node.fingerprint = ""
+    seen: list[NodeOutcome] = []
+    cache = FakeCache()
+
+    Pipeline(
+        nodes=[node], state=MemoryState(), observer=seen.append, run_cache=cache
+    ).run(seed_values(flow, {"lr": 0.5}))
+
+    assert calls == [1]
+    assert cache.asked == []
+    assert seen[0].cache_key == ""

backend/tests/flow/test_stage_cache_build.py

@@ -0,0 +1,83 @@
+"""What a node is fingerprinted with, decided where it is built.
+
+The pipeline half — what a hit restores — is in `test_runs.py`. This is the
+link between the two: a node only carries a fingerprint when it is built for a
+run, and only when it is allowed to be cached at all.
+"""
+
+from pathlib import Path
+
+import pytest
+
+from fluksio.flow.controller import FlowController, RunContext
+from fluksio.flow.messages import DType, MessageSpec
+from fluksio.flow.schemas import FlowDef, NodeDef
+from fluksio.flow.state import MemoryState
+from fluksio.flow.store import FlowStore
+
+SOURCE = "def process(reading, factor):\n    return {'scaled': reading * factor}\n"
+
+
+def a_flow(**params: object) -> FlowDef:
+    return FlowDef(
+        name="house",
+        mode="batch",
+        nodes=[
+            NodeDef(
+                id="scale",
+                params=dict(params),
+                requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
+                provides=[MessageSpec(name="scaled", dtype=DType.FLOAT)],
+            )
+        ],
+    )
+
+
+@pytest.fixture
+def store(tmp_path: Path) -> FlowStore:
+    return FlowStore(tmp_path / "flows")
+
+
+def fingerprint_of(store: FlowStore, flow: FlowDef) -> str:
+    store.write_flow(flow)
+    store.write_node_source(flow.name, "scale", SOURCE)
+    pipeline = FlowController(store).build_run_pipeline(
+        store.read_flow(flow.name),
+        state=MemoryState(),
+        run=RunContext(run_id="r-1"),
+    )
+    return pipeline.nodes[0].fingerprint
+
+
+def test_a_settings_change_is_a_different_node(store: FlowStore):
+    first = fingerprint_of(store, a_flow(factor=3))
+    assert first
+    assert fingerprint_of(store, a_flow(factor=4)) != first
+    # And the same flow again is the same node, which is the whole point.
+    assert fingerprint_of(store, a_flow(factor=3)) == first
+
+
+def test_a_source_change_is_a_different_node(store: FlowStore):
+    first = fingerprint_of(store, a_flow(factor=3))
+    store.write_node_source("house", "scale", SOURCE.replace("*", "+"))
+    pipeline = FlowController(store).build_run_pipeline(
+        store.read_flow("house"), state=MemoryState(), run=RunContext(run_id="r-2")
+    )
+    assert pipeline.nodes[0].fingerprint != first
+
+
+def test_a_node_that_opted_out_carries_none(store: FlowStore):
+    flow = a_flow(factor=3)
+    flow.nodes[0].cache = False
+    assert fingerprint_of(store, flow) == ""
+
+
+def test_a_live_pipeline_fingerprints_nothing(store: FlowStore):
+    """Only a run may reuse a result; a cascade is about what just happened."""
+    store.write_flow(a_flow(factor=3))
+    store.write_node_source("house", "scale", SOURCE)
+    controller = FlowController(store)
+    nodes, _loaded, _initial, _inputs = controller._build_flows(
+        [(store.read_flow("house"), False)]
+    )
+    assert nodes[0].fingerprint == ""