Stage caching for batch runs, and an engine that lives in the command

A code node in a batch run is now fingerprinted by its source, its raw
settings and the values it reads — an artifact input counting as its digest,
which is what the content addressing was always for. A run that finds the key
restores what the earlier one returned and skips the node, recorded as
`cached`. The run history is the cache: `run_node.outputs` beside the
`cache_key` the schema already had, no second store. On for code nodes, never
for the built-in and connector types that have side effects; off per node with
`@node(cache=False)` and per run with `--no-cache`.

Emissions are not replayed on a hit, so a cached training node returns its
result without redrawing its curve. Recorded in NOTEPAD.md with the two other
deliberate limits.

`fluksio run --local` boots the real app in the command's own process and
drives it through its ASGI interface behind the ordinary client, so a run no
longer needs a `serve` terminal beside it — same data directory, same history,
and the cache carries between the two. It always waits, because the engine it
starts lives exactly as long as the command.

Also: `fluksio sweep --param lr=0.1,0.01` for the product of the lists,
`run --follow` for a run's numbers as they arrive, Ctrl-C cancelling a waited
run rather than abandoning it, coloured statuses on a terminal, and `name`
made optional on the metrics endpoint so a follower can ask for every series.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/fluksio/flow/pipeline.py

@@ -13,6 +13,8 @@ construction. A deploy does that rather than building a second pipeline.
 
 from __future__ import annotations
 
+import hashlib
+import json
 import logging
 import threading
 import time
@@ -21,7 +23,7 @@ from collections import deque
 from collections.abc import Callable, Iterator
 from concurrent.futures import Future, ThreadPoolExecutor, wait
 from contextlib import contextmanager
-from typing import Any, Literal
+from typing import Any, Literal, Protocol
 
 from pydantic import BaseModel
 
@@ -99,6 +101,46 @@ class NodeOutcome(BaseModel):
     #: Artifact references this node emitted, keyed by the message carrying
     #: them — what a run records so a result can be opened later.
     artifacts: dict[str, dict[str, Any]] = {}
+    #: Restored from an earlier run rather than executed.
+    cached: bool = False
+    #: What an equal execution of this node would be looked up by. Empty when
+    #: the node is not cacheable at all.
+    cache_key: str = ""
+    #: What it returned, for whoever stores the cache. None when it published
+    #: nothing, which is a result a later run has to be able to restore too.
+    output_values: dict[str, Any] | None = None
+
+
+class RunCacheLookup(Protocol):
+    """Where a pipeline asks whether a node has already been run.
+
+    Kept to one method so the pipeline never learns there is a database: a run
+    hands it one of these, a test hands it a dict.
+    """
+
+    def lookup(self, key: str) -> tuple[bool, dict[str, Any] | None]:
+        """(hit, outputs). Outputs None on a hit means it published nothing."""
+
+
+def run_cache_key(fingerprint: str, inputs: dict[str, Any]) -> str:
+    """What this node, with these inputs, is known by.
+
+    An artifact input counts as its digest: the reference carries a name and a
+    size beside it, and the same bytes under another name are the same input.
+    A value JSON cannot carry cannot be part of a key, and a node reading one
+    is simply not cacheable.
+    """
+    reduced = {
+        name: value["digest"] if is_reference(value) else value
+        for name, value in inputs.items()
+    }
+    try:
+        canonical = json.dumps(
+            {"fp": fingerprint, "in": reduced}, sort_keys=True, separators=(",", ":")
+        )
+    except (TypeError, ValueError):
+        return ""
+    return hashlib.sha256(canonical.encode()).hexdigest()
 
 
 def _derive(
@@ -156,6 +198,7 @@ class Pipeline:
         "history_limits",
         "observer",
         "emission_observer",
+        "run_cache",
     )
 
     def __init__(
@@ -170,6 +213,7 @@ class Pipeline:
         node_pool: ThreadPoolExecutor | None = None,
         observer: Callable[[NodeOutcome], None] | None = None,
         emission_observer: Callable[[str, dict[str, Any]], None] | None = None,
+        run_cache: RunCacheLookup | None = None,
     ) -> None:
         self._nodes = nodes or []
         # Stopped flows are stored and survive a restart; paused ones are a
@@ -200,6 +244,10 @@ class Pipeline:
         # And every value a node produced on the way, which is what a
         # training curve is once it goes out a port rather than into a log.
         self.emission_observer = emission_observer
+        # Set by a run that may reuse earlier results. A live pipeline has
+        # none: a cascade is about what just happened, not about what a node
+        # once returned for the same inputs.
+        self.run_cache = run_cache
         # How deep to keep each message's series; a chart asking for more
         # than the default puts its message in here. Swapped, never mutated.
         self.history_limits: dict[str, int] = {}
@@ -758,6 +806,60 @@ class Pipeline:
             # for a value that may never have been delivered at all.
             return False
 
+    def _from_cache(
+        self, node: Node, key: str, state: StateBackend, entry_id: str
+    ) -> tuple[bool, dict[str, Any] | None]:
+        """Restore an earlier run of this node: (hit, what it published).
+
+        Both halves are needed, because a node that published nothing is a
+        result worth restoring and looks exactly like a miss otherwise.
+
+        The outputs go into state as if the node had just returned them, which
+        is what everything downstream reads — a run's state namespace is its
+        own, so a skipped node leaves nothing behind for the next one to find.
+        What it emitted on the way is not restored: those values were the
+        story of an execution that is not happening this time.
+        """
+        assert self.run_cache is not None
+        try:
+            hit, outputs = self.run_cache.lookup(key)
+        except Exception:
+            # A cache that cannot answer is a cache miss, never a failed node.
+            logger.exception("Cache lookup failed for '%s'", node.id)
+            return False, None
+        if not hit:
+            return False, None
+
+        if outputs:
+            self._record_outputs(node, outputs, state)
+        self._publish(
+            {
+                "type": "node_executed",
+                "flow": node.flow,
+                "node": node.id,
+                "outputs": len(outputs or {}),
+                "duration_ms": 0.0,
+                "run": entry_id,
+                "ts": time.time(),
+            }
+        )
+        self._observe(
+            NodeOutcome(
+                node=node.id,
+                ok=True,
+                cached=True,
+                cache_key=key,
+                outputs=len(outputs or {}),
+                output_values=outputs,
+                artifacts={
+                    name: value
+                    for name, value in (outputs or {}).items()
+                    if is_reference(value)
+                },
+            )
+        )
+        return True, outputs
+
     def _execute_node(
         self, node: Node, state: StateBackend, entry_id: str = ""
     ) -> dict[str, Any] | None:
@@ -768,6 +870,14 @@ class Pipeline:
             with state.lock():
                 inputs = {k: state[k] for k in node.requires if k in state}
 
+            key = ""
+            if self.run_cache is not None and node.fingerprint:
+                key = run_cache_key(node.fingerprint, inputs)
+            if key:
+                hit, restored = self._from_cache(node, key, state, entry_id)
+                if hit:
+                    return restored
+
             with logs.capture(collected):
                 result = node.execute(inputs)
             self.publish_log(node, collected, "")
@@ -818,6 +928,10 @@ class Pipeline:
                         for name, value in (result or {}).items()
                         if is_reference(value)
                     },
+                    cache_key=key,
+                    # Post-throttle: what went into state is what a later run
+                    # restoring this node has to find.
+                    output_values=result,
                 )
             )
             return result