Stage caching for batch runs, and an engine that lives in the command

A code node in a batch run is now fingerprinted by its source, its raw
settings and the values it reads — an artifact input counting as its digest,
which is what the content addressing was always for. A run that finds the key
restores what the earlier one returned and skips the node, recorded as
`cached`. The run history is the cache: `run_node.outputs` beside the
`cache_key` the schema already had, no second store. On for code nodes, never
for the built-in and connector types that have side effects; off per node with
`@node(cache=False)` and per run with `--no-cache`.

Emissions are not replayed on a hit, so a cached training node returns its
result without redrawing its curve. Recorded in NOTEPAD.md with the two other
deliberate limits.

`fluksio run --local` boots the real app in the command's own process and
drives it through its ASGI interface behind the ordinary client, so a run no
longer needs a `serve` terminal beside it — same data directory, same history,
and the cache carries between the two. It always waits, because the engine it
starts lives exactly as long as the command.

Also: `fluksio sweep --param lr=0.1,0.01` for the product of the lists,
`run --follow` for a run's numbers as they arrive, Ctrl-C cancelling a waited
run rather than abandoning it, coloured statuses on a terminal, and `name`
made optional on the metrics endpoint so a follower can ask for every series.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/fluksio/flow/controller.py

@@ -15,6 +15,8 @@ nodes are built like any other's, so `set_enabled` only starts or stops them.
 from __future__ import annotations
 
 import asyncio
+import hashlib
+import json
 import logging
 import time
 import traceback
@@ -51,7 +53,13 @@ from fluksio.flow.nodes import (
     SwitchNode,
     TriggerNode,
 )
-from fluksio.flow.pipeline import NodeOutcome, Pipeline, ValidationIssue, ValueSource
+from fluksio.flow.pipeline import (
+    NodeOutcome,
+    Pipeline,
+    RunCacheLookup,
+    ValidationIssue,
+    ValueSource,
+)
 from fluksio.flow.remote import RemoteWorkerHub
 from fluksio.flow.schemas import (
     BrainEdge,
@@ -904,6 +912,17 @@ class FlowController:
                     params=params,
                     name=node_def.id,
                 )
+                if run is not None and node_def.cache:
+                    # The raw params, not the resolved ones: a secret's value
+                    # must not end up in a key, and its name is what changes
+                    # when the node is reconfigured anyway.
+                    node.fingerprint = hashlib.sha256(
+                        json.dumps(
+                            {"source": code, "params": node_def.params},
+                            sort_keys=True,
+                            separators=(",", ":"),
+                        ).encode()
+                    ).hexdigest()
                 emissions.node = node
             else:
                 node = node_type.cls(
@@ -1339,6 +1358,7 @@ class FlowController:
         observer: Callable[[NodeOutcome], None] | None = None,
         emission_observer: Callable[[str, dict[str, Any]], None] | None = None,
         run: RunContext | None = None,
+        run_cache: RunCacheLookup | None = None,
     ) -> Pipeline:
         """Build one flow as a pipeline of its own, for a single run.
 
@@ -1359,6 +1379,7 @@ class FlowController:
             initial_values=initial_values,
             observer=observer,
             emission_observer=emission_observer,
+            run_cache=run_cache,
         )
         pipeline.history_limits = self.history_limits
         return pipeline

backend/fluksio/flow/nodes/base.py

@@ -91,6 +91,7 @@ class Node:
         "synchronous",
         "_on_health",
         "supervisor",
+        "fingerprint",
     )
 
     def __init__(
@@ -107,6 +108,10 @@ class Node:
         # Set by the controller before start(); a node built on its own (tests,
         # previews) runs its loops unsupervised.
         self.supervisor: Supervisor | None = None
+        # Everything about this node a cached result would depend on, hashed.
+        # Empty means it is not cacheable — which is every node type that is
+        # not code, and every node built for anything but a run.
+        self.fingerprint: str = ""
         self.params = dict(params) if params else {}
         self.synchronous = bool(self.params.get("synchronous", False))
 

backend/fluksio/flow/pipeline.py

@@ -13,6 +13,8 @@ construction. A deploy does that rather than building a second pipeline.
 
 from __future__ import annotations
 
+import hashlib
+import json
 import logging
 import threading
 import time
@@ -21,7 +23,7 @@ from collections import deque
 from collections.abc import Callable, Iterator
 from concurrent.futures import Future, ThreadPoolExecutor, wait
 from contextlib import contextmanager
-from typing import Any, Literal
+from typing import Any, Literal, Protocol
 
 from pydantic import BaseModel
 
@@ -99,6 +101,46 @@ class NodeOutcome(BaseModel):
     #: Artifact references this node emitted, keyed by the message carrying
     #: them — what a run records so a result can be opened later.
     artifacts: dict[str, dict[str, Any]] = {}
+    #: Restored from an earlier run rather than executed.
+    cached: bool = False
+    #: What an equal execution of this node would be looked up by. Empty when
+    #: the node is not cacheable at all.
+    cache_key: str = ""
+    #: What it returned, for whoever stores the cache. None when it published
+    #: nothing, which is a result a later run has to be able to restore too.
+    output_values: dict[str, Any] | None = None
+
+
+class RunCacheLookup(Protocol):
+    """Where a pipeline asks whether a node has already been run.
+
+    Kept to one method so the pipeline never learns there is a database: a run
+    hands it one of these, a test hands it a dict.
+    """
+
+    def lookup(self, key: str) -> tuple[bool, dict[str, Any] | None]:
+        """(hit, outputs). Outputs None on a hit means it published nothing."""
+
+
+def run_cache_key(fingerprint: str, inputs: dict[str, Any]) -> str:
+    """What this node, with these inputs, is known by.
+
+    An artifact input counts as its digest: the reference carries a name and a
+    size beside it, and the same bytes under another name are the same input.
+    A value JSON cannot carry cannot be part of a key, and a node reading one
+    is simply not cacheable.
+    """
+    reduced = {
+        name: value["digest"] if is_reference(value) else value
+        for name, value in inputs.items()
+    }
+    try:
+        canonical = json.dumps(
+            {"fp": fingerprint, "in": reduced}, sort_keys=True, separators=(",", ":")
+        )
+    except (TypeError, ValueError):
+        return ""
+    return hashlib.sha256(canonical.encode()).hexdigest()
 
 
 def _derive(
@@ -156,6 +198,7 @@ class Pipeline:
         "history_limits",
         "observer",
         "emission_observer",
+        "run_cache",
     )
 
     def __init__(
@@ -170,6 +213,7 @@ class Pipeline:
         node_pool: ThreadPoolExecutor | None = None,
         observer: Callable[[NodeOutcome], None] | None = None,
         emission_observer: Callable[[str, dict[str, Any]], None] | None = None,
+        run_cache: RunCacheLookup | None = None,
     ) -> None:
         self._nodes = nodes or []
         # Stopped flows are stored and survive a restart; paused ones are a
@@ -200,6 +244,10 @@ class Pipeline:
         # And every value a node produced on the way, which is what a
         # training curve is once it goes out a port rather than into a log.
         self.emission_observer = emission_observer
+        # Set by a run that may reuse earlier results. A live pipeline has
+        # none: a cascade is about what just happened, not about what a node
+        # once returned for the same inputs.
+        self.run_cache = run_cache
         # How deep to keep each message's series; a chart asking for more
         # than the default puts its message in here. Swapped, never mutated.
         self.history_limits: dict[str, int] = {}
@@ -758,6 +806,60 @@ class Pipeline:
             # for a value that may never have been delivered at all.
             return False
 
+    def _from_cache(
+        self, node: Node, key: str, state: StateBackend, entry_id: str
+    ) -> tuple[bool, dict[str, Any] | None]:
+        """Restore an earlier run of this node: (hit, what it published).
+
+        Both halves are needed, because a node that published nothing is a
+        result worth restoring and looks exactly like a miss otherwise.
+
+        The outputs go into state as if the node had just returned them, which
+        is what everything downstream reads — a run's state namespace is its
+        own, so a skipped node leaves nothing behind for the next one to find.
+        What it emitted on the way is not restored: those values were the
+        story of an execution that is not happening this time.
+        """
+        assert self.run_cache is not None
+        try:
+            hit, outputs = self.run_cache.lookup(key)
+        except Exception:
+            # A cache that cannot answer is a cache miss, never a failed node.
+            logger.exception("Cache lookup failed for '%s'", node.id)
+            return False, None
+        if not hit:
+            return False, None
+
+        if outputs:
+            self._record_outputs(node, outputs, state)
+        self._publish(
+            {
+                "type": "node_executed",
+                "flow": node.flow,
+                "node": node.id,
+                "outputs": len(outputs or {}),
+                "duration_ms": 0.0,
+                "run": entry_id,
+                "ts": time.time(),
+            }
+        )
+        self._observe(
+            NodeOutcome(
+                node=node.id,
+                ok=True,
+                cached=True,
+                cache_key=key,
+                outputs=len(outputs or {}),
+                output_values=outputs,
+                artifacts={
+                    name: value
+                    for name, value in (outputs or {}).items()
+                    if is_reference(value)
+                },
+            )
+        )
+        return True, outputs
+
     def _execute_node(
         self, node: Node, state: StateBackend, entry_id: str = ""
     ) -> dict[str, Any] | None:
@@ -768,6 +870,14 @@ class Pipeline:
             with state.lock():
                 inputs = {k: state[k] for k in node.requires if k in state}
 
+            key = ""
+            if self.run_cache is not None and node.fingerprint:
+                key = run_cache_key(node.fingerprint, inputs)
+            if key:
+                hit, restored = self._from_cache(node, key, state, entry_id)
+                if hit:
+                    return restored
+
             with logs.capture(collected):
                 result = node.execute(inputs)
             self.publish_log(node, collected, "")
@@ -818,6 +928,10 @@ class Pipeline:
                         for name, value in (result or {}).items()
                         if is_reference(value)
                     },
+                    cache_key=key,
+                    # Post-throttle: what went into state is what a later run
+                    # restoring this node has to find.
+                    output_values=result,
                 )
             )
             return result

backend/fluksio/flow/runs.py

@@ -43,6 +43,7 @@ from sqlalchemy.dialects.sqlite import insert as upsert
 from sqlmodel import Session, col, select
 
 from fluksio.core.db import engine as db_engine
+from fluksio.flow.artifacts import ArtifactStore, is_reference
 from fluksio.flow.controller import FlowController, RunContext
 from fluksio.flow.messages import qualify
 from fluksio.flow.pipeline import NodeOutcome, Pipeline
@@ -69,6 +70,10 @@ CLAIM_COUNT = 4
 CLAIM_BLOCK_MS = 1000
 ERROR_CAP = 2000
 LOG_CAP = 8000
+#: How much of a node's returned outputs is kept so a later run can restore
+#: them. Past it the node simply is not cacheable — a cache is not a store,
+#: and an artifact is how a large result is meant to travel.
+OUTPUT_CAP = 256_000
 #: Reported numbers held before they are written. A training loop reporting
 #: every step must not be a round trip every step.
 METRIC_BATCH = 500
@@ -288,6 +293,68 @@ def origin_commit(flow: FlowDef) -> str:
     return f"{origin.commit}-dirty" if origin.dirty else origin.commit
 
 
+def _cacheable(outcome: NodeOutcome) -> str | None:
+    """A node's outputs as stored, or None when it may not be reused."""
+    if not outcome.ok or not outcome.cache_key:
+        return None
+    try:
+        text = json.dumps(outcome.output_values, sort_keys=True, separators=(",", ":"))
+    except (TypeError, ValueError):
+        return None
+    return text if len(text) <= OUTPUT_CAP else None
+
+
+class RunCache:
+    """What earlier runs already worked out, looked up by cache key.
+
+    The rows are the run history itself — a node that ran successfully and
+    whose outputs were small enough to keep is a cache entry, without a store
+    of its own. An entry whose artifacts have gone from the content store is
+    not a hit: the reference would name bytes nobody can open.
+    """
+
+    def __init__(self, artifacts: ArtifactStore | None = None) -> None:
+        self.artifacts = artifacts
+
+    def lookup(self, key: str) -> tuple[bool, dict[str, Any] | None]:
+        if not key:
+            return False, None
+        try:
+            with Session(db_engine) as session:
+                rows = session.exec(
+                    select(RunNode)
+                    .where(
+                        col(RunNode.cache_key) == key,
+                        col(RunNode.status).in_(("ok", "cached")),
+                        col(RunNode.outputs).is_not(None),
+                    )
+                    # Run ids are time-ordered, so this is the most recent
+                    # first. A handful is enough to get past entries whose
+                    # artifacts have been collected.
+                    .order_by(col(RunNode.run_id).desc())
+                    .limit(5)
+                ).all()
+                for row in rows:
+                    outputs = json.loads(row.outputs or "null")
+                    if outputs and self._unrestorable(outputs):
+                        continue
+                    return True, outputs
+        except Exception:
+            # A cache that cannot answer is a cache miss, never a failed run.
+            logger.exception("Cache lookup failed; running the node instead")
+        return False, None
+
+    def _unrestorable(self, outputs: dict[str, Any]) -> bool:
+        return any(
+            is_reference(value)
+            and (
+                self.artifacts is None
+                or self.artifacts.path(str(value["digest"])) is None
+            )
+            for value in outputs.values()
+        )
+
+
 class RunService:
     """Accepts runs, drives them, and writes down what they did."""
 
@@ -297,9 +364,11 @@ class RunService:
         queue: WorkQueue,
         state_factory: Callable[[str], StateBackend] | None = None,
         parallel: int = MAX_PARALLEL,
+        artifacts: ArtifactStore | None = None,
     ) -> None:
         self.controller = controller
         self.queue = queue
+        self._cache = RunCache(artifacts)
         # Without one, a run gets a private in-memory state — which is exactly
         # the isolation it wants, minus surviving the process.
         self._state_factory = state_factory or (lambda _ns: MemoryState())
@@ -356,6 +425,7 @@ class RunService:
         cause: str = "api",
         actor: str = "",
         draft: bool = False,
+        no_cache: bool = False,
     ) -> Run:
         """Journal a run and wake an engine up for it. Never blocks on it."""
         flow = self.controller.store.read_flow(flow_name, draft=draft)
@@ -378,6 +448,7 @@ class RunService:
             seed=seed,
             group_id=group_id,
             cause=cause,
+            no_cache=no_cache,
             status="queued",
             labels=required_labels(flow),
             created_at=datetime.now(UTC),
@@ -605,6 +676,7 @@ class RunService:
                 observer=observe,
                 emission_observer=sink.handle,
                 run=RunContext(run_id=run_id),
+                run_cache=None if run.no_cache else self._cache,
             )
             with self._lock:
                 self._active[run_id] = pipeline
@@ -644,14 +716,19 @@ class RunService:
                     logger.warning("Could not clear state of run %s", run_id)
 
     def _record_node(self, run_id: str, outcome: NodeOutcome) -> None:
+        outputs = _cacheable(outcome)
         row = RunNode(
             run_id=run_id,
             node=outcome.node[:255],
-            status="ok" if outcome.ok else "error",
+            status="cached" if outcome.cached else ("ok" if outcome.ok else "error"),
             started_at=datetime.now(UTC),
             duration_ms=outcome.duration_ms,
             error=outcome.error[:ERROR_CAP],
             logs=outcome.logs[:LOG_CAP],
+            # Together or not at all: a row carrying a key must be one a
+            # lookup can actually restore from.
+            cache_key=outcome.cache_key if outputs is not None else "",
+            outputs=outputs,
         )
         try:
             with Session(db_engine) as session:

backend/fluksio/flow/schemas.py

@@ -70,6 +70,14 @@ class NodeDef(BaseModel):
             "locally anyway."
         ),
     )
+    cache: bool = Field(
+        default=True,
+        description=(
+            "Whether a batch run may reuse an earlier execution of this node "
+            "with the same source, settings and inputs. Turn it off for a "
+            "function whose answer can change on its own."
+        ),
+    )
 
     @field_validator("id")
     @classmethod