Name the code a run ran, and let an interrupted sync finish

Three faults with one root: the stored body of a code-defined node is an
import shim, and nothing that mattered was ever read from the code itself.

- The run stamp could not identify what ran. The shim imports whatever is on
  disk when the worker starts, and an uncommitted tree stamps <commit>-dirty
  for every run it ever produces. Run.code_digest hashes the repository's .py
  files, memoized on their stat state, and it is read again when the run is
  actually claimed -- so a sweep queued for hours records the code each of its
  runs executed, not the code that was there when it was submitted.
- The stage cache adopted code that was too new. The fingerprint hashed the
  shim, which is invariant under any edit to the imported function or anything
  it calls into, so a re-run was served from cache and answered without the
  outputs the edit added. It now carries the repo digest and the node's
  declared ports. Every fingerprint changes once, which invalidates the
  existing cache; a canvas flow has no repository and keys as before.
- An interrupted sync looked like a hand-edited canvas. The engine answers a
  new-node template for a node with no stored body, and the template carries
  no marker, so the drift check read "somebody edited this" and demanded
  --force -- for the one state that re-running the sync is the fix for.
  NodeSource.missing states the fact, and sync skips those and reuses the
  bodies it read instead of asking for each one twice.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/fluksio/flow/controller.py

@@ -150,6 +150,10 @@ class RunContext:
     """
 
     run_id: str
+    #: What the code-defined flow's repository hashed to when this run started.
+    #: Part of every node's fingerprint, so editing a function the node calls
+    #: into invalidates the cache the way editing the node itself does.
+    code_digest: str = ""
 
 
 @dataclass(frozen=True, slots=True)
@@ -919,9 +923,30 @@ class FlowController:
                     # The raw params, not the resolved ones: a secret's value
                     # must not end up in a key, and its name is what changes
                     # when the node is reconfigured anyway.
+                    #
+                    # `source` is the stored body, which for a code-defined
+                    # flow is a shim that imports the real function — it does
+                    # not move when that function does, and it says nothing
+                    # about what the function calls into. `code` is what
+                    # covers both: the digest of the repository the shim
+                    # imports from. The ports are here because declaring a new
+                    # one changes what the node answers with, which a hit
+                    # would otherwise restore without.
                     node.fingerprint = hashlib.sha256(
                         json.dumps(
-                            {"source": code, "params": node_def.params},
+                            {
+                                "source": code,
+                                "params": node_def.params,
+                                "requires": [
+                                    spec.model_dump(mode="json")
+                                    for spec in node_def.requires
+                                ],
+                                "provides": [
+                                    spec.model_dump(mode="json")
+                                    for spec in node_def.provides
+                                ],
+                                "code": run.code_digest,
+                            },
                             sort_keys=True,
                             separators=(",", ":"),
                         ).encode()

backend/fluksio/flow/runs.py

@@ -38,6 +38,7 @@ import uuid
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
 from datetime import UTC, datetime, timedelta
+from pathlib import Path
 from typing import Any
 
 from sqlalchemy import update
@@ -409,6 +410,77 @@ def origin_commit(flow: FlowDef) -> str:
     return f"{origin.commit}-dirty" if origin.dirty else origin.commit
 
 
+#: What a repository's python files hashed to, and the stat state that answer
+#: was read from. Keyed by path: the walk is cheap, opening every file is not.
+_digests: dict[str, tuple[tuple[tuple[str, int, int], ...], str]] = {}
+
+#: Directories a project's own code is never in, and which are large.
+_SKIP_DIRS = frozenset({"__pycache__", "node_modules"})
+
+
+def repo_digest(repo: str) -> str:
+    """What the python files under ``repo`` currently hash to.
+
+    A code-defined node is stored as a shim that imports the real function, so
+    the engine runs whatever is on disk when the worker starts. The commit
+    cannot name that: an uncommitted tree stamps every run `-dirty`, however
+    many times the code changed underneath. This can — it is read at run time,
+    from the files themselves.
+
+    Empty when there is no repository to read here, which is the honest answer
+    on an engine that does not share a filesystem with the author: a stamp
+    without a digest says the engine could not see the code, and the cache key
+    falls back to what it keyed on before.
+    """
+    if not repo or not os.path.isdir(repo):
+        return ""
+    state: list[tuple[str, int, int]] = []
+    for root, dirs, files in os.walk(repo):
+        dirs[:] = [
+            d
+            for d in dirs
+            if not d.startswith(".")
+            and d not in _SKIP_DIRS
+            # A virtualenv carries this, and its site-packages is not the
+            # project's code — nor is it small.
+            and not os.path.exists(os.path.join(root, d, "pyvenv.cfg"))
+        ]
+        for name in files:
+            if not name.endswith(".py"):
+                continue
+            path = os.path.join(root, name)
+            try:
+                info = os.stat(path)
+            except OSError:
+                continue
+            state.append((os.path.relpath(path, repo), info.st_mtime_ns, info.st_size))
+    state.sort()
+    stamp = tuple(state)
+    cached = _digests.get(repo)
+    if cached is not None and cached[0] == stamp:
+        return cached[1]
+
+    digest = hashlib.sha256()
+    for relative, _mtime, _size in state:
+        try:
+            body = Path(repo, relative).read_bytes()
+        except OSError:
+            continue
+        digest.update(relative.encode())
+        digest.update(b"\0")
+        digest.update(body)
+        digest.update(b"\0")
+    answer = digest.hexdigest()
+    _digests[repo] = (stamp, answer)
+    return answer
+
+
+def code_digest(flow: FlowDef) -> str:
+    """The digest of the repository a code-defined flow was declared in."""
+    origin = flow.origin
+    return repo_digest(origin.repo) if origin is not None else ""
+
+
 def _cacheable(outcome: NodeOutcome) -> str | None:
     """A node's outputs as stored, or None when it may not be reused."""
     if not outcome.ok or not outcome.cache_key:
@@ -579,6 +651,7 @@ class RunService:
             flow_version=flow.version,
             commit=self.controller.store.head(),
             origin_commit=origin_commit(flow),
+            code_digest=code_digest(flow),
             params=params,
             params_digest=digest_of(params, seed),
             seed=seed,
@@ -803,6 +876,17 @@ class RunService:
                 return None
             return session.exec(select(Run).where(col(Run.id) == run_id)).first()
 
+    def _restamp(self, run: Run, digest: str) -> str:
+        """Record the code this run is about to execute, if it moved."""
+        if digest == run.code_digest:
+            return digest
+        with Session(db_engine) as session:
+            session.exec(
+                update(Run).where(col(Run.id) == run.id).values(code_digest=digest)
+            )
+            session.commit()
+        return digest
+
     def _drive(self, run_id: str) -> None:
         run = self._claim(run_id)
         if run is None:
@@ -823,13 +907,18 @@ class RunService:
         sink = MetricSink(run_id)
         try:
             flow = self.controller.store.read_flow(run.flow)
+            # Read again, now that it is this run's turn: a sweep queues every
+            # run at once, and the code on disk is free to move in the hours
+            # before the last of them starts. What the record must name is the
+            # state that ran, not the state that was submitted.
+            run.code_digest = self._restamp(run, code_digest(flow))
             state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}")
             pipeline = self.controller.build_run_pipeline(
                 flow,
                 state=state,
                 observer=observe,
                 emission_observer=sink.handle,
-                run=RunContext(run_id=run_id),
+                run=RunContext(run_id=run_id, code_digest=run.code_digest),
                 run_cache=None if run.no_cache else self._cache,
             )
             with self._lock:

backend/fluksio/flow/schemas.py

@@ -158,6 +158,10 @@ class NodeSource(BaseModel):
     """The Python source of a node."""
 
     code: str
+    #: True when nothing is stored and `code` is the new-node template. An
+    #: editor opens on it either way; a client deciding whether somebody wrote
+    #: that code needs to know it was nobody. Read-only — set on the way out.
+    missing: bool = False
 
 
 Health = Literal["ok", "degraded", "down"]

backend/fluksio/flow/store.py

@@ -549,6 +549,18 @@ class FlowStore:
         path = self._node_file(flow, node_id)
         return path.read_text() if path.exists() else DEFAULT_SOURCE
 
+    def has_node_source(self, flow: str, node_id: str, draft: bool = False) -> bool:
+        """Whether a body was ever stored for this node.
+
+        `read_node_source` answers a template when there is none, which is what
+        a new node's editor should open with — but it is a placeholder, not
+        code anybody wrote, and telling the two apart is the difference between
+        "someone edited this" and "this was never written".
+        """
+        if draft and self._draft_node_file(flow, node_id).exists():
+            return True
+        return self._node_file(flow, node_id).exists()
+
     def write_node_source(
         self, flow: str, node_id: str, code: str, draft: bool = False
     ) -> bool: