Name the code a run ran, and let an interrupted sync finish

Three faults with one root: the stored body of a code-defined node is an
import shim, and nothing that mattered was ever read from the code itself.

- The run stamp could not identify what ran. The shim imports whatever is on
  disk when the worker starts, and an uncommitted tree stamps <commit>-dirty
  for every run it ever produces. Run.code_digest hashes the repository's .py
  files, memoized on their stat state, and it is read again when the run is
  actually claimed -- so a sweep queued for hours records the code each of its
  runs executed, not the code that was there when it was submitted.
- The stage cache adopted code that was too new. The fingerprint hashed the
  shim, which is invariant under any edit to the imported function or anything
  it calls into, so a re-run was served from cache and answered without the
  outputs the edit added. It now carries the repo digest and the node's
  declared ports. Every fingerprint changes once, which invalidates the
  existing cache; a canvas flow has no repository and keys as before.
- An interrupted sync looked like a hand-edited canvas. The engine answers a
  new-node template for a node with no stored body, and the template carries
  no marker, so the drift check read "somebody edited this" and demanded
  --force -- for the one state that re-running the sync is the fix for.
  NodeSource.missing states the fact, and sync skips those and reuses the
  bodies it read instead of asking for each one twice.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/fluksio/flow/runs.py

@@ -38,6 +38,7 @@ import uuid
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
 from datetime import UTC, datetime, timedelta
+from pathlib import Path
 from typing import Any
 
 from sqlalchemy import update
@@ -409,6 +410,77 @@ def origin_commit(flow: FlowDef) -> str:
     return f"{origin.commit}-dirty" if origin.dirty else origin.commit
 
 
+#: What a repository's python files hashed to, and the stat state that answer
+#: was read from. Keyed by path: the walk is cheap, opening every file is not.
+_digests: dict[str, tuple[tuple[tuple[str, int, int], ...], str]] = {}
+
+#: Directories a project's own code is never in, and which are large.
+_SKIP_DIRS = frozenset({"__pycache__", "node_modules"})
+
+
+def repo_digest(repo: str) -> str:
+    """What the python files under ``repo`` currently hash to.
+
+    A code-defined node is stored as a shim that imports the real function, so
+    the engine runs whatever is on disk when the worker starts. The commit
+    cannot name that: an uncommitted tree stamps every run `-dirty`, however
+    many times the code changed underneath. This can — it is read at run time,
+    from the files themselves.
+
+    Empty when there is no repository to read here, which is the honest answer
+    on an engine that does not share a filesystem with the author: a stamp
+    without a digest says the engine could not see the code, and the cache key
+    falls back to what it keyed on before.
+    """
+    if not repo or not os.path.isdir(repo):
+        return ""
+    state: list[tuple[str, int, int]] = []
+    for root, dirs, files in os.walk(repo):
+        dirs[:] = [
+            d
+            for d in dirs
+            if not d.startswith(".")
+            and d not in _SKIP_DIRS
+            # A virtualenv carries this, and its site-packages is not the
+            # project's code — nor is it small.
+            and not os.path.exists(os.path.join(root, d, "pyvenv.cfg"))
+        ]
+        for name in files:
+            if not name.endswith(".py"):
+                continue
+            path = os.path.join(root, name)
+            try:
+                info = os.stat(path)
+            except OSError:
+                continue
+            state.append((os.path.relpath(path, repo), info.st_mtime_ns, info.st_size))
+    state.sort()
+    stamp = tuple(state)
+    cached = _digests.get(repo)
+    if cached is not None and cached[0] == stamp:
+        return cached[1]
+
+    digest = hashlib.sha256()
+    for relative, _mtime, _size in state:
+        try:
+            body = Path(repo, relative).read_bytes()
+        except OSError:
+            continue
+        digest.update(relative.encode())
+        digest.update(b"\0")
+        digest.update(body)
+        digest.update(b"\0")
+    answer = digest.hexdigest()
+    _digests[repo] = (stamp, answer)
+    return answer
+
+
+def code_digest(flow: FlowDef) -> str:
+    """The digest of the repository a code-defined flow was declared in."""
+    origin = flow.origin
+    return repo_digest(origin.repo) if origin is not None else ""
+
+
 def _cacheable(outcome: NodeOutcome) -> str | None:
     """A node's outputs as stored, or None when it may not be reused."""
     if not outcome.ok or not outcome.cache_key:
@@ -579,6 +651,7 @@ class RunService:
             flow_version=flow.version,
             commit=self.controller.store.head(),
             origin_commit=origin_commit(flow),
+            code_digest=code_digest(flow),
             params=params,
             params_digest=digest_of(params, seed),
             seed=seed,
@@ -803,6 +876,17 @@ class RunService:
                 return None
             return session.exec(select(Run).where(col(Run.id) == run_id)).first()
 
+    def _restamp(self, run: Run, digest: str) -> str:
+        """Record the code this run is about to execute, if it moved."""
+        if digest == run.code_digest:
+            return digest
+        with Session(db_engine) as session:
+            session.exec(
+                update(Run).where(col(Run.id) == run.id).values(code_digest=digest)
+            )
+            session.commit()
+        return digest
+
     def _drive(self, run_id: str) -> None:
         run = self._claim(run_id)
         if run is None:
@@ -823,13 +907,18 @@ class RunService:
         sink = MetricSink(run_id)
         try:
             flow = self.controller.store.read_flow(run.flow)
+            # Read again, now that it is this run's turn: a sweep queues every
+            # run at once, and the code on disk is free to move in the hours
+            # before the last of them starts. What the record must name is the
+            # state that ran, not the state that was submitted.
+            run.code_digest = self._restamp(run, code_digest(flow))
             state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}")
             pipeline = self.controller.build_run_pipeline(
                 flow,
                 state=state,
                 observer=observe,
                 emission_observer=sink.handle,
-                run=RunContext(run_id=run_id),
+                run=RunContext(run_id=run_id, code_digest=run.code_digest),
                 run_cache=None if run.no_cache else self._cache,
             )
             with self._lock: