Name the code a run ran, and let an interrupted sync finish

Three faults with one root: the stored body of a code-defined node is an
import shim, and nothing that mattered was ever read from the code itself.

- The run stamp could not identify what ran. The shim imports whatever is on
  disk when the worker starts, and an uncommitted tree stamps <commit>-dirty
  for every run it ever produces. Run.code_digest hashes the repository's .py
  files, memoized on their stat state, and it is read again when the run is
  actually claimed -- so a sweep queued for hours records the code each of its
  runs executed, not the code that was there when it was submitted.
- The stage cache adopted code that was too new. The fingerprint hashed the
  shim, which is invariant under any edit to the imported function or anything
  it calls into, so a re-run was served from cache and answered without the
  outputs the edit added. It now carries the repo digest and the node's
  declared ports. Every fingerprint changes once, which invalidates the
  existing cache; a canvas flow has no repository and keys as before.
- An interrupted sync looked like a hand-edited canvas. The engine answers a
  new-node template for a node with no stored body, and the template carries
  no marker, so the drift check read "somebody edited this" and demanded
  --force -- for the one state that re-running the sync is the fix for.
  NodeSource.missing states the fact, and sync skips those and reuses the
  bodies it read instead of asking for each one twice.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/fluksio/alembic/versions/a4d9e2b71c68_run_code_digest.py

@@ -0,0 +1,38 @@
+"""run.code_digest
+
+`origin_commit` names a commit, and a dirty tree has the same one all day. The
+node bodies are import shims, so what actually ran is whatever was on disk when
+the worker started. This column hashes those files at run time, which is the
+only thing that tells two runs of an uncommitted tree apart.
+
+Revision ID: a4d9e2b71c68
+Revises: f2c6a8d15e93
+Create Date: 2026-08-26
+
+"""
+
+import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a4d9e2b71c68"
+down_revision = "f2c6a8d15e93"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "run",
+        sa.Column(
+            "code_digest",
+            sqlmodel.sql.sqltypes.AutoString(length=64),
+            nullable=False,
+            server_default="",
+        ),
+    )
+
+
+def downgrade():
+    op.drop_column("run", "code_digest")

backend/fluksio/api/routes/flows.py

@@ -529,7 +529,10 @@ def read_node_source(
             return NodeSource(code=controller.store.read_lib_source(ref))
         except LibNotFound:
             raise HTTPException(status_code=404, detail=f"No shared node '{ref}'")
-    return NodeSource(code=controller.store.read_node_source(name, node_id, draft=True))
+    return NodeSource(
+        code=controller.store.read_node_source(name, node_id, draft=True),
+        missing=not controller.store.has_node_source(name, node_id, draft=True),
+    )
 
 
 @router.put("/{name}/nodes/{node_id}/source", response_model=NodeStatusPublic)

backend/fluksio/api/routes/runs.py

@@ -107,6 +107,10 @@ class RunRow(BaseModel):
     #: decorators. Empty for one drawn on the canvas, where `commit` is the
     #: whole answer to what produced the number.
     origin_commit: str = ""
+    #: What that repository's python files hashed to when the run started.
+    #: Two runs of one dirty tree share a commit and differ here, which is the
+    #: only way to tell apart what they actually executed.
+    code_digest: str = ""
     #: The flow store's own commit. Short, unlike `result`, so the list
     #: carries it: "what code produced this" is a question asked of a table.
     commit: str = ""

backend/fluksio/flow/controller.py

@@ -150,6 +150,10 @@ class RunContext:
     """
 
     run_id: str
+    #: What the code-defined flow's repository hashed to when this run started.
+    #: Part of every node's fingerprint, so editing a function the node calls
+    #: into invalidates the cache the way editing the node itself does.
+    code_digest: str = ""
 
 
 @dataclass(frozen=True, slots=True)
@@ -919,9 +923,30 @@ class FlowController:
                     # The raw params, not the resolved ones: a secret's value
                     # must not end up in a key, and its name is what changes
                     # when the node is reconfigured anyway.
+                    #
+                    # `source` is the stored body, which for a code-defined
+                    # flow is a shim that imports the real function — it does
+                    # not move when that function does, and it says nothing
+                    # about what the function calls into. `code` is what
+                    # covers both: the digest of the repository the shim
+                    # imports from. The ports are here because declaring a new
+                    # one changes what the node answers with, which a hit
+                    # would otherwise restore without.
                     node.fingerprint = hashlib.sha256(
                         json.dumps(
-                            {"source": code, "params": node_def.params},
+                            {
+                                "source": code,
+                                "params": node_def.params,
+                                "requires": [
+                                    spec.model_dump(mode="json")
+                                    for spec in node_def.requires
+                                ],
+                                "provides": [
+                                    spec.model_dump(mode="json")
+                                    for spec in node_def.provides
+                                ],
+                                "code": run.code_digest,
+                            },
                             sort_keys=True,
                             separators=(",", ":"),
                         ).encode()

backend/fluksio/flow/runs.py

@@ -38,6 +38,7 @@ import uuid
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
 from datetime import UTC, datetime, timedelta
+from pathlib import Path
 from typing import Any
 
 from sqlalchemy import update
@@ -409,6 +410,77 @@ def origin_commit(flow: FlowDef) -> str:
     return f"{origin.commit}-dirty" if origin.dirty else origin.commit
 
 
+#: What a repository's python files hashed to, and the stat state that answer
+#: was read from. Keyed by path: the walk is cheap, opening every file is not.
+_digests: dict[str, tuple[tuple[tuple[str, int, int], ...], str]] = {}
+
+#: Directories a project's own code is never in, and which are large.
+_SKIP_DIRS = frozenset({"__pycache__", "node_modules"})
+
+
+def repo_digest(repo: str) -> str:
+    """What the python files under ``repo`` currently hash to.
+
+    A code-defined node is stored as a shim that imports the real function, so
+    the engine runs whatever is on disk when the worker starts. The commit
+    cannot name that: an uncommitted tree stamps every run `-dirty`, however
+    many times the code changed underneath. This can — it is read at run time,
+    from the files themselves.
+
+    Empty when there is no repository to read here, which is the honest answer
+    on an engine that does not share a filesystem with the author: a stamp
+    without a digest says the engine could not see the code, and the cache key
+    falls back to what it keyed on before.
+    """
+    if not repo or not os.path.isdir(repo):
+        return ""
+    state: list[tuple[str, int, int]] = []
+    for root, dirs, files in os.walk(repo):
+        dirs[:] = [
+            d
+            for d in dirs
+            if not d.startswith(".")
+            and d not in _SKIP_DIRS
+            # A virtualenv carries this, and its site-packages is not the
+            # project's code — nor is it small.
+            and not os.path.exists(os.path.join(root, d, "pyvenv.cfg"))
+        ]
+        for name in files:
+            if not name.endswith(".py"):
+                continue
+            path = os.path.join(root, name)
+            try:
+                info = os.stat(path)
+            except OSError:
+                continue
+            state.append((os.path.relpath(path, repo), info.st_mtime_ns, info.st_size))
+    state.sort()
+    stamp = tuple(state)
+    cached = _digests.get(repo)
+    if cached is not None and cached[0] == stamp:
+        return cached[1]
+
+    digest = hashlib.sha256()
+    for relative, _mtime, _size in state:
+        try:
+            body = Path(repo, relative).read_bytes()
+        except OSError:
+            continue
+        digest.update(relative.encode())
+        digest.update(b"\0")
+        digest.update(body)
+        digest.update(b"\0")
+    answer = digest.hexdigest()
+    _digests[repo] = (stamp, answer)
+    return answer
+
+
+def code_digest(flow: FlowDef) -> str:
+    """The digest of the repository a code-defined flow was declared in."""
+    origin = flow.origin
+    return repo_digest(origin.repo) if origin is not None else ""
+
+
 def _cacheable(outcome: NodeOutcome) -> str | None:
     """A node's outputs as stored, or None when it may not be reused."""
     if not outcome.ok or not outcome.cache_key:
@@ -579,6 +651,7 @@ class RunService:
             flow_version=flow.version,
             commit=self.controller.store.head(),
             origin_commit=origin_commit(flow),
+            code_digest=code_digest(flow),
             params=params,
             params_digest=digest_of(params, seed),
             seed=seed,
@@ -803,6 +876,17 @@ class RunService:
                 return None
             return session.exec(select(Run).where(col(Run.id) == run_id)).first()
 
+    def _restamp(self, run: Run, digest: str) -> str:
+        """Record the code this run is about to execute, if it moved."""
+        if digest == run.code_digest:
+            return digest
+        with Session(db_engine) as session:
+            session.exec(
+                update(Run).where(col(Run.id) == run.id).values(code_digest=digest)
+            )
+            session.commit()
+        return digest
+
     def _drive(self, run_id: str) -> None:
         run = self._claim(run_id)
         if run is None:
@@ -823,13 +907,18 @@ class RunService:
         sink = MetricSink(run_id)
         try:
             flow = self.controller.store.read_flow(run.flow)
+            # Read again, now that it is this run's turn: a sweep queues every
+            # run at once, and the code on disk is free to move in the hours
+            # before the last of them starts. What the record must name is the
+            # state that ran, not the state that was submitted.
+            run.code_digest = self._restamp(run, code_digest(flow))
             state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}")
             pipeline = self.controller.build_run_pipeline(
                 flow,
                 state=state,
                 observer=observe,
                 emission_observer=sink.handle,
-                run=RunContext(run_id=run_id),
+                run=RunContext(run_id=run_id, code_digest=run.code_digest),
                 run_cache=None if run.no_cache else self._cache,
             )
             with self._lock:

backend/fluksio/flow/schemas.py

@@ -158,6 +158,10 @@ class NodeSource(BaseModel):
     """The Python source of a node."""
 
     code: str
+    #: True when nothing is stored and `code` is the new-node template. An
+    #: editor opens on it either way; a client deciding whether somebody wrote
+    #: that code needs to know it was nobody. Read-only — set on the way out.
+    missing: bool = False
 
 
 Health = Literal["ok", "degraded", "down"]

backend/fluksio/flow/store.py

@@ -549,6 +549,18 @@ class FlowStore:
         path = self._node_file(flow, node_id)
         return path.read_text() if path.exists() else DEFAULT_SOURCE
 
+    def has_node_source(self, flow: str, node_id: str, draft: bool = False) -> bool:
+        """Whether a body was ever stored for this node.
+
+        `read_node_source` answers a template when there is none, which is what
+        a new node's editor should open with — but it is a placeholder, not
+        code anybody wrote, and telling the two apart is the difference between
+        "someone edited this" and "this was never written".
+        """
+        if draft and self._draft_node_file(flow, node_id).exists():
+            return True
+        return self._node_file(flow, node_id).exists()
+
     def write_node_source(
         self, flow: str, node_id: str, code: str, draft: bool = False
     ) -> bool:

backend/fluksio/models.py

@@ -331,6 +331,12 @@ class Run(SQLModel, table=True):
     #: one names the code it imported. A `-dirty` suffix is git's own way of
     #: saying the tree had changes that are in no commit at all.
     origin_commit: str = Field(default="", max_length=80)
+    #: sha256 over that repository's python files, read when the run actually
+    #: started. The commit cannot tell two runs of an uncommitted tree apart —
+    #: both stamp `-dirty` — and the shim imports whatever is on disk, so this
+    #: is the half that names the code. Empty when the engine cannot see the
+    #: repository, or for a flow drawn on the canvas.
+    code_digest: str = Field(default="", max_length=64)
     params: dict[str, Any] = Field(sa_column=Column(JSON), default_factory=dict)
     #: sha256 over params and seed. Two runs of the same thing share it, which
     #: is what makes "have I already run this?" a lookup.

backend/fluksio/sdk/cli.py

@@ -695,6 +695,19 @@ def cmd_status(args: argparse.Namespace) -> int:
 PARAMS_WIDTH = 80
 
 
+def _stamp(row: dict[str, Any]) -> str:
+    """What code a run ran: the commit, whether it was dirty, and the digest.
+
+    The digest is the part that separates two runs of one uncommitted tree —
+    the commit says `-dirty` for both, however much changed in between.
+    """
+    commit, dirty, _ = (row.get("origin_commit") or "").partition("-dirty")
+    digest = row.get("code_digest") or ""
+    return (
+        commit[:7] + ("-dirty" if dirty else "") + (f"+{digest[:7]}" if digest else "")
+    )
+
+
 def cmd_runs(args: argparse.Namespace) -> int:
     try:
         with _client_for(args) as client:
@@ -704,13 +717,12 @@ def cmd_runs(args: argparse.Namespace) -> int:
     except httpx.HTTPError as exc:
         return _unreachable(exc)
     for row in rows:
-        commit = (row.get("origin_commit") or "")[:7]
         params = json.dumps(row["params"])
         if len(params) > PARAMS_WIDTH:
             params = params[: PARAMS_WIDTH - 3] + "..."
         _say(
             f"{row['id']}  {_status(row['status'], 9)} {row['flow']:<16} "
-            f"{row['duration_ms'] / 1000:7.1f}s  {commit:<8} {params}"
+            f"{row['duration_ms'] / 1000:7.1f}s  {_stamp(row):<22} {params}"
         )
     return 0
 

backend/fluksio/sdk/client.py

@@ -237,8 +237,17 @@ class Client:
         return result
 
     def get_source(self, flow: str, node: str) -> str:
-        result = self._call("GET", f"/flows/{flow}/nodes/{node}/source")
-        return str(result.get("code", ""))
+        return str(self.get_source_entry(flow, node).get("code", ""))
+
+    def get_source_entry(self, flow: str, node: str) -> dict[str, Any]:
+        """A node's stored body, and whether there is one at all.
+
+        `missing` is what separates a node nobody has written from a node
+        somebody edited: the engine answers a new-node template for both, and
+        only one of them is work worth refusing to overwrite.
+        """
+        result: dict[str, Any] = self._call("GET", f"/flows/{flow}/nodes/{node}/source")
+        return result or {}
 
     def put_source(self, flow: str, node: str, code: str) -> dict[str, Any]:
         result: dict[str, Any] = self._call(
@@ -623,13 +632,16 @@ def _sync_one(
     report = SyncReport(target.name)
     stored = client.get_flow(target.name)
     version = 1
+    # What the drift check already read, so the write loop below does not ask
+    # the engine for every node's body a second time.
+    known: dict[str, str] = {}
     if stored is None:
         report.created = True
     else:
         definition = stored.get("definition") or {}
         version = int(definition.get("version") or 1)
         if not force:
-            _refuse_on_drift(client, target, definition)
+            known = _refuse_on_drift(client, target, definition)
 
     saved = client.put_flow(target.document(origin) | {"version": version})
     stored_version = int((saved.get("definition") or {}).get("version") or version)
@@ -640,8 +652,12 @@ def _sync_one(
     version = stored_version
 
     for node_id, code in target.shims().items():
-        if not report.created and client.get_source(target.name, node_id) == code:
-            continue
+        if not report.created:
+            stored_code = known.get(node_id)
+            if stored_code is None:
+                stored_code = client.get_source(target.name, node_id)
+            if stored_code == code:
+                continue
         client.put_source(target.name, node_id, code)
         report.changed.append(node_id)
 
@@ -651,26 +667,43 @@ def _sync_one(
     return report
 
 
-def _refuse_on_drift(client: Client, target: Flow, definition: dict[str, Any]) -> None:
+def _refuse_on_drift(
+    client: Client, target: Flow, definition: dict[str, Any]
+) -> dict[str, str]:
     """Stop before overwriting work that was done somewhere else.
 
     Two ways a stored flow is not ours to replace: it was drawn on the canvas
     and has no origin at all, or one of its node bodies no longer carries the
     line saying it was generated — which means somebody edited the code there.
+
+    A node with *no* stored body is neither. That is what a sync interrupted
+    part of the way through leaves, and it used to read as a canvas edit,
+    because the engine answers the new-node template for it and the template
+    carries no marker either. Re-running the sync is the fix for that state,
+    so refusing to is precisely wrong.
+
+    Returns the bodies it read, so the caller can compare against them without
+    asking again.
     """
     if not definition.get("origin"):
         raise SyncError(
             f"flow '{target.name}' was not created by sync, so replacing it would "
             "discard whoever drew it. Rename yours, or pass --force."
         )
+    seen: dict[str, str] = {}
     for stored_node in definition.get("nodes") or []:
         node_id = str(stored_node.get("id"))
         try:
-            code = client.get_source(target.name, node_id)
+            entry = client.get_source_entry(target.name, node_id)
         except ApiError:
             continue
+        if entry.get("missing"):
+            continue
+        code = str(entry.get("code", ""))
+        seen[node_id] = code
         if code and not code.startswith(MARKER):
             raise SyncError(
                 f"node '{target.name}.{node_id}' was edited on the canvas, and "
                 "syncing would throw that edit away. Copy it out, or pass --force."
             )
+    return seen