Name the code a run ran, and let an interrupted sync finish

Three faults with one root: the stored body of a code-defined node is an
import shim, and nothing that mattered was ever read from the code itself.

- The run stamp could not identify what ran. The shim imports whatever is on
  disk when the worker starts, and an uncommitted tree stamps <commit>-dirty
  for every run it ever produces. Run.code_digest hashes the repository's .py
  files, memoized on their stat state, and it is read again when the run is
  actually claimed -- so a sweep queued for hours records the code each of its
  runs executed, not the code that was there when it was submitted.
- The stage cache adopted code that was too new. The fingerprint hashed the
  shim, which is invariant under any edit to the imported function or anything
  it calls into, so a re-run was served from cache and answered without the
  outputs the edit added. It now carries the repo digest and the node's
  declared ports. Every fingerprint changes once, which invalidates the
  existing cache; a canvas flow has no repository and keys as before.
- An interrupted sync looked like a hand-edited canvas. The engine answers a
  new-node template for a node with no stored body, and the template carries
  no marker, so the drift check read "somebody edited this" and demanded
  --force -- for the one state that re-running the sync is the fix for.
  NodeSource.missing states the fact, and sync skips those and reuses the
  bodies it read instead of asking for each one twice.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/tests/api/routes/test_sync.py

@@ -87,6 +87,31 @@ def test_sync_refuses_to_overwrite_a_canvas_edit(api, flows):
     assert api.get_source("train", "evaluate").startswith(MARKER)
 
 
+def test_an_interrupted_sync_finishes_on_the_next_one(api, flows):
+    """A sync that died between the document and the bodies must be resumable.
+
+    That state — nodes named by a stored flow, with no source written for them
+    yet — used to read as a canvas edit, because the engine answers a new-node
+    template for a node with no body and the template carries no marker. The
+    advice was `--force`, for a canvas nobody had touched.
+    """
+    target = flows["train"]
+    target.name = "resumed"
+    try:
+        # Exactly what the document PUT leaves behind before any body is written.
+        api.put_flow(target.document(ORIGIN) | {"version": 1})
+        assert api.get_source_entry("resumed", "fit")["missing"]
+
+        reports = sync([target], api, origin=ORIGIN)
+
+        assert "fit" in reports[0].changed
+        assert api.get_source("resumed", "fit").startswith(MARKER)
+        assert not api.get_source_entry("resumed", "fit")["missing"]
+        assert not api.get_flow("resumed")["has_draft"]
+    finally:
+        target.name = "train"
+
+
 def test_sync_refuses_a_flow_it_did_not_create(api, flows):
     api.put_flow({"name": "drawn", "version": 1, "nodes": [], "mode": "batch"})
     api.publish("drawn", 1)

backend/tests/flow/test_repo_digest.py

@@ -0,0 +1,91 @@
+"""What the engine hashes to decide a repository's code moved.
+
+The commit cannot answer this: an uncommitted tree stamps `-dirty` for every
+run it ever produces, and the node bodies are import shims that pick up
+whatever is on disk when the worker starts.
+"""
+
+from fluksio.flow import runs
+from fluksio.flow.runs import code_digest, repo_digest
+from fluksio.flow.schemas import FlowDef, FlowOrigin
+
+
+def a_repo(tmp_path, body="x = 1\n"):
+    (tmp_path / "pkg").mkdir()
+    (tmp_path / "pkg" / "train.py").write_text(body)
+    return str(tmp_path)
+
+
+def touch(path, body, mtime):
+    """Write and stamp: two writes inside one clock tick look identical."""
+    path.write_text(body)
+    import os
+
+    os.utime(path, ns=(mtime, mtime))
+
+
+def test_the_same_tree_hashes_the_same(tmp_path):
+    repo = a_repo(tmp_path)
+    assert repo_digest(repo) == repo_digest(repo)
+
+
+def test_editing_a_file_changes_it(tmp_path):
+    repo = a_repo(tmp_path)
+    before = repo_digest(repo)
+    touch(tmp_path / "pkg" / "train.py", "x = 2\n", 2_000_000_000_000_000_000)
+
+    assert repo_digest(repo) != before
+
+
+def test_a_new_file_changes_it(tmp_path):
+    repo = a_repo(tmp_path)
+    before = repo_digest(repo)
+    (tmp_path / "pkg" / "helper.py").write_text("y = 1\n")
+
+    assert repo_digest(repo) != before
+
+
+def test_what_is_not_the_project_is_left_out(tmp_path):
+    """A venv is bigger than the project and is not its code."""
+    repo = a_repo(tmp_path)
+    before = repo_digest(repo)
+
+    (tmp_path / ".venv" / "lib").mkdir(parents=True)
+    (tmp_path / ".venv" / "pyvenv.cfg").write_text("home = /usr\n")
+    (tmp_path / ".venv" / "lib" / "numpy.py").write_text("huge = True\n")
+    (tmp_path / "pkg" / "__pycache__").mkdir()
+    (tmp_path / "pkg" / "__pycache__" / "train.py").write_text("compiled\n")
+    (tmp_path / ".git").mkdir()
+    (tmp_path / ".git" / "hook.py").write_text("hook\n")
+    (tmp_path / "pkg" / "notes.md").write_text("prose\n")
+
+    assert repo_digest(repo) == before
+
+
+def test_a_repository_the_engine_cannot_see_is_empty(tmp_path):
+    """Not an error: a worker elsewhere is a normal deployment."""
+    assert repo_digest(str(tmp_path / "nowhere")) == ""
+    assert repo_digest("") == ""
+
+
+def test_only_a_code_defined_flow_has_one(tmp_path):
+    """A flow drawn on the canvas has no repository, and keys as it always did."""
+    repo = a_repo(tmp_path)
+    declared = FlowDef(
+        name="train", mode="batch", origin=FlowOrigin(repo=repo, commit="abc")
+    )
+
+    assert code_digest(declared) == repo_digest(repo)
+    assert code_digest(FlowDef(name="drawn", mode="batch")) == ""
+
+
+def test_unchanged_files_are_not_read_again(tmp_path, monkeypatch):
+    """The warm path is a stat per file, which is what makes this per-submit."""
+    repo = a_repo(tmp_path)
+    repo_digest(repo)
+
+    def refuse(*_args, **_kwargs):
+        raise AssertionError("an unchanged tree must not be re-read")
+
+    monkeypatch.setattr(runs.Path, "read_bytes", refuse)
+    assert repo_digest(repo)

backend/tests/flow/test_stage_cache_build.py

@@ -18,7 +18,7 @@ from fluksio.flow.store import FlowStore
 SOURCE = "def process(reading, factor):\n    return {'scaled': reading * factor}\n"
 
 
-def a_flow(**params: object) -> FlowDef:
+def a_flow(provides: list[MessageSpec] | None = None, **params: object) -> FlowDef:
     return FlowDef(
         name="house",
         mode="batch",
@@ -27,7 +27,7 @@ def a_flow(**params: object) -> FlowDef:
                 id="scale",
                 params=dict(params),
                 requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
-                provides=[MessageSpec(name="scaled", dtype=DType.FLOAT)],
+                provides=provides or [MessageSpec(name="scaled", dtype=DType.FLOAT)],
             )
         ],
     )
@@ -38,13 +38,13 @@ def store(tmp_path: Path) -> FlowStore:
     return FlowStore(tmp_path / "flows")
 
 
-def fingerprint_of(store: FlowStore, flow: FlowDef) -> str:
+def fingerprint_of(store: FlowStore, flow: FlowDef, code_digest: str = "") -> str:
     store.write_flow(flow)
     store.write_node_source(flow.name, "scale", SOURCE)
     pipeline = FlowController(store).build_run_pipeline(
         store.read_flow(flow.name),
         state=MemoryState(),
-        run=RunContext(run_id="r-1"),
+        run=RunContext(run_id="r-1", code_digest=code_digest),
     )
     return pipeline.nodes[0].fingerprint
 
@@ -66,6 +66,37 @@ def test_a_source_change_is_a_different_node(store: FlowStore):
     assert pipeline.nodes[0].fingerprint != first
 
 
+def test_moving_the_repository_is_a_different_node(store: FlowStore):
+    """The stored body of a code-defined node is a shim, and it never moves.
+
+    Editing the function it imports — or anything that function calls into —
+    leaves the shim byte-identical, so without this a re-run is served from
+    the cache and answers with the old code's numbers.
+    """
+    first = fingerprint_of(store, a_flow(factor=3), code_digest="abc")
+
+    assert fingerprint_of(store, a_flow(factor=3), code_digest="def") != first
+    assert fingerprint_of(store, a_flow(factor=3), code_digest="abc") == first
+    # A canvas flow has no repository to hash, and keys the way it always did.
+    assert fingerprint_of(store, a_flow(factor=3)) == fingerprint_of(
+        store, a_flow(factor=3)
+    )
+
+
+def test_declaring_a_new_port_is_a_different_node(store: FlowStore):
+    """A hit restores what the node returned, so it must key on what it returns."""
+    first = fingerprint_of(store, a_flow(factor=3))
+    wider = a_flow(
+        provides=[
+            MessageSpec(name="scaled", dtype=DType.FLOAT),
+            MessageSpec(name="offset", dtype=DType.FLOAT),
+        ],
+        factor=3,
+    )
+
+    assert fingerprint_of(store, wider) != first
+
+
 def test_a_node_that_opted_out_carries_none(store: FlowStore):
     flow = a_flow(factor=3)
     flow.nodes[0].cache = False