stroblmeandClaude Opus 5
4a38c6ed31
Three faults with one root: the stored body of a code-defined node is an import shim, and nothing that mattered was ever read from the code itself. - The run stamp could not identify what ran. The shim imports whatever is on disk when the worker starts, and an uncommitted tree stamps <commit>-dirty for every run it ever produces. Run.code_digest hashes the repository's .py files, memoized on their stat state, and it is read again when the run is actually claimed -- so a sweep queued for hours records the code each of its runs executed, not the code that was there when it was submitted. - The stage cache adopted code that was too new. The fingerprint hashed the shim, which is invariant under any edit to the imported function or anything it calls into, so a re-run was served from cache and answered without the outputs the edit added. It now carries the repo digest and the node's declared ports. Every fingerprint changes once, which invalidates the existing cache; a canvas flow has no repository and keys as before. - An interrupted sync looked like a hand-edited canvas. The engine answers a new-node template for a node with no stored body, and the template carries no marker, so the drift check read "somebody edited this" and demanded --force -- for the one state that re-running the sync is the fix for. NodeSource.missing states the fact, and sync skips those and reuses the bodies it read instead of asking for each one twice. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
92 lines
2.9 KiB
Python
92 lines
2.9 KiB
Python
"""What the engine hashes to decide a repository's code moved.
|
|
|
|
The commit cannot answer this: an uncommitted tree stamps `-dirty` for every
|
|
run it ever produces, and the node bodies are import shims that pick up
|
|
whatever is on disk when the worker starts.
|
|
"""
|
|
|
|
from fluksio.flow import runs
|
|
from fluksio.flow.runs import code_digest, repo_digest
|
|
from fluksio.flow.schemas import FlowDef, FlowOrigin
|
|
|
|
|
|
def a_repo(tmp_path, body="x = 1\n"):
|
|
(tmp_path / "pkg").mkdir()
|
|
(tmp_path / "pkg" / "train.py").write_text(body)
|
|
return str(tmp_path)
|
|
|
|
|
|
def touch(path, body, mtime):
|
|
"""Write and stamp: two writes inside one clock tick look identical."""
|
|
path.write_text(body)
|
|
import os
|
|
|
|
os.utime(path, ns=(mtime, mtime))
|
|
|
|
|
|
def test_the_same_tree_hashes_the_same(tmp_path):
|
|
repo = a_repo(tmp_path)
|
|
assert repo_digest(repo) == repo_digest(repo)
|
|
|
|
|
|
def test_editing_a_file_changes_it(tmp_path):
|
|
repo = a_repo(tmp_path)
|
|
before = repo_digest(repo)
|
|
touch(tmp_path / "pkg" / "train.py", "x = 2\n", 2_000_000_000_000_000_000)
|
|
|
|
assert repo_digest(repo) != before
|
|
|
|
|
|
def test_a_new_file_changes_it(tmp_path):
|
|
repo = a_repo(tmp_path)
|
|
before = repo_digest(repo)
|
|
(tmp_path / "pkg" / "helper.py").write_text("y = 1\n")
|
|
|
|
assert repo_digest(repo) != before
|
|
|
|
|
|
def test_what_is_not_the_project_is_left_out(tmp_path):
|
|
"""A venv is bigger than the project and is not its code."""
|
|
repo = a_repo(tmp_path)
|
|
before = repo_digest(repo)
|
|
|
|
(tmp_path / ".venv" / "lib").mkdir(parents=True)
|
|
(tmp_path / ".venv" / "pyvenv.cfg").write_text("home = /usr\n")
|
|
(tmp_path / ".venv" / "lib" / "numpy.py").write_text("huge = True\n")
|
|
(tmp_path / "pkg" / "__pycache__").mkdir()
|
|
(tmp_path / "pkg" / "__pycache__" / "train.py").write_text("compiled\n")
|
|
(tmp_path / ".git").mkdir()
|
|
(tmp_path / ".git" / "hook.py").write_text("hook\n")
|
|
(tmp_path / "pkg" / "notes.md").write_text("prose\n")
|
|
|
|
assert repo_digest(repo) == before
|
|
|
|
|
|
def test_a_repository_the_engine_cannot_see_is_empty(tmp_path):
|
|
"""Not an error: a worker elsewhere is a normal deployment."""
|
|
assert repo_digest(str(tmp_path / "nowhere")) == ""
|
|
assert repo_digest("") == ""
|
|
|
|
|
|
def test_only_a_code_defined_flow_has_one(tmp_path):
|
|
"""A flow drawn on the canvas has no repository, and keys as it always did."""
|
|
repo = a_repo(tmp_path)
|
|
declared = FlowDef(
|
|
name="train", mode="batch", origin=FlowOrigin(repo=repo, commit="abc")
|
|
)
|
|
|
|
assert code_digest(declared) == repo_digest(repo)
|
|
assert code_digest(FlowDef(name="drawn", mode="batch")) == ""
|
|
|
|
|
|
def test_unchanged_files_are_not_read_again(tmp_path, monkeypatch):
|
|
"""The warm path is a stat per file, which is what makes this per-submit."""
|
|
repo = a_repo(tmp_path)
|
|
repo_digest(repo)
|
|
|
|
def refuse(*_args, **_kwargs):
|
|
raise AssertionError("an unchanged tree must not be re-read")
|
|
|
|
monkeypatch.setattr(runs.Path, "read_bytes", refuse)
|
|
assert repo_digest(repo)
|