From 4a38c6ed31304de766d77146f42fe89804a36532 Mon Sep 17 00:00:00 2001 From: stroblme Date: Wed, 26 Aug 2026 21:27:10 +0200 Subject: [PATCH] Name the code a run ran, and let an interrupted sync finish Three faults with one root: the stored body of a code-defined node is an import shim, and nothing that mattered was ever read from the code itself. - The run stamp could not identify what ran. The shim imports whatever is on disk when the worker starts, and an uncommitted tree stamps -dirty for every run it ever produces. Run.code_digest hashes the repository's .py files, memoized on their stat state, and it is read again when the run is actually claimed -- so a sweep queued for hours records the code each of its runs executed, not the code that was there when it was submitted. - The stage cache adopted code that was too new. The fingerprint hashed the shim, which is invariant under any edit to the imported function or anything it calls into, so a re-run was served from cache and answered without the outputs the edit added. It now carries the repo digest and the node's declared ports. Every fingerprint changes once, which invalidates the existing cache; a canvas flow has no repository and keys as before. - An interrupted sync looked like a hand-edited canvas. The engine answers a new-node template for a node with no stored body, and the template carries no marker, so the drift check read "somebody edited this" and demanded --force -- for the one state that re-running the sync is the fix for. NodeSource.missing states the fact, and sync skips those and reuses the bodies it read instead of asking for each one twice. Co-Authored-By: Claude Opus 5 (1M context) --- .../versions/a4d9e2b71c68_run_code_digest.py | 38 ++++++++ backend/fluksio/api/routes/flows.py | 5 +- backend/fluksio/api/routes/runs.py | 4 + backend/fluksio/flow/controller.py | 27 +++++- backend/fluksio/flow/runs.py | 91 ++++++++++++++++++- backend/fluksio/flow/schemas.py | 4 + backend/fluksio/flow/store.py | 12 +++ backend/fluksio/models.py | 6 ++ backend/fluksio/sdk/cli.py | 16 +++- backend/fluksio/sdk/client.py | 47 ++++++++-- backend/tests/api/routes/test_sync.py | 25 +++++ backend/tests/flow/test_repo_digest.py | 91 +++++++++++++++++++ backend/tests/flow/test_stage_cache_build.py | 39 +++++++- docs/concepts/runs.md | 22 +++-- docs/getting-started/data-science.md | 23 +++-- 15 files changed, 422 insertions(+), 28 deletions(-) create mode 100644 backend/fluksio/alembic/versions/a4d9e2b71c68_run_code_digest.py create mode 100644 backend/tests/flow/test_repo_digest.py diff --git a/backend/fluksio/alembic/versions/a4d9e2b71c68_run_code_digest.py b/backend/fluksio/alembic/versions/a4d9e2b71c68_run_code_digest.py new file mode 100644 index 0000000..fad451f --- /dev/null +++ b/backend/fluksio/alembic/versions/a4d9e2b71c68_run_code_digest.py @@ -0,0 +1,38 @@ +"""run.code_digest + +`origin_commit` names a commit, and a dirty tree has the same one all day. The +node bodies are import shims, so what actually ran is whatever was on disk when +the worker started. This column hashes those files at run time, which is the +only thing that tells two runs of an uncommitted tree apart. + +Revision ID: a4d9e2b71c68 +Revises: f2c6a8d15e93 +Create Date: 2026-08-26 + +""" + +import sqlalchemy as sa +import sqlmodel.sql.sqltypes +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a4d9e2b71c68" +down_revision = "f2c6a8d15e93" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "run", + sa.Column( + "code_digest", + sqlmodel.sql.sqltypes.AutoString(length=64), + nullable=False, + server_default="", + ), + ) + + +def downgrade(): + op.drop_column("run", "code_digest") diff --git a/backend/fluksio/api/routes/flows.py b/backend/fluksio/api/routes/flows.py index ec72429..3d3be6e 100644 --- a/backend/fluksio/api/routes/flows.py +++ b/backend/fluksio/api/routes/flows.py @@ -529,7 +529,10 @@ def read_node_source( return NodeSource(code=controller.store.read_lib_source(ref)) except LibNotFound: raise HTTPException(status_code=404, detail=f"No shared node '{ref}'") - return NodeSource(code=controller.store.read_node_source(name, node_id, draft=True)) + return NodeSource( + code=controller.store.read_node_source(name, node_id, draft=True), + missing=not controller.store.has_node_source(name, node_id, draft=True), + ) @router.put("/{name}/nodes/{node_id}/source", response_model=NodeStatusPublic) diff --git a/backend/fluksio/api/routes/runs.py b/backend/fluksio/api/routes/runs.py index 3eb5c58..dc46000 100644 --- a/backend/fluksio/api/routes/runs.py +++ b/backend/fluksio/api/routes/runs.py @@ -107,6 +107,10 @@ class RunRow(BaseModel): #: decorators. Empty for one drawn on the canvas, where `commit` is the #: whole answer to what produced the number. origin_commit: str = "" + #: What that repository's python files hashed to when the run started. + #: Two runs of one dirty tree share a commit and differ here, which is the + #: only way to tell apart what they actually executed. + code_digest: str = "" #: The flow store's own commit. Short, unlike `result`, so the list #: carries it: "what code produced this" is a question asked of a table. commit: str = "" diff --git a/backend/fluksio/flow/controller.py b/backend/fluksio/flow/controller.py index 4883eb9..f321936 100644 --- a/backend/fluksio/flow/controller.py +++ b/backend/fluksio/flow/controller.py @@ -150,6 +150,10 @@ class RunContext: """ run_id: str + #: What the code-defined flow's repository hashed to when this run started. + #: Part of every node's fingerprint, so editing a function the node calls + #: into invalidates the cache the way editing the node itself does. + code_digest: str = "" @dataclass(frozen=True, slots=True) @@ -919,9 +923,30 @@ class FlowController: # The raw params, not the resolved ones: a secret's value # must not end up in a key, and its name is what changes # when the node is reconfigured anyway. + # + # `source` is the stored body, which for a code-defined + # flow is a shim that imports the real function — it does + # not move when that function does, and it says nothing + # about what the function calls into. `code` is what + # covers both: the digest of the repository the shim + # imports from. The ports are here because declaring a new + # one changes what the node answers with, which a hit + # would otherwise restore without. node.fingerprint = hashlib.sha256( json.dumps( - {"source": code, "params": node_def.params}, + { + "source": code, + "params": node_def.params, + "requires": [ + spec.model_dump(mode="json") + for spec in node_def.requires + ], + "provides": [ + spec.model_dump(mode="json") + for spec in node_def.provides + ], + "code": run.code_digest, + }, sort_keys=True, separators=(",", ":"), ).encode() diff --git a/backend/fluksio/flow/runs.py b/backend/fluksio/flow/runs.py index 2ab5827..a984bb5 100644 --- a/backend/fluksio/flow/runs.py +++ b/backend/fluksio/flow/runs.py @@ -38,6 +38,7 @@ import uuid from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta +from pathlib import Path from typing import Any from sqlalchemy import update @@ -409,6 +410,77 @@ def origin_commit(flow: FlowDef) -> str: return f"{origin.commit}-dirty" if origin.dirty else origin.commit +#: What a repository's python files hashed to, and the stat state that answer +#: was read from. Keyed by path: the walk is cheap, opening every file is not. +_digests: dict[str, tuple[tuple[tuple[str, int, int], ...], str]] = {} + +#: Directories a project's own code is never in, and which are large. +_SKIP_DIRS = frozenset({"__pycache__", "node_modules"}) + + +def repo_digest(repo: str) -> str: + """What the python files under ``repo`` currently hash to. + + A code-defined node is stored as a shim that imports the real function, so + the engine runs whatever is on disk when the worker starts. The commit + cannot name that: an uncommitted tree stamps every run `-dirty`, however + many times the code changed underneath. This can — it is read at run time, + from the files themselves. + + Empty when there is no repository to read here, which is the honest answer + on an engine that does not share a filesystem with the author: a stamp + without a digest says the engine could not see the code, and the cache key + falls back to what it keyed on before. + """ + if not repo or not os.path.isdir(repo): + return "" + state: list[tuple[str, int, int]] = [] + for root, dirs, files in os.walk(repo): + dirs[:] = [ + d + for d in dirs + if not d.startswith(".") + and d not in _SKIP_DIRS + # A virtualenv carries this, and its site-packages is not the + # project's code — nor is it small. + and not os.path.exists(os.path.join(root, d, "pyvenv.cfg")) + ] + for name in files: + if not name.endswith(".py"): + continue + path = os.path.join(root, name) + try: + info = os.stat(path) + except OSError: + continue + state.append((os.path.relpath(path, repo), info.st_mtime_ns, info.st_size)) + state.sort() + stamp = tuple(state) + cached = _digests.get(repo) + if cached is not None and cached[0] == stamp: + return cached[1] + + digest = hashlib.sha256() + for relative, _mtime, _size in state: + try: + body = Path(repo, relative).read_bytes() + except OSError: + continue + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(body) + digest.update(b"\0") + answer = digest.hexdigest() + _digests[repo] = (stamp, answer) + return answer + + +def code_digest(flow: FlowDef) -> str: + """The digest of the repository a code-defined flow was declared in.""" + origin = flow.origin + return repo_digest(origin.repo) if origin is not None else "" + + def _cacheable(outcome: NodeOutcome) -> str | None: """A node's outputs as stored, or None when it may not be reused.""" if not outcome.ok or not outcome.cache_key: @@ -579,6 +651,7 @@ class RunService: flow_version=flow.version, commit=self.controller.store.head(), origin_commit=origin_commit(flow), + code_digest=code_digest(flow), params=params, params_digest=digest_of(params, seed), seed=seed, @@ -803,6 +876,17 @@ class RunService: return None return session.exec(select(Run).where(col(Run.id) == run_id)).first() + def _restamp(self, run: Run, digest: str) -> str: + """Record the code this run is about to execute, if it moved.""" + if digest == run.code_digest: + return digest + with Session(db_engine) as session: + session.exec( + update(Run).where(col(Run.id) == run.id).values(code_digest=digest) + ) + session.commit() + return digest + def _drive(self, run_id: str) -> None: run = self._claim(run_id) if run is None: @@ -823,13 +907,18 @@ class RunService: sink = MetricSink(run_id) try: flow = self.controller.store.read_flow(run.flow) + # Read again, now that it is this run's turn: a sweep queues every + # run at once, and the code on disk is free to move in the hours + # before the last of them starts. What the record must name is the + # state that ran, not the state that was submitted. + run.code_digest = self._restamp(run, code_digest(flow)) state = self._state_factory(f"{RUN_NAMESPACE}:{run_id}") pipeline = self.controller.build_run_pipeline( flow, state=state, observer=observe, emission_observer=sink.handle, - run=RunContext(run_id=run_id), + run=RunContext(run_id=run_id, code_digest=run.code_digest), run_cache=None if run.no_cache else self._cache, ) with self._lock: diff --git a/backend/fluksio/flow/schemas.py b/backend/fluksio/flow/schemas.py index 78b3395..eace210 100644 --- a/backend/fluksio/flow/schemas.py +++ b/backend/fluksio/flow/schemas.py @@ -158,6 +158,10 @@ class NodeSource(BaseModel): """The Python source of a node.""" code: str + #: True when nothing is stored and `code` is the new-node template. An + #: editor opens on it either way; a client deciding whether somebody wrote + #: that code needs to know it was nobody. Read-only — set on the way out. + missing: bool = False Health = Literal["ok", "degraded", "down"] diff --git a/backend/fluksio/flow/store.py b/backend/fluksio/flow/store.py index 9e33da7..4f857d7 100644 --- a/backend/fluksio/flow/store.py +++ b/backend/fluksio/flow/store.py @@ -549,6 +549,18 @@ class FlowStore: path = self._node_file(flow, node_id) return path.read_text() if path.exists() else DEFAULT_SOURCE + def has_node_source(self, flow: str, node_id: str, draft: bool = False) -> bool: + """Whether a body was ever stored for this node. + + `read_node_source` answers a template when there is none, which is what + a new node's editor should open with — but it is a placeholder, not + code anybody wrote, and telling the two apart is the difference between + "someone edited this" and "this was never written". + """ + if draft and self._draft_node_file(flow, node_id).exists(): + return True + return self._node_file(flow, node_id).exists() + def write_node_source( self, flow: str, node_id: str, code: str, draft: bool = False ) -> bool: diff --git a/backend/fluksio/models.py b/backend/fluksio/models.py index 6116bd4..3f8e8d7 100644 --- a/backend/fluksio/models.py +++ b/backend/fluksio/models.py @@ -331,6 +331,12 @@ class Run(SQLModel, table=True): #: one names the code it imported. A `-dirty` suffix is git's own way of #: saying the tree had changes that are in no commit at all. origin_commit: str = Field(default="", max_length=80) + #: sha256 over that repository's python files, read when the run actually + #: started. The commit cannot tell two runs of an uncommitted tree apart — + #: both stamp `-dirty` — and the shim imports whatever is on disk, so this + #: is the half that names the code. Empty when the engine cannot see the + #: repository, or for a flow drawn on the canvas. + code_digest: str = Field(default="", max_length=64) params: dict[str, Any] = Field(sa_column=Column(JSON), default_factory=dict) #: sha256 over params and seed. Two runs of the same thing share it, which #: is what makes "have I already run this?" a lookup. diff --git a/backend/fluksio/sdk/cli.py b/backend/fluksio/sdk/cli.py index 4254477..4a00537 100644 --- a/backend/fluksio/sdk/cli.py +++ b/backend/fluksio/sdk/cli.py @@ -695,6 +695,19 @@ def cmd_status(args: argparse.Namespace) -> int: PARAMS_WIDTH = 80 +def _stamp(row: dict[str, Any]) -> str: + """What code a run ran: the commit, whether it was dirty, and the digest. + + The digest is the part that separates two runs of one uncommitted tree — + the commit says `-dirty` for both, however much changed in between. + """ + commit, dirty, _ = (row.get("origin_commit") or "").partition("-dirty") + digest = row.get("code_digest") or "" + return ( + commit[:7] + ("-dirty" if dirty else "") + (f"+{digest[:7]}" if digest else "") + ) + + def cmd_runs(args: argparse.Namespace) -> int: try: with _client_for(args) as client: @@ -704,13 +717,12 @@ def cmd_runs(args: argparse.Namespace) -> int: except httpx.HTTPError as exc: return _unreachable(exc) for row in rows: - commit = (row.get("origin_commit") or "")[:7] params = json.dumps(row["params"]) if len(params) > PARAMS_WIDTH: params = params[: PARAMS_WIDTH - 3] + "..." _say( f"{row['id']} {_status(row['status'], 9)} {row['flow']:<16} " - f"{row['duration_ms'] / 1000:7.1f}s {commit:<8} {params}" + f"{row['duration_ms'] / 1000:7.1f}s {_stamp(row):<22} {params}" ) return 0 diff --git a/backend/fluksio/sdk/client.py b/backend/fluksio/sdk/client.py index 357c676..138097c 100644 --- a/backend/fluksio/sdk/client.py +++ b/backend/fluksio/sdk/client.py @@ -237,8 +237,17 @@ class Client: return result def get_source(self, flow: str, node: str) -> str: - result = self._call("GET", f"/flows/{flow}/nodes/{node}/source") - return str(result.get("code", "")) + return str(self.get_source_entry(flow, node).get("code", "")) + + def get_source_entry(self, flow: str, node: str) -> dict[str, Any]: + """A node's stored body, and whether there is one at all. + + `missing` is what separates a node nobody has written from a node + somebody edited: the engine answers a new-node template for both, and + only one of them is work worth refusing to overwrite. + """ + result: dict[str, Any] = self._call("GET", f"/flows/{flow}/nodes/{node}/source") + return result or {} def put_source(self, flow: str, node: str, code: str) -> dict[str, Any]: result: dict[str, Any] = self._call( @@ -623,13 +632,16 @@ def _sync_one( report = SyncReport(target.name) stored = client.get_flow(target.name) version = 1 + # What the drift check already read, so the write loop below does not ask + # the engine for every node's body a second time. + known: dict[str, str] = {} if stored is None: report.created = True else: definition = stored.get("definition") or {} version = int(definition.get("version") or 1) if not force: - _refuse_on_drift(client, target, definition) + known = _refuse_on_drift(client, target, definition) saved = client.put_flow(target.document(origin) | {"version": version}) stored_version = int((saved.get("definition") or {}).get("version") or version) @@ -640,8 +652,12 @@ def _sync_one( version = stored_version for node_id, code in target.shims().items(): - if not report.created and client.get_source(target.name, node_id) == code: - continue + if not report.created: + stored_code = known.get(node_id) + if stored_code is None: + stored_code = client.get_source(target.name, node_id) + if stored_code == code: + continue client.put_source(target.name, node_id, code) report.changed.append(node_id) @@ -651,26 +667,43 @@ def _sync_one( return report -def _refuse_on_drift(client: Client, target: Flow, definition: dict[str, Any]) -> None: +def _refuse_on_drift( + client: Client, target: Flow, definition: dict[str, Any] +) -> dict[str, str]: """Stop before overwriting work that was done somewhere else. Two ways a stored flow is not ours to replace: it was drawn on the canvas and has no origin at all, or one of its node bodies no longer carries the line saying it was generated — which means somebody edited the code there. + + A node with *no* stored body is neither. That is what a sync interrupted + part of the way through leaves, and it used to read as a canvas edit, + because the engine answers the new-node template for it and the template + carries no marker either. Re-running the sync is the fix for that state, + so refusing to is precisely wrong. + + Returns the bodies it read, so the caller can compare against them without + asking again. """ if not definition.get("origin"): raise SyncError( f"flow '{target.name}' was not created by sync, so replacing it would " "discard whoever drew it. Rename yours, or pass --force." ) + seen: dict[str, str] = {} for stored_node in definition.get("nodes") or []: node_id = str(stored_node.get("id")) try: - code = client.get_source(target.name, node_id) + entry = client.get_source_entry(target.name, node_id) except ApiError: continue + if entry.get("missing"): + continue + code = str(entry.get("code", "")) + seen[node_id] = code if code and not code.startswith(MARKER): raise SyncError( f"node '{target.name}.{node_id}' was edited on the canvas, and " "syncing would throw that edit away. Copy it out, or pass --force." ) + return seen diff --git a/backend/tests/api/routes/test_sync.py b/backend/tests/api/routes/test_sync.py index fef81bf..8431561 100644 --- a/backend/tests/api/routes/test_sync.py +++ b/backend/tests/api/routes/test_sync.py @@ -87,6 +87,31 @@ def test_sync_refuses_to_overwrite_a_canvas_edit(api, flows): assert api.get_source("train", "evaluate").startswith(MARKER) +def test_an_interrupted_sync_finishes_on_the_next_one(api, flows): + """A sync that died between the document and the bodies must be resumable. + + That state — nodes named by a stored flow, with no source written for them + yet — used to read as a canvas edit, because the engine answers a new-node + template for a node with no body and the template carries no marker. The + advice was `--force`, for a canvas nobody had touched. + """ + target = flows["train"] + target.name = "resumed" + try: + # Exactly what the document PUT leaves behind before any body is written. + api.put_flow(target.document(ORIGIN) | {"version": 1}) + assert api.get_source_entry("resumed", "fit")["missing"] + + reports = sync([target], api, origin=ORIGIN) + + assert "fit" in reports[0].changed + assert api.get_source("resumed", "fit").startswith(MARKER) + assert not api.get_source_entry("resumed", "fit")["missing"] + assert not api.get_flow("resumed")["has_draft"] + finally: + target.name = "train" + + def test_sync_refuses_a_flow_it_did_not_create(api, flows): api.put_flow({"name": "drawn", "version": 1, "nodes": [], "mode": "batch"}) api.publish("drawn", 1) diff --git a/backend/tests/flow/test_repo_digest.py b/backend/tests/flow/test_repo_digest.py new file mode 100644 index 0000000..4db7f37 --- /dev/null +++ b/backend/tests/flow/test_repo_digest.py @@ -0,0 +1,91 @@ +"""What the engine hashes to decide a repository's code moved. + +The commit cannot answer this: an uncommitted tree stamps `-dirty` for every +run it ever produces, and the node bodies are import shims that pick up +whatever is on disk when the worker starts. +""" + +from fluksio.flow import runs +from fluksio.flow.runs import code_digest, repo_digest +from fluksio.flow.schemas import FlowDef, FlowOrigin + + +def a_repo(tmp_path, body="x = 1\n"): + (tmp_path / "pkg").mkdir() + (tmp_path / "pkg" / "train.py").write_text(body) + return str(tmp_path) + + +def touch(path, body, mtime): + """Write and stamp: two writes inside one clock tick look identical.""" + path.write_text(body) + import os + + os.utime(path, ns=(mtime, mtime)) + + +def test_the_same_tree_hashes_the_same(tmp_path): + repo = a_repo(tmp_path) + assert repo_digest(repo) == repo_digest(repo) + + +def test_editing_a_file_changes_it(tmp_path): + repo = a_repo(tmp_path) + before = repo_digest(repo) + touch(tmp_path / "pkg" / "train.py", "x = 2\n", 2_000_000_000_000_000_000) + + assert repo_digest(repo) != before + + +def test_a_new_file_changes_it(tmp_path): + repo = a_repo(tmp_path) + before = repo_digest(repo) + (tmp_path / "pkg" / "helper.py").write_text("y = 1\n") + + assert repo_digest(repo) != before + + +def test_what_is_not_the_project_is_left_out(tmp_path): + """A venv is bigger than the project and is not its code.""" + repo = a_repo(tmp_path) + before = repo_digest(repo) + + (tmp_path / ".venv" / "lib").mkdir(parents=True) + (tmp_path / ".venv" / "pyvenv.cfg").write_text("home = /usr\n") + (tmp_path / ".venv" / "lib" / "numpy.py").write_text("huge = True\n") + (tmp_path / "pkg" / "__pycache__").mkdir() + (tmp_path / "pkg" / "__pycache__" / "train.py").write_text("compiled\n") + (tmp_path / ".git").mkdir() + (tmp_path / ".git" / "hook.py").write_text("hook\n") + (tmp_path / "pkg" / "notes.md").write_text("prose\n") + + assert repo_digest(repo) == before + + +def test_a_repository_the_engine_cannot_see_is_empty(tmp_path): + """Not an error: a worker elsewhere is a normal deployment.""" + assert repo_digest(str(tmp_path / "nowhere")) == "" + assert repo_digest("") == "" + + +def test_only_a_code_defined_flow_has_one(tmp_path): + """A flow drawn on the canvas has no repository, and keys as it always did.""" + repo = a_repo(tmp_path) + declared = FlowDef( + name="train", mode="batch", origin=FlowOrigin(repo=repo, commit="abc") + ) + + assert code_digest(declared) == repo_digest(repo) + assert code_digest(FlowDef(name="drawn", mode="batch")) == "" + + +def test_unchanged_files_are_not_read_again(tmp_path, monkeypatch): + """The warm path is a stat per file, which is what makes this per-submit.""" + repo = a_repo(tmp_path) + repo_digest(repo) + + def refuse(*_args, **_kwargs): + raise AssertionError("an unchanged tree must not be re-read") + + monkeypatch.setattr(runs.Path, "read_bytes", refuse) + assert repo_digest(repo) diff --git a/backend/tests/flow/test_stage_cache_build.py b/backend/tests/flow/test_stage_cache_build.py index 2d036ce..a54dfc1 100644 --- a/backend/tests/flow/test_stage_cache_build.py +++ b/backend/tests/flow/test_stage_cache_build.py @@ -18,7 +18,7 @@ from fluksio.flow.store import FlowStore SOURCE = "def process(reading, factor):\n return {'scaled': reading * factor}\n" -def a_flow(**params: object) -> FlowDef: +def a_flow(provides: list[MessageSpec] | None = None, **params: object) -> FlowDef: return FlowDef( name="house", mode="batch", @@ -27,7 +27,7 @@ def a_flow(**params: object) -> FlowDef: id="scale", params=dict(params), requires=[MessageSpec(name="reading", dtype=DType.FLOAT)], - provides=[MessageSpec(name="scaled", dtype=DType.FLOAT)], + provides=provides or [MessageSpec(name="scaled", dtype=DType.FLOAT)], ) ], ) @@ -38,13 +38,13 @@ def store(tmp_path: Path) -> FlowStore: return FlowStore(tmp_path / "flows") -def fingerprint_of(store: FlowStore, flow: FlowDef) -> str: +def fingerprint_of(store: FlowStore, flow: FlowDef, code_digest: str = "") -> str: store.write_flow(flow) store.write_node_source(flow.name, "scale", SOURCE) pipeline = FlowController(store).build_run_pipeline( store.read_flow(flow.name), state=MemoryState(), - run=RunContext(run_id="r-1"), + run=RunContext(run_id="r-1", code_digest=code_digest), ) return pipeline.nodes[0].fingerprint @@ -66,6 +66,37 @@ def test_a_source_change_is_a_different_node(store: FlowStore): assert pipeline.nodes[0].fingerprint != first +def test_moving_the_repository_is_a_different_node(store: FlowStore): + """The stored body of a code-defined node is a shim, and it never moves. + + Editing the function it imports — or anything that function calls into — + leaves the shim byte-identical, so without this a re-run is served from + the cache and answers with the old code's numbers. + """ + first = fingerprint_of(store, a_flow(factor=3), code_digest="abc") + + assert fingerprint_of(store, a_flow(factor=3), code_digest="def") != first + assert fingerprint_of(store, a_flow(factor=3), code_digest="abc") == first + # A canvas flow has no repository to hash, and keys the way it always did. + assert fingerprint_of(store, a_flow(factor=3)) == fingerprint_of( + store, a_flow(factor=3) + ) + + +def test_declaring_a_new_port_is_a_different_node(store: FlowStore): + """A hit restores what the node returned, so it must key on what it returns.""" + first = fingerprint_of(store, a_flow(factor=3)) + wider = a_flow( + provides=[ + MessageSpec(name="scaled", dtype=DType.FLOAT), + MessageSpec(name="offset", dtype=DType.FLOAT), + ], + factor=3, + ) + + assert fingerprint_of(store, wider) != first + + def test_a_node_that_opted_out_carries_none(store: FlowStore): flow = a_flow(factor=3) flow.nodes[0].cache = False diff --git a/docs/concepts/runs.md b/docs/concepts/runs.md index a2fd80e..0bb0dc2 100644 --- a/docs/concepts/runs.md +++ b/docs/concepts/runs.md @@ -177,18 +177,28 @@ downloadable at `GET /api/v1/artifacts/{digest}`. ## Stage caching A run mostly does not redo what an earlier one already did. Before a node -executes it is fingerprinted — a sha256 over its source, its settings and the -values it is about to read — and if some earlier run of that same fingerprint -finished, what that one returned is restored into this run's state and the node -is skipped. It is recorded with the status `cached` and a duration of zero, and -its artifacts are listed on the new run as well, so they stay downloadable from -either. +executes it is fingerprinted — a sha256 over its source, its settings, the ports +it declares and the values it is about to read — and if some earlier run of that +same fingerprint finished, what that one returned is restored into this run's +state and the node is skipped. It is recorded with the status `cached` and a +duration of zero, and its artifacts are listed on the new run as well, so they +stay downloadable from either. The settings go into the key raw, so a secret contributes its `{"$secret": name}` reference and never its value. An artifact input counts as its content digest: the same bytes under a different filename are the same input. Each node on a run carries the `cache_key` it was looked up by. +For a [code-defined flow](../getting-started/data-science.md), "its source" is +the generated shim, which imports the real function and does not change when +that function does. So the key carries one thing more: a digest of every +`.py` file under the repository the flow was declared in, read when the run +starts. Editing a helper three calls down from the node invalidates it, which +is the point — the alternative is a re-run answering with the previous code's +numbers. It is deliberately blunt: any edit anywhere in the repository re-runs +every node of its flows. An engine that cannot see the repository — a worker on +another machine — records no digest and keys as it did before. + The run history *is* the cache; there is no second store. A node's returned outputs are kept on its run record as canonical JSON, up to 256000 characters — a node returning more than that is simply not cacheable that run. An entry diff --git a/docs/getting-started/data-science.md b/docs/getting-started/data-science.md index 6f94dfa..876ca13 100644 --- a/docs/getting-started/data-science.md +++ b/docs/getting-started/data-science.md @@ -416,12 +416,15 @@ already up, so it is for the run you want now rather than the loop you are iterating in. A second run of a flow you did not change mostly does not execute. Each of -your nodes is fingerprinted by its source, its settings and the values it -reads, and one an earlier run already answered is restored from that run -rather than run again — reported as `cached`, so a flow with nothing left to -do finishes as `(3/3 cached)`. Change `--lr` and only the nodes downstream of -it run. `--no-cache` turns that off for one run, `@node(..., cache=False)` for -one node; the caveat and the details are in +your nodes is fingerprinted by its settings, the ports it declares, the values +it reads, and a digest of your repository's Python files — so editing anything +your node calls into invalidates it, not only the decorated function itself. +One an earlier run already answered is restored from that run rather than run +again — reported as `cached`, so a flow with nothing left to do finishes as +`(3/3 cached)`. Change `--lr` and only the nodes downstream of it run. The +digest is blunt on purpose: an edit anywhere in the repository re-runs +everything. `--no-cache` turns caching off for one run, `@node(..., +cache=False)` for one node; the caveat and the details are in [Stage caching](../concepts/runs.md#stage-caching). From Python, the flow you declared is also the handle to its runs: @@ -452,6 +455,14 @@ commits — the flow store's and your repository's. That is the answer to "what was the learning rate on the run that got 94%?". When a run failed, `run.failures` is the node that did it, with its traceback and its logs. +Beside your commit is `code_digest`, and `fluksio runs` prints the pair as +`a1b2c3d-dirty+9f0e1a2`. The commit alone cannot identify what ran: your node +bodies are imports, so the engine executes whatever is on disk when the worker +starts, and an uncommitted tree stamps `-dirty` for every run it ever produces. +The digest is read at the moment the run starts — so in a sweep whose runs +queue for hours, each one records the code that actually executed it, not the +code that was there when you submitted. + ### When the engine is busy A driver script outlives the engine being slow, because a sweep is hours long