Key a node on the code it reaches, not on the repository around it
Docs / docs (push) Successful in 21s
Playwright Tests / test-playwright (1, 2) (push) Successful in 2m13s
Playwright Tests / test-playwright (2, 2) (push) Successful in 1m52s
pre-commit / pre-commit (push) Failing after 2m6s
Test Backend / test-backend (push) Successful in 2m30s
Compose Smoke Test / test-compose (push) Successful in 32s
Playwright Tests / merge-reports (push) Successful in 1m53s
Docs / docs (push) Successful in 21s
Playwright Tests / test-playwright (1, 2) (push) Successful in 2m13s
Playwright Tests / test-playwright (2, 2) (push) Successful in 1m52s
pre-commit / pre-commit (push) Failing after 2m6s
Test Backend / test-backend (push) Successful in 2m30s
Compose Smoke Test / test-compose (push) Successful in 32s
Playwright Tests / merge-reports (push) Successful in 1m53s
`sync` follows each node function's imports through the project's own modules — stopping at the standard library, at anything installed, and at Fluksio itself, whose checkout would otherwise be most of every digest — and records the file list with what it hashed to. The engine hashes those files again when the run is claimed, so the fingerprint is live rather than a snapshot, and falls back to what sync recorded when it cannot see them: a remote worker's runs used to share one empty digest, and therefore one key. Three things follow. Editing a helper a node calls into re-runs that node, as before. Editing something the node never reaches no longer re-runs anything — a notebook two directories away was invalidating every arm. And `Run.code_digest` is now the hash of its nodes' digests, so it is neither looser nor tighter than "the code behind these numbers", which is what makes it worth joining an exported table on. `sync` says so too: it compares the per-node digest against the stored one, so a helper edit prints `train: updated (flow, fit)` instead of `unchanged`. The digest is read when the document is built rather than when the flow is declared, so a second `sync()` in one process sees an edit between them. Also: `fluksio runs` shows only the inputs that differ from what the flow declares, fitted to the terminal, so a flow taking a few kB of json no longer wraps every line. Every existing cache entry misses once — the fingerprint changed shape. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01A9Hdrmf2cwNABCnE5x9UJa
This commit is contained in:
@@ -65,6 +65,45 @@ def test_sync_stores_a_runnable_flow_with_its_origin(api, flows):
|
||||
assert "from myresearch.train import fit" in source
|
||||
|
||||
|
||||
def test_a_node_carries_the_modules_its_code_reaches(api, flows):
|
||||
"""The shim names one function; the cache has to key on what it calls."""
|
||||
sync([flows["train"]], api, origin=ORIGIN)
|
||||
|
||||
nodes = {n["id"]: n for n in api.get_flow("train")["definition"]["nodes"]}
|
||||
files = nodes["fit"]["code_files"]
|
||||
|
||||
# The module the function is in, and the package whose `__init__` runs on
|
||||
# the way to it. Not the engine's own code, and not the standard library.
|
||||
assert "myresearch.train" in files
|
||||
assert all(path.endswith(".py") for path in files.values())
|
||||
assert not any(name.startswith("fluksio") for name in files)
|
||||
assert len(nodes["fit"]["code_digest"]) == 64
|
||||
|
||||
|
||||
def test_a_node_whose_helper_moved_is_named_by_sync():
|
||||
"""Editing a helper changes no shim, so nothing used to say it happened."""
|
||||
from fluksio.sdk.client import _moved_code
|
||||
|
||||
stored = {
|
||||
"definition": {
|
||||
"nodes": [
|
||||
{"id": "fit", "code_digest": "aaa"},
|
||||
{"id": "prepare", "code_digest": "bbb"},
|
||||
]
|
||||
}
|
||||
}
|
||||
document = {
|
||||
"nodes": [
|
||||
{"id": "fit", "code_digest": "ccc"},
|
||||
{"id": "prepare", "code_digest": "bbb"},
|
||||
]
|
||||
}
|
||||
|
||||
assert _moved_code(stored, document) == ["fit"]
|
||||
# Nothing to compare against is not a change.
|
||||
assert _moved_code(None, document) == []
|
||||
|
||||
|
||||
def test_a_second_sync_changes_nothing_and_commits_nothing(api, flows):
|
||||
sync([flows["train"]], api, origin=ORIGIN)
|
||||
before = commits()
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
"""What a node's code reaches — the narrow answer a cache key is keyed on."""
|
||||
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
from fluksio.sdk.imports import digest_of, reached
|
||||
|
||||
|
||||
def _write(root, name, body):
|
||||
path = root / f"{name}.py"
|
||||
path.write_text(textwrap.dedent(body).lstrip())
|
||||
return path
|
||||
|
||||
|
||||
def _study(tmp_path):
|
||||
"""A package whose node function delegates to a helper beside it."""
|
||||
root = tmp_path / "study"
|
||||
root.mkdir()
|
||||
(root / "__init__.py").write_text("VERSION = 1\n")
|
||||
_write(
|
||||
root,
|
||||
"train",
|
||||
"""
|
||||
import json
|
||||
|
||||
from study.helpers import curve
|
||||
|
||||
|
||||
def fit(lr):
|
||||
return json.dumps(curve(lr))
|
||||
""",
|
||||
)
|
||||
_write(root, "helpers", "def curve(lr):\n return [lr]\n")
|
||||
_write(root, "unused", "def nothing():\n return 0\n")
|
||||
return root
|
||||
|
||||
|
||||
def test_the_walk_follows_a_helper_and_stops_at_installed_code(tmp_path, monkeypatch):
|
||||
"""The shim says nothing about this, and the repository says too much."""
|
||||
_study(tmp_path)
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
for name in [n for n in sys.modules if n == "study" or n.startswith("study.")]:
|
||||
del sys.modules[name]
|
||||
|
||||
from study.train import fit
|
||||
|
||||
files = reached(fit)
|
||||
|
||||
# The module, the helper it calls into, and the package whose `__init__`
|
||||
# runs on the way in. Not the sibling nobody imports, and not `json`.
|
||||
assert set(files) == {"study", "study.train", "study.helpers"}
|
||||
assert all(path.endswith(".py") for path in files.values())
|
||||
|
||||
|
||||
def test_the_digest_moves_with_the_helper_and_not_with_the_neighbour(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
"""The whole point: it is neither necessary nor sufficient otherwise."""
|
||||
root = _study(tmp_path)
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
for name in [n for n in sys.modules if n == "study" or n.startswith("study.")]:
|
||||
del sys.modules[name]
|
||||
|
||||
from study.train import fit
|
||||
|
||||
files = reached(fit)
|
||||
before = digest_of(files)
|
||||
|
||||
_write(root, "unused", "def nothing():\n return 1\n")
|
||||
assert digest_of(files) == before
|
||||
|
||||
_write(root, "helpers", "def curve(lr):\n return [lr, lr]\n")
|
||||
assert digest_of(files) != before
|
||||
|
||||
# A file this engine cannot see is not half an answer.
|
||||
assert digest_of({**files, "study.gone": str(root / "gone.py")}) == ""
|
||||
|
||||
|
||||
def test_the_tool_itself_is_not_the_projects_code(tmp_path, monkeypatch):
|
||||
"""Every node imports the authoring API, and from a checkout that is source."""
|
||||
root = tmp_path / "arm"
|
||||
root.mkdir()
|
||||
(root / "__init__.py").write_text("")
|
||||
_write(
|
||||
root,
|
||||
"work",
|
||||
"""
|
||||
from fluksio import node
|
||||
|
||||
|
||||
def fit(lr):
|
||||
return lr
|
||||
""",
|
||||
)
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
for name in [n for n in sys.modules if n == "arm" or n.startswith("arm.")]:
|
||||
del sys.modules[name]
|
||||
|
||||
from arm.work import fit
|
||||
|
||||
assert set(reached(fit)) == {"arm", "arm.work"}
|
||||
Reference in New Issue
Block a user