stroblmeandClaude Opus 5
4a38c6ed31
Three faults with one root: the stored body of a code-defined node is an import shim, and nothing that mattered was ever read from the code itself. - The run stamp could not identify what ran. The shim imports whatever is on disk when the worker starts, and an uncommitted tree stamps <commit>-dirty for every run it ever produces. Run.code_digest hashes the repository's .py files, memoized on their stat state, and it is read again when the run is actually claimed -- so a sweep queued for hours records the code each of its runs executed, not the code that was there when it was submitted. - The stage cache adopted code that was too new. The fingerprint hashed the shim, which is invariant under any edit to the imported function or anything it calls into, so a re-run was served from cache and answered without the outputs the edit added. It now carries the repo digest and the node's declared ports. Every fingerprint changes once, which invalidates the existing cache; a canvas flow has no repository and keys as before. - An interrupted sync looked like a hand-edited canvas. The engine answers a new-node template for a node with no stored body, and the template carries no marker, so the drift check read "somebody edited this" and demanded --force -- for the one state that re-running the sync is the fix for. NodeSource.missing states the fact, and sync skips those and reuses the bodies it read instead of asking for each one twice. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
141 lines
4.8 KiB
Python
141 lines
4.8 KiB
Python
"""`fluksio sync` against a real engine: what it stores, and what it refuses."""
|
|
|
|
import sys
|
|
from collections.abc import Generator
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from fluksio.core.config import settings
|
|
from fluksio.sdk import FLOWS, MARKER, SyncError
|
|
from fluksio.sdk.client import Client, sync
|
|
|
|
EXAMPLES = Path(__file__).parents[4] / "examples"
|
|
PREFIX = settings.API_V1_STR
|
|
ORIGIN = {
|
|
"kind": "python",
|
|
"repo": str(EXAMPLES),
|
|
"commit": "a1b2c3d4",
|
|
"dirty": False,
|
|
}
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def flows() -> Generator[dict, None, None]:
|
|
"""The example research package, imported the way `sync` imports it."""
|
|
sys.path.insert(0, str(EXAMPLES))
|
|
FLOWS.clear()
|
|
import myresearch.pipeline # noqa: F401
|
|
|
|
yield dict(FLOWS)
|
|
FLOWS.clear()
|
|
sys.path.remove(str(EXAMPLES))
|
|
|
|
|
|
@pytest.fixture
|
|
def api(client: TestClient, superuser_token_headers: dict[str, str]) -> Client:
|
|
return Client(http=client, token=superuser_token_headers["Authorization"][7:])
|
|
|
|
|
|
def commits() -> int:
|
|
"""How many commits the flow store has made."""
|
|
from fluksio.flow.store import FlowStore
|
|
|
|
store = FlowStore(settings.FLOWS_DIR)
|
|
result = store._git("rev-list", "--count", "HEAD")
|
|
return int(result.stdout.strip()) if result.returncode == 0 else 0
|
|
|
|
|
|
def test_sync_stores_a_runnable_flow_with_its_origin(api, flows):
|
|
reports = sync([flows["train"]], api, origin=ORIGIN)
|
|
|
|
assert [r.flow for r in reports] == ["train"]
|
|
assert reports[0].created and reports[0].published
|
|
|
|
stored = api.get_flow("train")
|
|
definition = stored["definition"]
|
|
assert definition["origin"]["commit"] == "a1b2c3d4"
|
|
assert [n["id"] for n in definition["nodes"]] == ["prepare", "fit", "evaluate"]
|
|
assert not stored["has_draft"]
|
|
# The store holds a complete definition, and the bodies say where the real
|
|
# code is rather than being a copy of it.
|
|
source = api.get_source("train", "fit")
|
|
assert source.startswith(MARKER)
|
|
assert "from myresearch.train import fit" in source
|
|
|
|
|
|
def test_a_second_sync_changes_nothing_and_commits_nothing(api, flows):
|
|
sync([flows["train"]], api, origin=ORIGIN)
|
|
before = commits()
|
|
|
|
reports = sync([flows["train"]], api, origin=ORIGIN)
|
|
|
|
assert reports[0].unchanged
|
|
assert commits() == before
|
|
|
|
|
|
def test_sync_refuses_to_overwrite_a_canvas_edit(api, flows):
|
|
sync([flows["train"]], api, origin=ORIGIN)
|
|
api.put_source("train", "evaluate", "def process(weights):\n return {}\n")
|
|
|
|
with pytest.raises(SyncError, match="edited on the canvas"):
|
|
sync([flows["train"]], api, origin=ORIGIN)
|
|
|
|
reports = sync([flows["train"]], api, origin=ORIGIN, force=True)
|
|
assert "evaluate" in reports[0].changed
|
|
assert api.get_source("train", "evaluate").startswith(MARKER)
|
|
|
|
|
|
def test_an_interrupted_sync_finishes_on_the_next_one(api, flows):
|
|
"""A sync that died between the document and the bodies must be resumable.
|
|
|
|
That state — nodes named by a stored flow, with no source written for them
|
|
yet — used to read as a canvas edit, because the engine answers a new-node
|
|
template for a node with no body and the template carries no marker. The
|
|
advice was `--force`, for a canvas nobody had touched.
|
|
"""
|
|
target = flows["train"]
|
|
target.name = "resumed"
|
|
try:
|
|
# Exactly what the document PUT leaves behind before any body is written.
|
|
api.put_flow(target.document(ORIGIN) | {"version": 1})
|
|
assert api.get_source_entry("resumed", "fit")["missing"]
|
|
|
|
reports = sync([target], api, origin=ORIGIN)
|
|
|
|
assert "fit" in reports[0].changed
|
|
assert api.get_source("resumed", "fit").startswith(MARKER)
|
|
assert not api.get_source_entry("resumed", "fit")["missing"]
|
|
assert not api.get_flow("resumed")["has_draft"]
|
|
finally:
|
|
target.name = "train"
|
|
|
|
|
|
def test_sync_refuses_a_flow_it_did_not_create(api, flows):
|
|
api.put_flow({"name": "drawn", "version": 1, "nodes": [], "mode": "batch"})
|
|
api.publish("drawn", 1)
|
|
theirs = flows["train"]
|
|
theirs.name = "drawn"
|
|
|
|
try:
|
|
with pytest.raises(SyncError, match="not created by sync"):
|
|
sync([theirs], api, origin=ORIGIN)
|
|
finally:
|
|
theirs.name = "train"
|
|
|
|
|
|
def test_use_stores_the_rewiring_it_was_given(api, flows):
|
|
sync([flows["finetune"]], api, origin=ORIGIN)
|
|
|
|
nodes = {n["id"]: n for n in api.get_flow("finetune")["definition"]["nodes"]}
|
|
assert [p["name"] for p in nodes["fit"]["requires"]] == ["augmented", "lr"]
|
|
assert nodes["fit"]["params"] == {"epochs": 3}
|
|
|
|
|
|
def test_refresh_retires_the_workers(client, superuser_token_headers):
|
|
response = client.post(f"{PREFIX}/modules/refresh", headers=superuser_token_headers)
|
|
|
|
assert response.status_code == 200
|
|
assert "afresh" in response.json()["message"]
|