stroblmeandClaude Fable 5
a38e2745eb
A data scientist keeps their code where it is and decorates it: `@node` declares a function's ports beside the function, `Flow(name, nodes=[...])` says which of them make a flow, and `use(fn, wire=..., **settings)` rebinds one for a single flow. `fluksio sync` uploads the document plus a generated import shim per node, so the store still holds a complete, runnable, git-versioned definition while the code it imports stays theirs. `fluksio login|run|runs` and `flow.submit().wait()` are the client half, over the run endpoints that already existed. Runs record the user repository's commit beside the store's, so "what code produced this number" is answerable on the side that now holds the code. - `fluksio/sdk/`: ports, decorators, the flow builder and its checks, the shim generator, an HTTP client and sync. Standard library only at import, so `from fluksio import node` in a training script pulls in no engine. - `FlowDef.origin` marks a flow code-defined; `Run.origin_commit` carries the repository's commit; `POST /modules/refresh` retires the workers without an install, which every sync calls — a worker holds the imported package in memory, so an edit to it is invisible until the process goes. - The canvas shows a generated body read-only and names the repository to edit instead; a body edited there stops the next sync rather than being discarded. - The worker's reporter carries inert `Port`, `node`, `use` and `Flow`, since the shim imports a module whose first line declares them. - `examples/myresearch` is the worked example, `make sync-example` uploads it. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012ue1tkFWB1bcGy3aWhCKpU
116 lines
3.7 KiB
Python
116 lines
3.7 KiB
Python
"""`fluksio sync` against a real engine: what it stores, and what it refuses."""
|
|
|
|
import sys
|
|
from collections.abc import Generator
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from fluksio.core.config import settings
|
|
from fluksio.sdk import FLOWS, MARKER, SyncError
|
|
from fluksio.sdk.client import Client, sync
|
|
|
|
EXAMPLES = Path(__file__).parents[4] / "examples"
|
|
PREFIX = settings.API_V1_STR
|
|
ORIGIN = {
|
|
"kind": "python",
|
|
"repo": str(EXAMPLES),
|
|
"commit": "a1b2c3d4",
|
|
"dirty": False,
|
|
}
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def flows() -> Generator[dict, None, None]:
|
|
"""The example research package, imported the way `sync` imports it."""
|
|
sys.path.insert(0, str(EXAMPLES))
|
|
FLOWS.clear()
|
|
import myresearch.pipeline # noqa: F401
|
|
|
|
yield dict(FLOWS)
|
|
FLOWS.clear()
|
|
sys.path.remove(str(EXAMPLES))
|
|
|
|
|
|
@pytest.fixture
|
|
def api(client: TestClient, superuser_token_headers: dict[str, str]) -> Client:
|
|
return Client(http=client, token=superuser_token_headers["Authorization"][7:])
|
|
|
|
|
|
def commits() -> int:
|
|
"""How many commits the flow store has made."""
|
|
from fluksio.flow.store import FlowStore
|
|
|
|
store = FlowStore(settings.FLOWS_DIR)
|
|
result = store._git("rev-list", "--count", "HEAD")
|
|
return int(result.stdout.strip()) if result.returncode == 0 else 0
|
|
|
|
|
|
def test_sync_stores_a_runnable_flow_with_its_origin(api, flows):
|
|
reports = sync([flows["train"]], api, origin=ORIGIN)
|
|
|
|
assert [r.flow for r in reports] == ["train"]
|
|
assert reports[0].created and reports[0].published
|
|
|
|
stored = api.get_flow("train")
|
|
definition = stored["definition"]
|
|
assert definition["origin"]["commit"] == "a1b2c3d4"
|
|
assert [n["id"] for n in definition["nodes"]] == ["prepare", "fit", "evaluate"]
|
|
assert not stored["has_draft"]
|
|
# The store holds a complete definition, and the bodies say where the real
|
|
# code is rather than being a copy of it.
|
|
source = api.get_source("train", "fit")
|
|
assert source.startswith(MARKER)
|
|
assert "from myresearch.train import fit" in source
|
|
|
|
|
|
def test_a_second_sync_changes_nothing_and_commits_nothing(api, flows):
|
|
sync([flows["train"]], api, origin=ORIGIN)
|
|
before = commits()
|
|
|
|
reports = sync([flows["train"]], api, origin=ORIGIN)
|
|
|
|
assert reports[0].unchanged
|
|
assert commits() == before
|
|
|
|
|
|
def test_sync_refuses_to_overwrite_a_canvas_edit(api, flows):
|
|
sync([flows["train"]], api, origin=ORIGIN)
|
|
api.put_source("train", "evaluate", "def process(weights):\n return {}\n")
|
|
|
|
with pytest.raises(SyncError, match="edited on the canvas"):
|
|
sync([flows["train"]], api, origin=ORIGIN)
|
|
|
|
reports = sync([flows["train"]], api, origin=ORIGIN, force=True)
|
|
assert "evaluate" in reports[0].changed
|
|
assert api.get_source("train", "evaluate").startswith(MARKER)
|
|
|
|
|
|
def test_sync_refuses_a_flow_it_did_not_create(api, flows):
|
|
api.put_flow({"name": "drawn", "version": 1, "nodes": [], "mode": "batch"})
|
|
api.publish("drawn", 1)
|
|
theirs = flows["train"]
|
|
theirs.name = "drawn"
|
|
|
|
try:
|
|
with pytest.raises(SyncError, match="not created by sync"):
|
|
sync([theirs], api, origin=ORIGIN)
|
|
finally:
|
|
theirs.name = "train"
|
|
|
|
|
|
def test_use_stores_the_rewiring_it_was_given(api, flows):
|
|
sync([flows["finetune"]], api, origin=ORIGIN)
|
|
|
|
nodes = {n["id"]: n for n in api.get_flow("finetune")["definition"]["nodes"]}
|
|
assert [p["name"] for p in nodes["fit"]["requires"]] == ["augmented", "lr"]
|
|
assert nodes["fit"]["params"] == {"epochs": 3}
|
|
|
|
|
|
def test_refresh_retires_the_workers(client, superuser_token_headers):
|
|
response = client.post(f"{PREFIX}/modules/refresh", headers=superuser_token_headers)
|
|
|
|
assert response.status_code == 200
|
|
assert "afresh" in response.json()["message"]
|