Add a Python SDK: flows declared in your own repository

A data scientist keeps their code where it is and decorates it: `@node`
declares a function's ports beside the function, `Flow(name, nodes=[...])`
says which of them make a flow, and `use(fn, wire=..., **settings)` rebinds
one for a single flow. `fluksio sync` uploads the document plus a generated
import shim per node, so the store still holds a complete, runnable,
git-versioned definition while the code it imports stays theirs.

`fluksio login|run|runs` and `flow.submit().wait()` are the client half, over
the run endpoints that already existed. Runs record the user repository's
commit beside the store's, so "what code produced this number" is answerable
on the side that now holds the code.

- `fluksio/sdk/`: ports, decorators, the flow builder and its checks, the shim
  generator, an HTTP client and sync. Standard library only at import, so
  `from fluksio import node` in a training script pulls in no engine.
- `FlowDef.origin` marks a flow code-defined; `Run.origin_commit` carries the
  repository's commit; `POST /modules/refresh` retires the workers without an
  install, which every sync calls — a worker holds the imported package in
  memory, so an edit to it is invisible until the process goes.
- The canvas shows a generated body read-only and names the repository to edit
  instead; a body edited there stops the next sync rather than being discarded.
- The worker's reporter carries inert `Port`, `node`, `use` and `Flow`, since
  the shim imports a module whose first line declares them.
- `examples/myresearch` is the worked example, `make sync-example` uploads it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ue1tkFWB1bcGy3aWhCKpU

backend/tests/api/routes/test_sync.py

@@ -0,0 +1,115 @@
+"""`fluksio sync` against a real engine: what it stores, and what it refuses."""
+
+import sys
+from collections.abc import Generator
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+
+from fluksio.core.config import settings
+from fluksio.sdk import FLOWS, MARKER, SyncError
+from fluksio.sdk.client import Client, sync
+
+EXAMPLES = Path(__file__).parents[4] / "examples"
+PREFIX = settings.API_V1_STR
+ORIGIN = {
+    "kind": "python",
+    "repo": str(EXAMPLES),
+    "commit": "a1b2c3d4",
+    "dirty": False,
+}
+
+
+@pytest.fixture(scope="module")
+def flows() -> Generator[dict, None, None]:
+    """The example research package, imported the way `sync` imports it."""
+    sys.path.insert(0, str(EXAMPLES))
+    FLOWS.clear()
+    import myresearch.pipeline  # noqa: F401
+
+    yield dict(FLOWS)
+    FLOWS.clear()
+    sys.path.remove(str(EXAMPLES))
+
+
+@pytest.fixture
+def api(client: TestClient, superuser_token_headers: dict[str, str]) -> Client:
+    return Client(http=client, token=superuser_token_headers["Authorization"][7:])
+
+
+def commits() -> int:
+    """How many commits the flow store has made."""
+    from fluksio.flow.store import FlowStore
+
+    store = FlowStore(settings.FLOWS_DIR)
+    result = store._git("rev-list", "--count", "HEAD")
+    return int(result.stdout.strip()) if result.returncode == 0 else 0
+
+
+def test_sync_stores_a_runnable_flow_with_its_origin(api, flows):
+    reports = sync([flows["train"]], api, origin=ORIGIN)
+
+    assert [r.flow for r in reports] == ["train"]
+    assert reports[0].created and reports[0].published
+
+    stored = api.get_flow("train")
+    definition = stored["definition"]
+    assert definition["origin"]["commit"] == "a1b2c3d4"
+    assert [n["id"] for n in definition["nodes"]] == ["prepare", "fit", "evaluate"]
+    assert not stored["has_draft"]
+    # The store holds a complete definition, and the bodies say where the real
+    # code is rather than being a copy of it.
+    source = api.get_source("train", "fit")
+    assert source.startswith(MARKER)
+    assert "from myresearch.train import fit" in source
+
+
+def test_a_second_sync_changes_nothing_and_commits_nothing(api, flows):
+    sync([flows["train"]], api, origin=ORIGIN)
+    before = commits()
+
+    reports = sync([flows["train"]], api, origin=ORIGIN)
+
+    assert reports[0].unchanged
+    assert commits() == before
+
+
+def test_sync_refuses_to_overwrite_a_canvas_edit(api, flows):
+    sync([flows["train"]], api, origin=ORIGIN)
+    api.put_source("train", "evaluate", "def process(weights):\n    return {}\n")
+
+    with pytest.raises(SyncError, match="edited on the canvas"):
+        sync([flows["train"]], api, origin=ORIGIN)
+
+    reports = sync([flows["train"]], api, origin=ORIGIN, force=True)
+    assert "evaluate" in reports[0].changed
+    assert api.get_source("train", "evaluate").startswith(MARKER)
+
+
+def test_sync_refuses_a_flow_it_did_not_create(api, flows):
+    api.put_flow({"name": "drawn", "version": 1, "nodes": [], "mode": "batch"})
+    api.publish("drawn", 1)
+    theirs = flows["train"]
+    theirs.name = "drawn"
+
+    try:
+        with pytest.raises(SyncError, match="not created by sync"):
+            sync([theirs], api, origin=ORIGIN)
+    finally:
+        theirs.name = "train"
+
+
+def test_use_stores_the_rewiring_it_was_given(api, flows):
+    sync([flows["finetune"]], api, origin=ORIGIN)
+
+    nodes = {n["id"]: n for n in api.get_flow("finetune")["definition"]["nodes"]}
+    assert [p["name"] for p in nodes["fit"]["requires"]] == ["augmented", "lr"]
+    assert nodes["fit"]["params"] == {"epochs": 3}
+
+
+def test_refresh_retires_the_workers(client, superuser_token_headers):
+    response = client.post(f"{PREFIX}/modules/refresh", headers=superuser_token_headers)
+
+    assert response.status_code == 200
+    assert "afresh" in response.json()["message"]

backend/tests/sdk/conftest.py

@@ -0,0 +1,19 @@
+from collections.abc import Generator
+
+import pytest
+
+from fluksio.sdk import FLOWS
+
+
+@pytest.fixture(scope="session", autouse=True)
+def db() -> Generator[None, None, None]:
+    """Declaring a flow touches no database."""
+    yield
+
+
+@pytest.fixture(autouse=True)
+def registry() -> Generator[None, None, None]:
+    """The registry is module-level, so one test's flows are not another's."""
+    FLOWS.clear()
+    yield
+    FLOWS.clear()

backend/tests/sdk/test_build.py

@@ -0,0 +1,238 @@
+"""What the decorators declare, and what the engine would be given."""
+
+import pytest
+
+from fluksio.flow.schemas import FlowDef
+from fluksio.sdk import MARKER, Flow, Port, SyncError, node, use
+
+# The functions live here rather than in each test: a node's module is what the
+# generated body imports, and `__main__` is refused for exactly that reason.
+
+
+@node(provides=[Port("dataset", "artifact"), Port("rows", "int")])
+def prepare(limit=8):
+    return {"dataset": {"digest": "sha256:x", "size": 1}, "rows": limit}
+
+
+@node(
+    requires=["dataset", Port("lr", "float")],
+    provides=[Port("loss", "float", stream=True), Port("weights", "artifact")],
+    device="gpu",
+    device_policy="prefer",
+)
+def fit(dataset, lr, epochs=3):
+    for step in range(epochs):
+        yield {"loss": 1.0 / (step + 1)}
+    return {"weights": {"digest": "sha256:y", "size": 1}}
+
+
+@node(requires=["weights"], provides=Port("score", "float"))
+def evaluate(weights):
+    return 0.5
+
+
+@node(requires=["dataset"], provides=[Port("augmented", "artifact")])
+def augment(dataset):
+    return {"augmented": dataset}
+
+
+def a_flow(name="train", **kwargs):
+    options = {
+        "nodes": [prepare, fit, evaluate],
+        "inputs": [Port("lr", "float", initial=0.01)],
+        "outputs": ["score"],
+    }
+    options.update(kwargs)
+    return Flow(name, **options)
+
+
+def test_document_is_a_flow_the_engine_accepts():
+    document = a_flow().document({"kind": "python", "repo": "/r", "commit": "abc"})
+    definition = FlowDef.model_validate(document)
+
+    assert definition.mode == "batch"
+    assert [n.id for n in definition.nodes] == ["prepare", "fit", "evaluate"]
+    assert definition.origin is not None
+    assert definition.origin.commit == "abc"
+    assert [i.spec.name for i in definition.inputs] == ["lr"]
+    assert definition.outputs == ["score"]
+
+
+def test_defaults_become_settings_and_ports_do_not():
+    definition = FlowDef.model_validate(a_flow().document())
+    nodes = {n.id: n for n in definition.nodes}
+
+    assert nodes["prepare"].params == {"limit": 8}
+    # `dataset` and `lr` are ports, so they are not settings as well.
+    assert nodes["fit"].params == {"epochs": 3}
+    assert {p.port for p in nodes["fit"].requires} == {"dataset", "lr"}
+
+
+def test_device_travels_to_the_node():
+    definition = FlowDef.model_validate(a_flow().document())
+    node_def = next(n for n in definition.nodes if n.id == "fit")
+
+    assert node_def.device == "gpu"
+    assert node_def.device_policy == "prefer"
+
+
+def test_a_required_port_takes_the_type_it_is_provided_as():
+    """`requires=["dataset"]` is not a second declaration of its type."""
+    definition = FlowDef.model_validate(a_flow().document())
+    node_def = next(n for n in definition.nodes if n.id == "fit")
+
+    assert next(p for p in node_def.requires if p.port == "dataset").dtype == "artifact"
+
+
+def test_use_rewires_and_reconfigures_one_flow_only():
+    train = a_flow()
+    finetune = Flow(
+        "finetune",
+        nodes=[prepare, augment, use(fit, wire={"dataset": "augmented"}, epochs=9)],
+        inputs=[Port("lr", "float")],
+    )
+
+    theirs = next(n for n in finetune.document()["nodes"] if n["id"] == "fit")
+    ours = next(n for n in train.document()["nodes"] if n["id"] == "fit")
+    assert [p["name"] for p in theirs["requires"]] == ["augmented", "lr"]
+    assert theirs["params"] == {"epochs": 9}
+    # The same function in another flow is untouched by that.
+    assert [p["name"] for p in ours["requires"]] == ["dataset", "lr"]
+    assert ours["params"] == {"epochs": 3}
+
+
+def unknown_port(a):
+    return a
+
+
+def two_arguments(a, b):
+    return a
+
+
+def one_default(a=1):
+    return a
+
+
+def a_generator():
+    yield {"x": 1}
+
+
+def reads_dataset(dataset):
+    return dataset
+
+
+def test_a_port_with_no_parameter_is_refused():
+    with pytest.raises(SyncError, match="no parameter to arrive in"):
+        node(requires=["nope"])(unknown_port)
+
+
+def test_a_parameter_that_is_neither_port_nor_setting_is_refused():
+    with pytest.raises(SyncError, match="neither a port nor a setting"):
+        node(requires=["a"])(two_arguments)
+
+
+def test_a_name_the_store_would_refuse_is_refused_here():
+    with pytest.raises(SyncError, match="node id"):
+        node(requires=["a"], id="Trainer")(one_default)
+
+
+def test_a_generator_cannot_use_the_bare_return_shorthand():
+    with pytest.raises(SyncError, match="yields"):
+        node(provides=Port("x", "float"))(a_generator)
+
+
+def test_a_setting_that_is_also_a_port_is_refused():
+    with pytest.raises(SyncError, match="both a port and a setting"):
+        Flow("clash", nodes=[use(prepare, limit=2), _sets("dataset")])
+
+
+def test_a_setting_that_is_not_a_parameter_is_refused():
+    with pytest.raises(SyncError, match="not a parameter"):
+        Flow("stray", nodes=[use(prepare, nonesuch=1)])
+
+
+def _sets(name):
+    """A node whose settings clash with its own port, built for the test."""
+    fn = node(requires=["dataset"], settings={name: 1})
+    return fn(reads_dataset)
+
+
+def test_a_message_nobody_provides_is_refused():
+    orphan = node(requires=["missing"], id="orphan")(reads_dataset_missing)
+    with pytest.raises(SyncError, match="which no node in it provides"):
+        Flow("broken", nodes=[orphan])
+
+
+def reads_dataset_missing(missing):
+    return missing
+
+
+def test_an_output_nobody_produces_is_refused():
+    with pytest.raises(SyncError, match="not produced by any"):
+        a_flow("typo", outputs=["scoer"])
+
+
+def test_two_nodes_of_one_name_are_refused():
+    with pytest.raises(SyncError, match="tell them apart"):
+        Flow("twice", nodes=[prepare, prepare])
+
+
+def test_a_type_both_sides_disagree_on_is_refused():
+    wrong = node(requires=[Port("dataset", "float")], id="wrong")(reads_dataset)
+    with pytest.raises(SyncError, match="provided as artifact"):
+        Flow("mistyped", nodes=[prepare, wrong])
+
+
+def test_shim_imports_rather_than_copies():
+    shims = a_flow().shims()
+
+    assert shims["prepare"].startswith(MARKER)
+    assert "from tests.sdk.test_build import prepare" in shims["prepare"]
+    assert "def process(**settings):" in shims["prepare"]
+    assert "return prepare(**settings)" in shims["prepare"]
+
+
+def test_shim_of_a_generator_delegates_and_keeps_its_return_value():
+    """A bare `yield from` streams but drops what the generator returns."""
+    assert (
+        "return (yield from fit(dataset=dataset, lr=lr, **settings))"
+        in a_flow().shims()["fit"]
+    )
+
+
+def test_a_generator_shim_publishes_both_the_stream_and_the_result():
+    """Driven the way the worker drives it: every yield, then the return."""
+    namespace: dict = {}
+    exec(compile(a_flow().shims()["fit"], "<shim>", "exec"), namespace)
+
+    generator = namespace["process"](dataset=None, lr=0.5, epochs=2)
+    streamed = []
+    result = None
+    while True:
+        try:
+            streamed.append(next(generator))
+        except StopIteration as stop:
+            result = stop.value
+            break
+
+    assert [step["loss"] for step in streamed] == [1.0, 0.5]
+    # A bare `yield from` would stream the same and lose this.
+    assert result == {"weights": {"digest": "sha256:y", "size": 1}}
+
+
+def test_shim_of_a_single_port_wraps_the_bare_return():
+    assert "return {'score': evaluate(weights=weights" in a_flow().shims()["evaluate"]
+
+
+def test_every_shim_compiles_and_defines_process():
+    for code in a_flow().shims().values():
+        namespace: dict = {}
+        exec(compile(code, "<shim>", "exec"), namespace)
+        assert callable(namespace["process"])
+
+
+def test_the_decorators_leave_the_function_alone():
+    """The point of the whole feature: it is still your code."""
+    assert prepare(limit=2)["rows"] == 2
+    assert [step["loss"] for step in fit(None, 0.5, epochs=2)] == [1.0, 0.5]
+    assert evaluate(None) == 0.5

backend/tests/test_cli.py

@@ -60,3 +60,29 @@ def test_the_store_works_without_git(tmp_path: Path, monkeypatch) -> None:
     assert store.head() == ""
     store.write_requirements("numpy\n")
     assert store.read_requirements() == "numpy\n"
+
+
+def test_run_arguments_are_typed_by_the_flow_they_are_for() -> None:
+    """`--lr 0.05` is a float because the flow says `lr` is one."""
+    from fluksio.sdk import SyncError
+    from fluksio.sdk.cli import _params
+
+    definition = {
+        "inputs": [
+            {"spec": {"name": "lr", "dtype": "float"}},
+            {"spec": {"name": "epochs", "dtype": "int"}},
+            {"spec": {"name": "resume", "dtype": "bool"}},
+        ]
+    }
+
+    assert _params(definition, ["--lr", "0.05", "--epochs", "3", "--resume"]) == {
+        "lr": 0.05,
+        "epochs": 3,
+        "resume": True,
+    }
+    assert _params(definition, ["--lr=1e-4"]) == {"lr": 0.0001}
+
+    import pytest
+
+    with pytest.raises(SyncError, match="not an input of this flow"):
+        _params(definition, ["--nonesuch", "1"])