SDK: sync and run import the declaring module in a child process

The command line no longer imports a research module, so import-time GPU work
ends with a short-lived child instead of holding a CUDA context for the length
of a followed run. fluksio.sdk.declare starts the child, which writes each
flow's document and shims as JSON; Declared stands in for Flow in sync().
Errors keep their type and message, a child's traceback reaches the user.

Claude-Session: https://claude.ai/code/session_01SSi5AvZ7pFmySK8zy3NPMh

backend/tests/api/routes/test_sync.py

@@ -8,7 +8,7 @@ import pytest
 from fastapi.testclient import TestClient
 
 from fluksio.core.config import settings
-from fluksio.sdk import FLOWS, MARKER, SyncError
+from fluksio.sdk import FLOWS, MARKER, Declared, SyncError
 from fluksio.sdk.client import ApiError, Client, sync
 
 EXAMPLES = Path(__file__).parents[4] / "examples"
@@ -74,6 +74,18 @@ def test_sync_stores_a_runnable_flow_with_its_origin(api, flows):
     assert "from myresearch.train import fit" in source
 
 
+def test_a_flow_read_by_another_process_syncs_as_the_flow_does(api, flows):
+    """`fluksio sync` imports in a child and uploads what it handed back."""
+    carried = Declared(flows["train"].declaration())
+
+    sync([carried], api, origin=ORIGIN)
+    stored = api.get_flow("train")["definition"]
+    assert stored["origin"]["declared_in"] == "myresearch/pipeline.py"
+
+    # Nothing differs from what the flow itself would have stored.
+    assert sync([flows["train"]], api, origin=ORIGIN)[0].unchanged
+
+
 def test_a_node_carries_the_modules_its_code_reaches(api, flows):
     """The shim names one function; the cache has to key on what it calls."""
     sync([flows["train"]], api, origin=ORIGIN)

backend/tests/sdk/test_declare.py

@@ -0,0 +1,141 @@
+"""Reading a module's flows in a child, so the command line never imports it."""
+
+import os
+import sys
+from pathlib import Path
+
+import pytest
+
+from fluksio.sdk import FLOWS, SyncError, cli
+from fluksio.sdk.declare import ImportFailed, declare, declare_module
+
+STUDY = """\
+import os
+
+from fluksio import Flow, Port, node
+
+with open({marker!r}, "a") as marker:
+    marker.write(str(os.getpid()) + "\\n")
+
+
+@node(provides=Port("out", "int"))
+def work():
+    return 1
+
+
+repro = Flow("repro", nodes=[work], outputs=["out"])
+"""
+
+
+def _study(tmp_path: Path, body: str | None = None) -> Path:
+    path = tmp_path / "study.py"
+    path.write_text(body or STUDY.format(marker=str(tmp_path / "marker")))
+    return path
+
+
+def test_the_module_is_imported_by_a_child_and_not_here(tmp_path):
+    """What opens a CUDA context at import must not do it in the command line."""
+    path = _study(tmp_path)
+
+    found = declare_module(str(tmp_path), "study", path)
+
+    assert [flow.name for flow in found] == ["repro"]
+    pids = (tmp_path / "marker").read_text().split()
+    assert pids and str(os.getpid()) not in pids
+    assert "study" not in sys.modules and "repro" not in FLOWS
+    assert str(tmp_path) not in sys.path
+
+
+def test_a_declared_flow_is_what_the_flow_itself_would_sync(tmp_path):
+    """Same document, same shims, same origin: sync cannot tell them apart."""
+    path = _study(tmp_path)
+    origin = {"kind": "python", "repo": str(tmp_path), "commit": "", "dirty": False}
+
+    (declared,) = declare([str(path.parent)])
+    try:
+        (local,) = cli.discover([str(path)])
+        assert declared.document(origin) == local.document(origin)
+        assert declared.shims() == local.shims()
+        assert declared.document(origin)["origin"]["declared_in"] == "study.py"
+    finally:
+        FLOWS.pop("repro", None)
+        sys.modules.pop("study", None)
+        sys.path[:] = [entry for entry in sys.path if entry != str(tmp_path)]
+
+
+def test_what_a_module_raises_reaches_the_user_as_it_did_in_process(tmp_path):
+    path = _study(tmp_path, "raise ValueError('boom')\n")
+    with pytest.raises(ImportFailed, match="^ValueError: boom$") as raised:
+        declare_module(str(tmp_path), "study", path)
+    assert "study.py" in raised.value.trace
+
+    path.write_text("import not_a_module_anywhere\n")
+    with pytest.raises(ModuleNotFoundError, match="not_a_module_anywhere"):
+        declare_module(str(tmp_path), "study", path)
+
+    # A refusal from the declaration itself keeps its own type, and message.
+    path.write_text(
+        "from fluksio import Flow\nFlow('Not A Name', nodes=[])\n",
+    )
+    with pytest.raises(SyncError, match="flow name"):
+        declare_module(str(tmp_path), "study", path)
+
+
+def test_a_child_that_ends_without_an_answer_is_named(tmp_path):
+    path = _study(tmp_path, "import os\nos._exit(3)\n")
+    with pytest.raises(SyncError, match="exit status 3"):
+        declare_module(str(tmp_path), "study", path)
+
+
+def test_a_walk_keeps_going_past_a_study_that_will_not_import(tmp_path, capfd):
+    (tmp_path / "bad.py").write_text("raise RuntimeError('half an edit')\n")
+    _study(tmp_path)
+
+    found = declare([str(tmp_path)], keep_going=True)
+
+    assert [flow.name for flow in found] == ["repro"]
+    assert "warning: skipped" in capfd.readouterr().out
+    with pytest.raises(SyncError, match="failed to import"):
+        declare([str(tmp_path)])
+
+
+def test_sync_prints_what_the_child_declared(tmp_path, capsys):
+    from fluksio.cli import _parser
+
+    path = _study(tmp_path)
+    args = _parser().parse_args(["sync", "--dry-run", str(path.parent)])
+
+    assert cli.cmd_sync(args) == 0
+    out = capsys.readouterr().out
+    assert "=== flow repro" in out
+    assert '"declared_in": "study.py"' in out
+    assert "=== repro.work" in out
+
+
+def test_a_failing_import_is_reported_by_sync_and_warned_about_by_run(
+    tmp_path, monkeypatch, capsys
+):
+    from fluksio.cli import _parser
+
+    path = _study(tmp_path, "raise ValueError('boom')\n")
+
+    # Found by a walk, it is the walk's sentence.
+    assert cli.cmd_sync(_parser().parse_args(["sync", str(tmp_path)])) == 1
+    assert "study.py failed to import — ValueError: boom" in capsys.readouterr().err
+
+    # Named outright, it is what the module raised, with the child's traceback
+    # where an uncaught exception used to print its own.
+    assert cli.cmd_sync(_parser().parse_args(["sync", str(path)])) == 1
+    seen = capsys.readouterr()
+    assert "fluksio: ValueError: boom" in seen.err
+    assert "Traceback" in seen.err and "study.py" in seen.err
+
+    class Engine:
+        def get_flow(self, name):
+            return {
+                "definition": {"origin": {"declared_in": "study.py", "module": "study"}}
+            }
+
+    monkeypatch.chdir(tmp_path)
+    cli._sync_first(Engine(), [], "repro")
+    assert "warning: nothing synced — ValueError: boom" in capsys.readouterr().out

backend/tests/test_cli.py

@@ -687,10 +687,12 @@ def test_run_imports_the_module_that_declared_the_flow(tmp_path, monkeypatch):
 
     imported: list[tuple] = []
     walked: list[list[str]] = []
-    monkeypatch.setattr(sdk_cli, "_import", lambda *args: imported.append(args))
+    monkeypatch.setattr(
+        sdk_cli, "declare_module", lambda *args: imported.append(args) or []
+    )
     monkeypatch.setattr(
         sdk_cli,
-        "discover",
+        "declare",
         lambda targets, keep_going=False: walked.append(targets) or [],
     )