Run python nodes out of process, with modules of their own

User code no longer execs in the engine. A pool of persistent worker
subprocesses speaks one JSON object per line; the controller installs a
proxy as the node's function, so every execution path funnels through it
and the pipeline is untouched. A crash costs one subprocess, a per-node
timeout is a kill, and cancelling from the canvas is that same kill.

The workers run a venv of the user's own on the data volume, filled from
a pip manifest versioned beside the flows. Applying it retires the
workers and rebuilds, so a package lands without restarting the engine.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt

backend/tests/flow/test_modules.py

@@ -0,0 +1,62 @@
+"""The venv node code imports from is built and kept in step with a manifest."""
+
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from app.flow import modules
+from app.flow.store import FlowStore
+
+pytestmark = pytest.mark.skipif(
+    shutil.which("uv") is None, reason="module management needs uv"
+)
+
+
+@pytest.fixture
+def venv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+    directory = tmp_path / "user-venv"
+    monkeypatch.setattr(modules, "VENV_DIR", directory)
+    return directory
+
+
+def test_an_empty_manifest_gives_a_venv_of_its_own(venv: Path, tmp_path: Path):
+    store = FlowStore(tmp_path / "flows")
+    store.write_requirements("")
+    modules.sync("")
+
+    assert Path(modules.venv_python()).exists()
+    assert modules.venv_python().startswith(str(venv))
+
+    information = modules.info(store)
+    assert information.applied is True
+    assert information.python_version
+
+
+def test_a_manifest_already_applied_is_not_installed_again(
+    venv: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+):
+    store = FlowStore(tmp_path / "flows")
+    store.write_requirements("")
+    modules.sync("")
+    marker = (venv / ".applied").read_text()
+
+    # Recorded rather than refused: reconcile swallows what it fails on, so an
+    # exception raised in here would never reach the test.
+    calls: list[object] = []
+    monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: calls.append(args))
+    modules.reconcile(store)
+
+    assert calls == []
+    assert (venv / ".applied").read_text() == marker
+
+
+def test_a_manifest_that_does_not_resolve_leaves_the_venv_alone(venv: Path):
+    modules.sync("")
+    ok, output = modules.sync("fluksio-no-such-package-anywhere==9.9.9")
+
+    assert ok is False
+    assert output
+    # The marker still describes the manifest that actually installed.
+    assert (venv / ".applied").read_text() == modules._digest("")

backend/tests/flow/test_workers.py

@@ -0,0 +1,109 @@
+"""Python nodes run in a worker process, and stay there when things go wrong."""
+
+import sys
+import threading
+import time
+from collections.abc import Iterator
+
+import pytest
+
+from app.flow.workers import NodeCancelled, NodeTimeout, PythonWorkerPool
+
+
+@pytest.fixture
+def pool() -> Iterator[PythonWorkerPool]:
+    # One worker: a respawn is then provably the same slot coming back.
+    worker_pool = PythonWorkerPool(python=sys.executable, size=1)
+    worker_pool.start()
+    yield worker_pool
+    worker_pool.stop()
+
+
+def run(pool: PythonWorkerPool, code: str, node: str = "demo", **kwargs):
+    return pool.run(
+        "demo", node, code, kwargs, {"factor": 2}, f"demo.{node}", timeout=5
+    )
+
+
+def test_a_node_returns_its_value_and_what_it_printed(pool, capsys):
+    result = run(
+        pool,
+        "def process(value, params):\n"
+        "    print('seen', value)\n"
+        "    return {'out': value * params['factor']}\n",
+        value=21,
+    )
+    assert result == {"out": 42}
+    # The proxy writes them to stdout, which is where the engine's tee is.
+    assert "seen 21" in capsys.readouterr().out
+
+
+def test_a_failure_keeps_its_class_and_points_at_the_node(pool):
+    with pytest.raises(Exception) as caught:
+        run(pool, "def process(params):\n    raise ValueError('bad input')\n")
+
+    # The engine renders a node error as "<class>: <message>", so both have to
+    # survive the trip.
+    assert type(caught.value).__name__ == "ValueError"
+    assert str(caught.value) == "bad input"
+    assert "<node demo." in caught.value.remote_traceback
+    assert "ValueError: bad input" in caught.value.remote_traceback
+
+
+def test_a_node_that_kills_its_worker_is_an_ordinary_error(pool):
+    with pytest.raises(Exception, match="worker died"):
+        run(pool, "import os\n\n\ndef process(params):\n    os._exit(1)\n")
+
+    assert run(pool, "def process(params):\n    return {'out': 1}\n") == {"out": 1}
+
+
+def test_a_node_that_runs_too_long_is_killed_and_the_pool_recovers(pool):
+    started = time.monotonic()
+    with pytest.raises(NodeTimeout):
+        pool.run(
+            "demo",
+            "slow",
+            "import time\n\n\ndef process(params):\n    time.sleep(30)\n",
+            {},
+            {},
+            "demo.slow",
+            timeout=1,
+        )
+    assert time.monotonic() - started < 10
+
+    # The killed worker's slot is refilled on the next call.
+    assert run(pool, "def process(params):\n    return {'out': 2}\n") == {"out": 2}
+
+
+def test_a_running_node_can_be_cancelled(pool):
+    def stop_it() -> None:
+        for _ in range(100):
+            if pool.cancel("demo.slow"):
+                return
+            time.sleep(0.05)
+
+    stopper = threading.Thread(target=stop_it)
+    stopper.start()
+    try:
+        with pytest.raises(NodeCancelled):
+            pool.run(
+                "demo",
+                "slow",
+                "import time\n\n\ndef process(params):\n    time.sleep(30)\n",
+                {},
+                {},
+                "demo.slow",
+                timeout=30,
+            )
+    finally:
+        stopper.join()
+
+
+def test_a_result_that_is_not_json_is_refused(pool):
+    with pytest.raises(Exception, match="cannot be sent back as JSON"):
+        run(pool, "def process(params):\n    return {'out': {1, 2}}\n")
+
+
+def test_compiling_reports_where_the_source_is_wrong(pool):
+    assert pool.compile("demo", "broken", "def process(params)\n    return {}\n")
+    assert pool.compile("demo", "fine", "def process(params):\n    return {}\n") is None