Run python nodes out of process, with modules of their own

User code no longer execs in the engine. A pool of persistent worker
subprocesses speaks one JSON object per line; the controller installs a
proxy as the node's function, so every execution path funnels through it
and the pipeline is untouched. A crash costs one subprocess, a per-node
timeout is a kill, and cancelling from the canvas is that same kill.

The workers run a venv of the user's own on the data volume, filled from
a pip manifest versioned beside the flows. Applying it retires the
workers and rebuilds, so a package lands without restarting the engine.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt
This commit is contained in:
2026-08-16 21:43:36 +02:00
co-authored by Claude Fable 5
parent 979c9d3c1f
commit f300c43f3a
27 changed files with 1536 additions and 46 deletions
+62
View File
@@ -0,0 +1,62 @@
"""The venv node code imports from is built and kept in step with a manifest."""
import shutil
import subprocess
from pathlib import Path
import pytest
from app.flow import modules
from app.flow.store import FlowStore
pytestmark = pytest.mark.skipif(
shutil.which("uv") is None, reason="module management needs uv"
)
@pytest.fixture
def venv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
directory = tmp_path / "user-venv"
monkeypatch.setattr(modules, "VENV_DIR", directory)
return directory
def test_an_empty_manifest_gives_a_venv_of_its_own(venv: Path, tmp_path: Path):
store = FlowStore(tmp_path / "flows")
store.write_requirements("")
modules.sync("")
assert Path(modules.venv_python()).exists()
assert modules.venv_python().startswith(str(venv))
information = modules.info(store)
assert information.applied is True
assert information.python_version
def test_a_manifest_already_applied_is_not_installed_again(
venv: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
store = FlowStore(tmp_path / "flows")
store.write_requirements("")
modules.sync("")
marker = (venv / ".applied").read_text()
# Recorded rather than refused: reconcile swallows what it fails on, so an
# exception raised in here would never reach the test.
calls: list[object] = []
monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: calls.append(args))
modules.reconcile(store)
assert calls == []
assert (venv / ".applied").read_text() == marker
def test_a_manifest_that_does_not_resolve_leaves_the_venv_alone(venv: Path):
modules.sync("")
ok, output = modules.sync("fluksio-no-such-package-anywhere==9.9.9")
assert ok is False
assert output
# The marker still describes the manifest that actually installed.
assert (venv / ".applied").read_text() == modules._digest("")