Run python nodes out of process, with modules of their own
User code no longer execs in the engine. A pool of persistent worker subprocesses speaks one JSON object per line; the controller installs a proxy as the node's function, so every execution path funnels through it and the pipeline is untouched. A crash costs one subprocess, a per-node timeout is a kill, and cancelling from the canvas is that same kill. The workers run a venv of the user's own on the data volume, filled from a pip manifest versioned beside the flows. Applying it retires the workers and rebuilds, so a package lands without restarting the engine. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
"""The venv node code imports from is built and kept in step with a manifest."""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.flow import modules
|
||||
from app.flow.store import FlowStore
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
shutil.which("uv") is None, reason="module management needs uv"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def venv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
directory = tmp_path / "user-venv"
|
||||
monkeypatch.setattr(modules, "VENV_DIR", directory)
|
||||
return directory
|
||||
|
||||
|
||||
def test_an_empty_manifest_gives_a_venv_of_its_own(venv: Path, tmp_path: Path):
|
||||
store = FlowStore(tmp_path / "flows")
|
||||
store.write_requirements("")
|
||||
modules.sync("")
|
||||
|
||||
assert Path(modules.venv_python()).exists()
|
||||
assert modules.venv_python().startswith(str(venv))
|
||||
|
||||
information = modules.info(store)
|
||||
assert information.applied is True
|
||||
assert information.python_version
|
||||
|
||||
|
||||
def test_a_manifest_already_applied_is_not_installed_again(
|
||||
venv: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
store = FlowStore(tmp_path / "flows")
|
||||
store.write_requirements("")
|
||||
modules.sync("")
|
||||
marker = (venv / ".applied").read_text()
|
||||
|
||||
# Recorded rather than refused: reconcile swallows what it fails on, so an
|
||||
# exception raised in here would never reach the test.
|
||||
calls: list[object] = []
|
||||
monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: calls.append(args))
|
||||
modules.reconcile(store)
|
||||
|
||||
assert calls == []
|
||||
assert (venv / ".applied").read_text() == marker
|
||||
|
||||
|
||||
def test_a_manifest_that_does_not_resolve_leaves_the_venv_alone(venv: Path):
|
||||
modules.sync("")
|
||||
ok, output = modules.sync("fluksio-no-such-package-anywhere==9.9.9")
|
||||
|
||||
assert ok is False
|
||||
assert output
|
||||
# The marker still describes the manifest that actually installed.
|
||||
assert (venv / ".applied").read_text() == modules._digest("")
|
||||
Reference in New Issue
Block a user