Run python nodes out of process, with modules of their own

User code no longer execs in the engine. A pool of persistent worker
subprocesses speaks one JSON object per line; the controller installs a
proxy as the node's function, so every execution path funnels through it
and the pipeline is untouched. A crash costs one subprocess, a per-node
timeout is a kill, and cancelling from the canvas is that same kill.

The workers run a venv of the user's own on the data volume, filled from
a pip manifest versioned beside the flows. Applying it retires the
workers and rebuilds, so a package lands without restarting the engine.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt

backend/app/api/deps.py

@@ -13,6 +13,7 @@ from app.core.config import settings
 from app.core.db import engine
 from app.flow.controller import FlowController
 from app.flow.dashboards import DashboardStore
+from app.flow.workers import PythonWorkerPool
 from app.models import TokenPayload, User
 
 reusable_oauth2 = OAuth2PasswordBearer(
@@ -113,6 +114,16 @@ def get_dashboard_store(request: Request) -> DashboardStore:
 DashboardStoreDep = Annotated[DashboardStore, Depends(get_dashboard_store)]
 
 
+def get_worker_pool(request: Request) -> PythonWorkerPool:
+    pool: PythonWorkerPool | None = getattr(request.app.state, "worker_pool", None)
+    if pool is None:
+        raise HTTPException(status_code=503, detail="The flow engine is not running")
+    return pool
+
+
+WorkerPoolDep = Annotated[PythonWorkerPool, Depends(get_worker_pool)]
+
+
 def get_current_active_superuser(current_user: CurrentUser) -> User:
     if not current_user.is_superuser:
         raise HTTPException(

backend/app/api/main.py

@@ -6,6 +6,7 @@ from app.api.routes import (
     flows,
     login,
     messages,
+    modules,
     oauth,
     private,
     secrets,
@@ -23,6 +24,7 @@ api_router.include_router(secrets.router)
 api_router.include_router(alerts.router)
 api_router.include_router(dashboards.router)
 api_router.include_router(messages.router)
+api_router.include_router(modules.router)
 # Always mounted so the generated SDK stays the same shape; the endpoints
 # themselves refuse to work unless MCP is switched on.
 api_router.include_router(oauth.router)

backend/app/api/routes/flows.py

@@ -606,6 +606,19 @@ async def trigger_node(
     return _flow_state(controller, name)
 
 
+@router.post("/{name}/nodes/{node_id}/cancel", response_model=Message)
+def cancel_node(name: str, node_id: str, controller: FlowControllerDep) -> Any:
+    """Stop a node that is running right now, by killing the worker running it.
+
+    Idempotent on purpose: by the time a click reaches here the node may well
+    have finished, and that is the outcome that was asked for.
+    """
+    pool = controller.workers
+    if pool is not None and pool.cancel(f"{name}.{node_id}"):
+        return Message(message=f"Stopped '{node_id}'")
+    return Message(message=f"'{node_id}' was not running")
+
+
 @router.get("/{name}/state", response_model=FlowStatePublic)
 def read_flow_state(name: str, controller: FlowControllerDep) -> Any:
     """The last value seen on every message of this flow."""

backend/app/api/routes/modules.py

@@ -0,0 +1,50 @@
+"""The python packages node code may import.
+
+A manifest in the flow store, a venv on the data volume, and one button that
+brings the second in line with the first. Nothing here restarts the engine:
+the worker pool retires its processes, and the next node call picks up the new
+packages.
+"""
+
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.concurrency import run_in_threadpool
+
+from app.api.deps import FlowControllerDep, WorkerPoolDep, get_current_user
+from app.flow import modules
+from app.flow.schemas import ApplyRequest, ApplyResult, ModulesInfo
+
+router = APIRouter(
+    prefix="/modules", tags=["modules"], dependencies=[Depends(get_current_user)]
+)
+
+
+@router.get("/", response_model=ModulesInfo)
+async def read_modules(controller: FlowControllerDep) -> Any:
+    """What node code can import, and whether it matches the manifest."""
+    return await run_in_threadpool(modules.info, controller.store)
+
+
+@router.post("/apply", response_model=ApplyResult)
+async def apply_modules(
+    body: ApplyRequest, controller: FlowControllerDep, pool: WorkerPoolDep
+) -> Any:
+    """Install exactly these requirements, then hand them to the workers.
+
+    A manifest that does not resolve changes nothing: the venv is left as it
+    was and the stored manifest is only written once the install succeeded.
+    """
+    ok, output = await run_in_threadpool(modules.sync, body.requirements)
+    if not ok:
+        raise HTTPException(
+            status_code=400,
+            detail=output or "These requirements could not be installed",
+        )
+    await run_in_threadpool(controller.store.write_requirements, body.requirements)
+    # Retire the workers first, so the rebuild compiles every node against the
+    # packages that were just installed — a node that could not import one is
+    # the reason this was called, and it stays red until it is built again.
+    pool.respawn_all()
+    await controller.reload()
+    return ApplyResult(ok=True, output=output)