Run python nodes out of process, with modules of their own

User code no longer execs in the engine. A pool of persistent worker
subprocesses speaks one JSON object per line; the controller installs a
proxy as the node's function, so every execution path funnels through it
and the pipeline is untouched. A crash costs one subprocess, a per-node
timeout is a kill, and cancelling from the canvas is that same kill.

The workers run a venv of the user's own on the data volume, filled from
a pip manifest versioned beside the flows. Applying it retires the
workers and rebuilds, so a package lands without restarting the engine.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt

backend/app/flow/worker_main.py

@@ -0,0 +1,166 @@
+"""One user-code process: compiles a node's source once, then runs it on demand.
+
+This runs under the *user* venv's interpreter, so a node's imports resolve
+against what the Modules page installed rather than against the engine's own
+packages. Nothing of the app is importable here — the file is handed to the
+interpreter by path and is deliberately standard library only.
+
+The protocol is one JSON object per line: a request arrives on stdin, the reply
+goes out on a private duplicate of fd 1 taken before anything else can write to
+it. fd 1 itself is pointed at stderr, so a stray ``write(1, ...)`` — from a
+native library, or a node printing during an import — lands in the server log
+instead of corrupting the reply stream.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import io
+import json
+import os
+import sys
+import traceback
+from collections.abc import Callable
+from types import ModuleType
+from typing import Any, cast
+
+#: Enough of a print to debug with. A node printing in a loop must not fill the
+#: pipe or the reply.
+MAX_LOG = 16 * 1024
+
+
+def load_function(flow: str, node_id: str, code: str) -> Callable[..., Any]:
+    """Compile a node's source and return the function to run.
+
+    A node file defines ``process(...)``; if it defines exactly one public
+    function under another name, that one is used.
+    """
+    digest = hashlib.md5(code.encode()).hexdigest()[:8]
+    module_name = f"_fluksio_node_{flow}_{node_id}_{digest}"
+
+    module = ModuleType(module_name)
+    module.__dict__["__name__"] = module_name
+    sys.modules[module_name] = module
+    exec(compile(code, f"<node {flow}.{node_id}>", "exec"), module.__dict__)
+
+    if callable(getattr(module, "process", None)):
+        return cast(Callable[..., Any], module.process)
+
+    functions = [
+        value
+        for name, value in vars(module).items()
+        if callable(value)
+        and not name.startswith("_")
+        and getattr(value, "__module__", None) == module_name
+    ]
+    if len(functions) == 1:
+        return cast(Callable[..., Any], functions[0])
+    raise ValueError(
+        "Define a function named 'process' — this file has "
+        f"{len(functions)} functions to choose from."
+    )
+
+
+class _Capped(io.StringIO):
+    """Keeps the first ``MAX_LOG`` characters and quietly drops the rest."""
+
+    def write(self, text: str) -> int:
+        room = MAX_LOG - self.tell()
+        if room > 0:
+            super().write(text[:room])
+        return len(text)
+
+
+def _short_error(exc: BaseException) -> str:
+    """One line a node author can act on — the engine's rule, applied here."""
+    if isinstance(exc, SyntaxError):
+        # Its own message already names the compiled file, which is noise here.
+        return f"{type(exc).__name__}: {exc.msg} (line {exc.lineno})"
+
+    frames = [
+        frame
+        for frame in traceback.extract_tb(exc.__traceback__)
+        if frame.filename.startswith("<node ")
+    ]
+    where = f" (line {frames[-1].lineno})" if frames else ""
+    return f"{type(exc).__name__}: {exc}{where}"
+
+
+def _node_traceback(exc: BaseException) -> str:
+    """The traceback from the node's own code onward; the rest is this loop."""
+    frames = traceback.extract_tb(exc.__traceback__)
+    start = next(
+        (i for i, frame in enumerate(frames) if frame.filename.startswith("<node ")),
+        0,
+    )
+    return "".join(
+        ["Traceback (most recent call last):\n"]
+        + traceback.format_list(frames[start:])
+        + traceback.format_exception_only(type(exc), exc)
+    )
+
+
+def _handle(request: dict[str, Any], cache: dict[tuple[str, str], Any]) -> Any:
+    """Compile if needed, then run — the part that may raise the user's error."""
+    flow, node = request["flow"], request["node"]
+    source = request.get("source") or ""
+    digest = hashlib.md5(source.encode()).hexdigest()
+
+    cached = cache.get((flow, node))
+    if cached is None or cached[0] != digest:
+        cache[(flow, node)] = (digest, load_function(flow, node, source))
+    function = cache[(flow, node)][1]
+
+    if request["op"] == "compile":
+        return None
+
+    result = function(
+        **(request.get("kwargs") or {}), params=request.get("params") or {}
+    )
+    try:
+        json.dumps(result)
+    except (TypeError, ValueError):
+        # The typed-message contract only carries JSON, and a node running out
+        # of process is where that stops being a formality.
+        raise ValueError(
+            f"returned {type(result).__name__}, which cannot be sent back as "
+            "JSON — return numbers, strings, booleans, lists or dicts."
+        ) from None
+    return result
+
+
+def main() -> None:
+    rpc = os.fdopen(os.dup(1), "w")
+    # Everything the node writes to the real stdout now goes to the server log.
+    os.dup2(2, 1)
+
+    cache: dict[tuple[str, str], Any] = {}
+    for line in sys.stdin:
+        if not line.strip():
+            continue
+        request = json.loads(line)
+        captured = _Capped()
+        response: dict[str, Any] = {"id": request.get("id")}
+        try:
+            with (
+                contextlib.redirect_stdout(captured),
+                contextlib.redirect_stderr(captured),
+            ):
+                response["result"] = _handle(request, cache)
+            response["ok"] = True
+        except Exception as exc:
+            response["ok"] = False
+            response["error"] = {
+                "type": type(exc).__name__,
+                "message": str(exc),
+                "short": _short_error(exc),
+                "traceback": _node_traceback(exc),
+            }
+        response["logs"] = captured.getvalue()
+        rpc.write(json.dumps(response) + "\n")
+        rpc.flush()
+
+
+if __name__ == "__main__":
+    main()