app/backend/app/flow/worker_main.py

"""One user-code process: compiles a node's source once, then runs it on demand.

This runs under the *user* venv's interpreter, so a node's imports resolve
against what the Modules page installed rather than against the engine's own
packages. Nothing of the app is importable here — the file is handed to the
interpreter by path and is deliberately standard library only.

The protocol is one JSON object per line: a request arrives on stdin, the reply
goes out on a private duplicate of fd 1 taken before anything else can write to
it. fd 1 itself is pointed at stderr, so a stray ``write(1, ...)`` — from a
native library, or a node printing during an import — lands in the server log
instead of corrupting the reply stream.
"""

from __future__ import annotations

import contextlib
import hashlib
import io
import json
import os
import sys
import traceback
from collections.abc import Callable
from types import ModuleType
from typing import Any, cast

#: Enough of a print to debug with. A node printing in a loop must not fill the
#: pipe or the reply.
MAX_LOG = 16 * 1024


def load_function(flow: str, node_id: str, code: str) -> Callable[..., Any]:
    """Compile a node's source and return the function to run.

    A node file defines ``process(...)``; if it defines exactly one public
    function under another name, that one is used.
    """
    digest = hashlib.md5(code.encode()).hexdigest()[:8]
    module_name = f"_fluksio_node_{flow}_{node_id}_{digest}"

    module = ModuleType(module_name)
    module.__dict__["__name__"] = module_name
    sys.modules[module_name] = module
    exec(compile(code, f"<node {flow}.{node_id}>", "exec"), module.__dict__)

    if callable(getattr(module, "process", None)):
        return cast(Callable[..., Any], module.process)

    functions = [
        value
        for name, value in vars(module).items()
        if callable(value)
        and not name.startswith("_")
        and getattr(value, "__module__", None) == module_name
    ]
    if len(functions) == 1:
        return cast(Callable[..., Any], functions[0])
    raise ValueError(
        "Define a function named 'process' — this file has "
        f"{len(functions)} functions to choose from."
    )


class _Capped(io.StringIO):
    """Keeps the first ``MAX_LOG`` characters and quietly drops the rest."""

    def write(self, text: str) -> int:
        room = MAX_LOG - self.tell()
        if room > 0:
            super().write(text[:room])
        return len(text)


def _short_error(exc: BaseException) -> str:
    """One line a node author can act on — the engine's rule, applied here."""
    if isinstance(exc, SyntaxError):
        # Its own message already names the compiled file, which is noise here.
        return f"{type(exc).__name__}: {exc.msg} (line {exc.lineno})"

    frames = [
        frame
        for frame in traceback.extract_tb(exc.__traceback__)
        if frame.filename.startswith("<node ")
    ]
    where = f" (line {frames[-1].lineno})" if frames else ""
    return f"{type(exc).__name__}: {exc}{where}"


def _node_traceback(exc: BaseException) -> str:
    """The traceback from the node's own code onward; the rest is this loop."""
    frames = traceback.extract_tb(exc.__traceback__)
    start = next(
        (i for i, frame in enumerate(frames) if frame.filename.startswith("<node ")),
        0,
    )
    return "".join(
        ["Traceback (most recent call last):\n"]
        + traceback.format_list(frames[start:])
        + traceback.format_exception_only(type(exc), exc)
    )


def _handle(request: dict[str, Any], cache: dict[tuple[str, str], Any]) -> Any:
    """Compile if needed, then run — the part that may raise the user's error."""
    flow, node = request["flow"], request["node"]
    source = request.get("source") or ""
    digest = hashlib.md5(source.encode()).hexdigest()

    cached = cache.get((flow, node))
    if cached is None or cached[0] != digest:
        cache[(flow, node)] = (digest, load_function(flow, node, source))
    function = cache[(flow, node)][1]

    if request["op"] == "compile":
        return None

    result = function(
        **(request.get("kwargs") or {}), params=request.get("params") or {}
    )
    try:
        json.dumps(result)
    except (TypeError, ValueError):
        # The typed-message contract only carries JSON, and a node running out
        # of process is where that stops being a formality.
        raise ValueError(
            f"returned {type(result).__name__}, which cannot be sent back as "
            "JSON — return numbers, strings, booleans, lists or dicts."
        ) from None
    return result


def main() -> None:
    rpc = os.fdopen(os.dup(1), "w")
    # Everything the node writes to the real stdout now goes to the server log.
    os.dup2(2, 1)

    cache: dict[tuple[str, str], Any] = {}
    for line in sys.stdin:
        if not line.strip():
            continue
        request = json.loads(line)
        captured = _Capped()
        response: dict[str, Any] = {"id": request.get("id")}
        try:
            with (
                contextlib.redirect_stdout(captured),
                contextlib.redirect_stderr(captured),
            ):
                response["result"] = _handle(request, cache)
            response["ok"] = True
        except Exception as exc:
            response["ok"] = False
            response["error"] = {
                "type": type(exc).__name__,
                "message": str(exc),
                "short": _short_error(exc),
                "traceback": _node_traceback(exc),
            }
        response["logs"] = captured.getvalue()
        rpc.write(json.dumps(response) + "\n")
        rpc.flush()


if __name__ == "__main__":
    main()