Run node code on the venv Fluksio was installed into

The workflow this serves: make a venv, install what you work with, then `pip
install fluksio` into the same one. Building a second environment beside it
was exactly wrong — the packages the nodes need are already here, and the
Modules screen was asking for them a second time.

`NODE_VENV=auto` (the default) adopts that venv. It declines in the three
cases where adopting would be wrong: `managed` says otherwise, a managed venv
already exists and may hold packages somebody installed on purpose, or the
engine is not running from a venv at all. The images set `managed`, since the
venv in them holds the app and nothing of anybody else's.

An adopted venv is never written to. `uv pip sync` makes a venv hold exactly
the manifest, so pointed at somebody's own environment it uninstalls their
work and the engine with it — `sync()` refuses outright and `reconcile()`
returns before it can be called at startup, which is where that would have
happened first. The Modules screen lists what is installed and drops its
editor; `pip` is how that environment changes.

`fluksio serve` now names the interpreter node code runs on, which is the
thing a data scientist most needs to know at that moment. `fluksio-worker`
already defaulted `--python` to its own interpreter, so a GPU box works the
same way — that was only ever undocumented.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ue1tkFWB1bcGy3aWhCKpU

backend/fluksio/cli.py

@@ -191,11 +191,24 @@ def cmd_serve(args: argparse.Namespace) -> int:
 
     import uvicorn
 
+    from fluksio.flow import modules
     from fluksio.main import app
 
     config = cloud_config.load()
     _say(f"Fluksio {fluksio.__version__} — data in {data_dir}")
     _say(f"  API      http://{args.host}:{args.port}{'/api/v1'}")
+    # Which environment node code imports from is the thing a data scientist
+    # most needs to know at this moment, and the answer differs depending on
+    # how Fluksio was installed. Saying it costs one line.
+    if modules.adopted() is not None:
+        _say(f"  Nodes    {modules.venv_python()}")
+        _say("           your environment, adopted. Add packages with pip.")
+    else:
+        # Not `venv_python()`: the managed venv is built when the engine comes
+        # up, which is after this prints, and until then that would answer with
+        # whatever interpreter happens to be running this.
+        _say(f"  Nodes    {modules.venv_dir() / 'bin' / 'python'}")
+        _say("           a venv of its own; the Modules screen installs into it.")
     if config is not None:
         _say(f"  Portal   {config.portal_url}, installation {config.installation_id}")
         _say("  The dashboard is served by the portal; nothing is served here.")

backend/fluksio/core/config.py

@@ -70,6 +70,12 @@ class Settings(BaseSettings):
     # alerting is: where a screen hangs is the deployment's concern rather than
     # any one dashboard's.
     PANELS_FILE: Path = Path("flow-data/panels.json")
+    # Which interpreter node code runs on. "auto" adopts the venv the engine
+    # was installed into, when it was installed into one and there is no venv
+    # of its own to lose — which is the `pip install fluksio` beside your own
+    # packages case. "managed" always builds a separate one, which is what a
+    # container wants. A path names an interpreter outright.
+    NODE_VENV: str = "auto"
     # The MCP endpoint, and the OAuth server agents authenticate against. Off
     # until someone asks for it: it opens client registration to the network.
     MCP_ENABLED: bool = False

backend/fluksio/flow/modules.py

@@ -9,6 +9,15 @@ and the worker processes that import them need nothing from the app.
 ``uv pip sync`` rather than install, so a line taken out of the manifest is
 uninstalled. The manifest is written only after a sync succeeds, which is all
 the rollback a failed resolve needs.
+
+None of that applies to an *adopted* venv. A data scientist makes a venv,
+installs what they work with, and then installs Fluksio into it too — at which
+point building a second environment beside it is exactly wrong: the packages
+the nodes need are already here. So when the engine is running from a venv of
+somebody else's, node code runs on it. That venv is theirs: `uv pip sync` is
+never pointed at it, because sync means "hold exactly this" and would uninstall
+their work along with the engine. It is read-only here, and `pip` is how they
+change it.
 """
 
 from __future__ import annotations
@@ -51,6 +60,39 @@ def uv_bin() -> str:
     return shutil.which("uv") or "uv"
 
 
+def adopted() -> Path | None:
+    """The venv this engine was installed into, when node code should use it.
+
+    ``None`` means the managed venv — one the engine builds and owns. The three
+    ways that is the answer, in order:
+
+    * ``NODE_VENV=managed`` says so. A container sets this: its venv holds the
+      app and nothing of anybody's, so there is nothing to adopt.
+    * There is already a managed venv. It may have packages in it that
+      somebody installed on purpose, and an upgrade must not take them away.
+    * The engine is not running from a venv at all.
+
+    Anything else — ``pip install fluksio`` into the environment you already
+    work in — is the case this exists for.
+    """
+    setting = settings.NODE_VENV.strip()
+    if setting == "managed":
+        return None
+    if setting and setting != "auto":
+        return Path(setting)
+    if (VENV_DIR / "bin" / "python").exists():
+        return None
+    prefix = Path(sys.prefix)
+    if prefix == Path(sys.base_prefix) or prefix == VENV_DIR:
+        return None
+    return prefix
+
+
+def venv_dir() -> Path:
+    """Whichever venv node code runs from, adopted or managed."""
+    return adopted() or VENV_DIR
+
+
 def venv_python() -> str:
     """The interpreter node code runs on.
 
@@ -58,6 +100,12 @@ def venv_python() -> str:
     could not build one still runs python nodes, it just cannot add packages
     to them.
     """
+    root = adopted()
+    if root is not None:
+        # NODE_VENV may name a venv or the interpreter inside one; both are
+        # useful things to be handed, and they are told apart by shape.
+        inner = root / "bin" / "python"
+        return str(inner if inner.exists() or root.is_dir() else root)
     path = VENV_DIR / "bin" / "python"
     return str(path) if path.exists() else sys.executable
 
@@ -73,6 +121,8 @@ def _digest(requirements: str) -> str:
 
 def ensure_venv() -> None:
     """Create the venv if it is missing or its interpreter has gone."""
+    if adopted() is not None:
+        return
     if (VENV_DIR / "bin" / "python").exists():
         return
     VENV_DIR.parent.mkdir(parents=True, exist_ok=True)
@@ -100,6 +150,16 @@ def sync(requirements: str) -> tuple[bool, str]:
     other — the caller answers 400 with what came back, which is the only thing
     a person can act on.
     """
+    if adopted() is not None:
+        # The one thing this must never do. `uv pip sync` makes a venv hold
+        # exactly the manifest, so pointed at somebody's own environment it
+        # uninstalls their packages — and the engine with them.
+        return False, (
+            "These packages are not Fluksio's to install: node code runs on "
+            f"{venv_python()}, the environment Fluksio itself was installed "
+            "into. Install into it with pip or uv, or set NODE_VENV=managed "
+            "for a venv the engine owns."
+        )
     manifest = ""
     try:
         ensure_venv()
@@ -143,6 +203,12 @@ def reconcile(store: FlowStore) -> None:
     import error each time they execute.
     """
     try:
+        root = adopted()
+        if root is not None:
+            # Nothing to reconcile: the environment is somebody else's, and
+            # bringing it "in line with the manifest" would empty it.
+            logger.info("Node code runs on the adopted venv at %s", root)
+            return
         # First, and unconditionally: the workers are started against this
         # interpreter, and it has to be the venv's one before anything is
         # installed into it, not after.
@@ -162,15 +228,20 @@ def reconcile(store: FlowStore) -> None:
 def info(store: FlowStore) -> ModulesInfo:
     """What is installed, what was asked for, and whether the two agree."""
     requirements = store.read_requirements()
-    config = VENV_DIR / "pyvenv.cfg"
+    root = venv_dir()
+    is_adopted = adopted() is not None
+
     version = ""
+    config = root / "pyvenv.cfg"
     if config.exists():
         for line in config.read_text().splitlines():
             if line.startswith("version"):
                 version = line.split("=", 1)[1].strip()
+    if not version:
+        version = ".".join(str(part) for part in sys.version_info[:3])
 
     packages: list[ModulePackage] = []
-    site = sorted(VENV_DIR.glob("lib/python*/site-packages"))
+    site = sorted(root.glob("lib/python*/site-packages"))
     if site:
         packages = sorted(
             (
@@ -182,13 +253,20 @@ def info(store: FlowStore) -> ModulesInfo:
 
     return ModulesInfo(
         python_version=version,
-        venv_path=str(VENV_DIR),
+        venv_path=str(root),
         requirements=requirements,
         packages=packages,
+        # An adopted venv is never "out of step": the manifest does not
+        # describe it, so there is nothing for it to disagree with.
         applied=(
-            _marker().read_text() == _digest(requirements)
-            if _marker().exists()
-            # Nothing asked for and nothing installed is already in step.
-            else not requirements.strip()
+            True
+            if is_adopted
+            else (
+                _marker().read_text() == _digest(requirements)
+                if _marker().exists()
+                # Nothing asked for and nothing installed is already in step.
+                else not requirements.strip()
+            )
         ),
+        adopted=is_adopted,
     )

backend/fluksio/flow/schemas.py

@@ -242,6 +242,10 @@ class ModulesInfo(BaseModel):
     packages: list[ModulePackage] = Field(default_factory=list)
     #: Whether what is installed matches the manifest.
     applied: bool = False
+    #: True when node code runs on the venv Fluksio was installed into rather
+    #: than one the engine built. That venv belongs to whoever made it, so the
+    #: manifest does not describe it and nothing here installs into it.
+    adopted: bool = False
 
 
 class ApplyRequest(BaseModel):