Merge branch 'main' of git.stroblme.de:Fluksio/app

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01C5H4uLCCpsbipL1R7WKCee

backend/fluksio/api/main.py

@@ -15,6 +15,7 @@ from fluksio.api.routes import (
     panels,
     private,
     runs,
+    search,
     secrets,
     users,
     utils,
@@ -38,6 +39,7 @@ api_router.include_router(observability.router)
 api_router.include_router(runs.router)
 api_router.include_router(artifacts.router)
 api_router.include_router(workers.router)
+api_router.include_router(search.router)
 # Remote access through a portal. Always mounted; with no enrolment the
 # endpoints only ever report that there is none.
 api_router.include_router(cloud.router)

backend/fluksio/api/routes/observability.py

@@ -176,10 +176,16 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
     paused = set(controller.paused_flows())
     entries = list(controller.loaded.values())
     errored = [e for e in entries if e.status is NodeStatus.ERROR]
+    unhealthy = [e for e in entries if e.health == "down"]
     if quarantined:
         problems.append(f"{len(quarantined)} flow(s) quarantined")
     if errored:
         problems.append(f"{len(errored)} node(s) failed to load")
+    if unhealthy:
+        problems.append(
+            f"{len(unhealthy)} node(s) down: "
+            f"{', '.join(sorted(e.id for e in unhealthy))}"
+        )
 
     # What the canvas flags on a flow — a dependency loop, an input nothing
     # feeds — stops that flow running just as surely as a node that will not
@@ -219,7 +225,11 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
             "quarantined": len(quarantined),
             "invalid": len(invalid),
         },
-        nodes={"total": len(entries), "error": len(errored)},
+        nodes={
+            "total": len(entries),
+            "error": len(errored),
+            "unhealthy": len(unhealthy),
+        },
         queue=queue,
         loop_lag=(
             watchdog.snapshot()

backend/fluksio/api/routes/runs.py

@@ -8,20 +8,22 @@ or to listen on the flow socket, which carries its start and finish.
 import csv
 import io
 import json
+import time
 from collections.abc import Iterator
 from datetime import UTC, datetime
 from itertools import groupby
 from typing import Any, Literal
 
-from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi import APIRouter, Depends, HTTPException, Request, Response
 from fastapi.concurrency import run_in_threadpool
 from fastapi.responses import StreamingResponse
 from pydantic import BaseModel, Field, model_validator
-from sqlalchemy import func
+from sqlalchemy import delete, func
 from sqlalchemy import select as sa_select
 from sqlmodel import Session, col, select
 
 from fluksio.api.deps import CurrentUser, SessionDep, get_current_user
+from fluksio.flow.events import event_bus
 from fluksio.flow.messages import requalify
 from fluksio.flow.runs import RunRejected, RunService, new_run_id
 from fluksio.flow.store import FlowNotFound
@@ -102,6 +104,10 @@ class ArtifactRow(BaseModel):
     digest: str
     size: int
     media_type: str
+    #: What it was called where it was written, so something downloading it can
+    #: give it that name back rather than the message's. Absent when the node
+    #: never said one.
+    filename: str | None = None
 
 
 class RunRow(BaseModel):
@@ -420,25 +426,6 @@ def _dig(record: dict[str, Any], path: str) -> Any:
     return value
 
 
-def _varying(runs: list[Run]) -> list[str]:
-    """The inputs that differ across these runs — the axis of a sweep.
-
-    What a reader comparing arms wants as columns, compared leaf by leaf: two
-    configurations differing in one field give that field as a column rather
-    than two blobs that are not the same. Under two runs nothing can differ,
-    and a table of one run with none of its inputs in it is not worth reading,
-    so all of them are kept.
-    """
-    keys = sorted({path for run in runs for path, _ in _leaves(run.params)})
-    if len(runs) < 2:
-        return keys
-    return [
-        key
-        for key in keys
-        if len({json.dumps(_dig(run.params, key), sort_keys=True) for run in runs}) > 1
-    ]
-
-
 def _scored(runs: list[Run]) -> list[str]:
     """A run's final numbers: every number its declared outputs carry, however
     deep it sits. A flag is not a number, and neither is a label."""
@@ -555,9 +542,9 @@ def export_runs(
     """One row per run: what it was given, what it scored, what code it ran.
 
     The arm-comparison table. Inputs are columns rather than one JSON blob —
-    by default the ones that vary across the selection, which is the sweep
-    axis; ``params`` names them instead. ``metrics`` narrows the final numbers
-    to a few of a run's declared outputs.
+    every input the selection recorded, so the schema is the same whichever
+    runs are asked for; ``params`` narrows it. ``metrics`` narrows the final
+    numbers to a few of a run's declared outputs.
 
     Both take dotted paths into a record a node returned:
     ``metrics=final_metrics.train_loss,test_metrics.known.perfect`` selects
@@ -565,7 +552,9 @@ def export_runs(
     depth.
     """
     runs = _selected(session, flow, status, group, ids, since, until)
-    inputs = [part for part in params.split(",") if part] or _varying(runs)
+    inputs = [part for part in params.split(",") if part] or sorted(
+        {path for run in runs for path, _ in _leaves(run.params)}
+    )
     scores = [part for part in metrics.split(",") if part] or _scored(runs)
     columns = [
         *RUN_COLUMNS,
@@ -613,6 +602,56 @@ async def cancel_run(run_id: str, request: Request, session: SessionDep) -> Any:
     return run
 
 
+@router.delete("/{run_id}", status_code=204)
+def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response:
+    """Forget a run and everything hanging off it.
+
+    The same four statements ``_forget_runs`` uses when a flow goes: the run
+    tables carry a plain string ``run_id`` and no foreign key, so nothing
+    cascades on its own. ``flow_run``, ``metric_minute`` and ``engine_event``
+    stay — they are the observability record and are pruned on their own window.
+
+    A live run is refused rather than raced: the driver writes its nodes back
+    when it finishes, and those rows would arrive for a run that no longer
+    exists. Cancel it first.
+
+    Two things this costs, both deliberate. ``RunNode.outputs`` *is* the stage
+    cache, so a later run loses hits this one would have served. And a node
+    restored from this run points here through ``cached_from`` — ``_series``
+    already reads a missing source as an empty curve, which is what ``NO_CURVE``
+    explains on the screen. The artifact bytes need no help: ``sweep_artifacts``
+    keeps whatever a ``run_artifact`` row or a live message still names, so
+    dropping the rows is enough and the hourly sweep reclaims the blobs.
+    """
+    run = session.get(Run, run_id)
+    if run is None:
+        raise HTTPException(status_code=404, detail="No such run")
+    if run.status in ("running", "queued"):
+        raise HTTPException(
+            status_code=409,
+            detail=(
+                f"Run {run_id} is {run.status}. Cancel it, or wait for it to "
+                "finish, before deleting it."
+            ),
+        )
+    flow = run.flow
+    session.execute(delete(RunNode).where(col(RunNode.run_id) == run_id))
+    session.execute(delete(RunMetric).where(col(RunMetric.run_id) == run_id))
+    session.execute(delete(RunArtifact).where(col(RunArtifact.run_id) == run_id))
+    session.execute(delete(Run).where(col(Run.id) == run_id))
+    session.commit()
+    event_bus.publish(
+        {
+            "type": "audit",
+            "action": f"deleted run {run_id}",
+            "flow": flow,
+            "user": user.email,
+            "ts": time.time(),
+        }
+    )
+    return Response(status_code=204)
+
+
 def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
     """A run's numbers, including the ones a cached node points at.
 

backend/fluksio/api/routes/search.py

@@ -0,0 +1,156 @@
+"""One index of everything in this installation worth jumping to by name."""
+
+from typing import Any, Literal
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.concurrency import run_in_threadpool
+from pydantic import BaseModel
+
+from fluksio.api.deps import (
+    CurrentUser,
+    DashboardStoreDep,
+    FlowControllerDep,
+    get_current_user,
+)
+from fluksio.api.routes.alerts import read_config as read_alerts_config
+from fluksio.flow import modules, panels
+from fluksio.flow.controller import FlowController
+from fluksio.flow.dashboards import DashboardNotFound, DashboardStore
+from fluksio.flow.secrets import get_secrets
+from fluksio.flow.store import FlowNotFound
+
+# A wall panel never reaches this route: ``deps._panel_may`` is a whitelist that
+# ends in a 403, and a whole-installation index is the opposite of what a screen
+# on a wall is allowed to read.
+router = APIRouter(
+    prefix="/search", tags=["search"], dependencies=[Depends(get_current_user)]
+)
+
+Category = Literal[
+    "flow",
+    "node",
+    "dashboard",
+    "widget",
+    "panel",
+    "secret",
+    "module",
+    "worker",
+    "alert",
+]
+
+
+class SearchEntry(BaseModel):
+    """One thing somebody might be looking for.
+
+    Deliberately not a route: where a category lands is the frontend's business,
+    and it already owns the router. This says what the thing is and what it is
+    called, which is all the matching needs.
+    """
+
+    category: Category
+    #: The id the frontend routes on.
+    name: str
+    #: Human title, often empty — a flow is usually only its name.
+    title: str = ""
+    #: The flow a node sits in, or the dashboard a widget sits on.
+    parent: str = ""
+    #: Node type, widget type, channel kind.
+    kind: str = ""
+
+
+def _build(
+    controller: FlowController, dashboards: DashboardStore, hub: Any, secrets: bool
+) -> list[SearchEntry]:
+    """Read every store once. Blocking: disk and git throughout.
+
+    # ponytail: rebuilt per call. Key it on ``controller.store.revision`` if a
+    # store large enough to feel it ever shows up in a profile.
+    """
+    entries: list[SearchEntry] = []
+
+    for name in controller.store.list_flows():
+        try:
+            flow = controller.store.read_flow(name, draft=True)
+        except FlowNotFound:
+            continue
+        entries.append(SearchEntry(category="flow", name=flow.name, title=flow.title))
+        entries.extend(
+            SearchEntry(
+                category="node",
+                name=node.id,
+                title=node.title,
+                parent=flow.name,
+                kind=node.type,
+            )
+            for node in flow.nodes
+        )
+
+    for summary in dashboards.list():
+        try:
+            dashboard = dashboards.read(summary.name, draft=True)
+        except DashboardNotFound:
+            continue
+        entries.append(
+            SearchEntry(
+                category="dashboard", name=dashboard.name, title=dashboard.title
+            )
+        )
+        entries.extend(
+            SearchEntry(
+                category="widget",
+                name=widget.id,
+                title=widget.title,
+                parent=dashboard.name,
+                kind=widget.type,
+            )
+            for widget in dashboard.widgets
+        )
+
+    entries.extend(
+        SearchEntry(category="panel", name=panel.id, title=panel.title)
+        for panel in panels.read_config().panels
+    )
+
+    if secrets:
+        entries.extend(
+            SearchEntry(category="secret", name=name) for name in get_secrets().list()
+        )
+
+    entries.extend(
+        SearchEntry(category="module", name=package.name, kind=package.version)
+        for package in modules.info(controller.store).packages
+    )
+
+    entries.extend(
+        SearchEntry(category="worker", name=worker.name)
+        for worker in (hub.workers() if hub is not None else [])
+    )
+
+    entries.extend(
+        SearchEntry(category="alert", name=channel.name, kind=channel.kind)
+        for channel in read_alerts_config().channels
+    )
+
+    return entries
+
+
+@router.get("/", response_model=list[SearchEntry])
+async def read_search_index(
+    current_user: CurrentUser,
+    request: Request,
+    controller: FlowControllerDep,
+    dashboards: DashboardStoreDep,
+) -> Any:
+    """Everything searchable, for the client to match against as it is typed.
+
+    The whole index rather than a query: it is a few hundred short rows for an
+    installation of any ordinary size, so one fetch when the panel opens beats a
+    round trip per keystroke — and the client already has a matcher.
+
+    Secrets are named only to a superuser, which is who ``/secrets`` answers to.
+    """
+    # Absent when remote workers are switched off — not a reason to fail a search.
+    hub = getattr(request.app.state, "worker_hub", None)
+    return await run_in_threadpool(
+        _build, controller, dashboards, hub, current_user.is_superuser
+    )

backend/fluksio/cli.py

@@ -17,10 +17,12 @@ from __future__ import annotations
 
 import argparse
 import copy
+import json
 import os
 import secrets
 import socket
 import sys
+from collections.abc import Callable
 from pathlib import Path
 from typing import Any
 
@@ -80,6 +82,39 @@ def _mention_other_installation(data_dir: Path) -> None:
     _say("           `fluksio serve --global` runs that one instead.")
 
 
+def _mention_undeclared_cards() -> None:
+    """Say when a stored flow asks for a card this engine does not have.
+
+    Cards are declared rather than detected, so an engine told nothing has
+    none — and a node asking for one is clamped to zero and runs beside every
+    other, which on a GPU is the deadlock the declaration exists to prevent.
+    The placer says so once it happens, into the log; this says it while
+    somebody is still reading the terminal.
+    """
+    from fluksio.core.config import settings
+
+    if settings.FLOW_GPUS:
+        return
+    from fluksio.flow.runs import required_resources
+    from fluksio.flow.store import FlowStore
+
+    store = FlowStore(settings.FLOWS_DIR)
+    asking = []
+    for name in store.list_flows():
+        try:
+            needs = required_resources(store.read_flow(name))
+        except Exception:
+            # A flow that will not parse is the engine's to complain about.
+            continue
+        if (needs or {}).get("gpus"):
+            asking.append(name)
+    if not asking:
+        return
+    named = ", ".join(sorted(asking)[:3]) + (" …" if len(asking) > 3 else "")
+    _say(f"  Cards    0 declared, but {named} asks for one.")
+    _say("           Nothing detects them: `--gpus N` says how many are here.")
+
+
 def _warn_if_networked(path: Path) -> None:
     """A cluster's $HOME is often NFS, and SQLite's WAL does not work there."""
     try:
@@ -228,9 +263,26 @@ CONCURRENCY_FLAGS = {
     "max_workers": "FLOW_MAX_WORKERS",
     "max_cascades": "FLOW_MAX_CASCADES",
     "max_runs": "FLOW_MAX_RUNS",
+    "gpus": "FLOW_GPUS",
 }
 
 
+def _at_least(minimum: int) -> Callable[[str], int]:
+    """A flag's value, checked here rather than by the settings.
+
+    These are written into the environment before the settings are built, so a
+    number they refuse dies inside a pydantic import with no flag named in it.
+    """
+
+    def parse(text: str) -> int:
+        value = int(text)
+        if value < minimum:
+            raise argparse.ArgumentTypeError(f"is {value}, needs at least {minimum}")
+        return value
+
+    return parse
+
+
 #: What `serve` listens on when nobody says. Taken often enough — another
 #: engine, another framework's dev server — that dying on it is the first
 #: thing a zero-config start would hit.
@@ -240,6 +292,85 @@ DEFAULT_PORT = 8000
 PORT_TRIES = 20
 
 
+#: Where a serving engine records itself, beside the data it is serving. Read
+#: to tell "this directory's engine is already up" from "something else has the
+#: port", which are the two ways a second `serve` fails to be what was wanted.
+PIDFILE = "serve.pid"
+
+
+def write_pidfile(data_dir: Path, port: int) -> Path:
+    """Record which process is serving this directory, and where."""
+    path = data_dir / PIDFILE
+    path.write_text(json.dumps({"pid": os.getpid(), "port": port}))
+    return path
+
+
+def read_pidfile(data_dir: Path) -> dict[str, int] | None:
+    """The engine serving this directory, if one still is.
+
+    A process killed outright leaves the file behind, so the pid is checked
+    rather than believed — a stale file is the same as no file.
+    """
+    try:
+        record = json.loads((data_dir / PIDFILE).read_text())
+        pid, port = int(record["pid"]), int(record["port"])
+    except (OSError, ValueError, KeyError, TypeError):
+        return None
+    try:
+        os.kill(pid, 0)
+    except (OSError, ProcessLookupError):
+        return None
+    return {"pid": pid, "port": port}
+
+
+def _token_for(data_dir: Path) -> str:
+    """The credential this directory's last engine wrote, if it wrote one."""
+    from fluksio.sdk.client import config_path
+
+    try:
+        return str(json.loads(config_path(data_dir).read_text()).get("token", ""))
+    except (OSError, ValueError, AttributeError):
+        return ""
+
+
+def probe_engine(url: str, token: str, client: Any = None) -> str:
+    """Who is on this port: ``ours``, ``foreign``, or ``other``.
+
+    ``ours`` means an engine serving *this* data directory, which is what
+    makes stopping it something this command may offer. The proof is the
+    token: it is signed with this directory's secret key, so an engine that
+    accepts it is one reading this directory's database. A Fluksio belonging
+    to another installation answers the health check and refuses the token,
+    and is only ever named — never stopped from here.
+    """
+    import logging
+
+    import httpx
+
+    # Two requests nobody asked for, on a start that is otherwise quiet until
+    # uvicorn's own banner. httpx logs every one of them at INFO.
+    noisy = logging.getLogger("httpx")
+    was = noisy.level
+    noisy.setLevel(logging.WARNING)
+    http = client or httpx.Client(timeout=2.0)
+    try:
+        health = http.get(f"{url}/api/v1/utils/health-check/")
+        if health.status_code != 200:
+            return "other"
+        # No token at all is asked unauthenticated: `Bearer ` is not a legal
+        # header value, and a Fluksio this directory cannot prove is its own
+        # is foreign — which is the answer that never stops anything.
+        auth = {"Authorization": f"Bearer {token}"} if token else {}
+        answer = http.get(f"{url}/api/v1/observability/summary", headers=auth)
+        return "ours" if answer.status_code == 200 else "foreign"
+    except Exception:
+        return "other"
+    finally:
+        noisy.setLevel(was)
+        if client is None:
+            http.close()
+
+
 def _free_port(host: str, start: int) -> int:
     """The first port from ``start`` that nothing is listening on.
 
@@ -258,6 +389,14 @@ def _free_port(host: str, start: int) -> int:
 
 
 def cmd_serve(args: argparse.Namespace) -> int:
+    # At a terminal this is a dashboard with the engine as a child of it. The
+    # import is here rather than at the top because it is only ever needed on
+    # that path, and `serve` in a container must not pay for it.
+    if not args.plain and sys.stdout.isatty() and sys.stdin.isatty():
+        from fluksio.tui import run_tui
+
+        return run_tui(args)
+
     data_dir = _data_dir(args.data_dir, args.shared)
     for flag, name in CONCURRENCY_FLAGS.items():
         value = getattr(args, flag, None)
@@ -294,15 +433,34 @@ def cmd_serve(args: argparse.Namespace) -> int:
     from fluksio.flow import modules
     from fluksio.main import app
 
+    # The client talks to this engine, and 0.0.0.0 is not an address to talk
+    # to — it is a statement about which interfaces to listen on.
+    reachable = "127.0.0.1" if args.host in ("0.0.0.0", "::", "") else args.host
+
     port = args.port
     if port is None:
         port = _free_port(args.host, DEFAULT_PORT)
         if port != DEFAULT_PORT:
-            _say(f"Port {DEFAULT_PORT} is in use; serving on {port} instead.")
+            # Moving off the port quietly makes starting a second engine for
+            # one directory look like it worked. Two of them on one SQLite
+            # file is not a supported shape — two *installations* on one
+            # machine is — so the one already up is named instead.
+            url = f"http://{reachable}:{DEFAULT_PORT}"
+            who = probe_engine(url, _token_for(data_dir))
+            if who == "ours":
+                running = read_pidfile(data_dir)
+                where = f" (pid {running['pid']})" if running else ""
+                _say(f"An engine for {data_dir} is already serving at {url}{where}.")
+                _say("  fluksio status talks to it; stop it to start another.")
+                return 0
+            if who == "foreign":
+                _say(
+                    f"Port {DEFAULT_PORT} holds another installation's Fluksio; "
+                    f"serving on {port} instead."
+                )
+            else:
+                _say(f"Port {DEFAULT_PORT} is in use; serving on {port} instead.")
 
-    # The client talks to this engine, and 0.0.0.0 is not an address to talk
-    # to — it is a statement about which interfaces to listen on.
-    reachable = "127.0.0.1" if args.host in ("0.0.0.0", "::", "") else args.host
     url = f"http://{reachable}:{port}"
     token_path = _sign_in(admin_id, url, data_dir)
 
@@ -329,17 +487,22 @@ def cmd_serve(args: argparse.Namespace) -> int:
         _say("    fluksio enroll <code>")
     _say(f"  Signed in as {admin_email}")
     _say(f"           token in {token_path}")
+    _mention_undeclared_cards()
     _mention_other_installation(data_dir)
 
     # One process: it holds the flow engine, and a second worker would be a
     # second engine — duplicated subscriptions, cron ticks and webhooks.
-    uvicorn.run(
-        app,
-        host=args.host,
-        port=port,
-        log_level=args.log_level,
-        log_config=_log_config(args.log_level),
-    )
+    pidfile = write_pidfile(data_dir, port)
+    try:
+        uvicorn.run(
+            app,
+            host=args.host,
+            port=port,
+            log_level=args.log_level,
+            log_config=_log_config(args.log_level),
+        )
+    finally:
+        pidfile.unlink(missing_ok=True)
     return 0
 
 
@@ -426,25 +589,37 @@ def _parser() -> argparse.ArgumentParser:
     )
     serve.add_argument(
         "--max-runs",
-        type=int,
+        type=_at_least(1),
         default=None,
         metavar="N",
         help="batch runs driven at once (default 4, FLOW_MAX_RUNS)",
     )
     serve.add_argument(
         "--max-cascades",
-        type=int,
+        type=_at_least(1),
         default=None,
         metavar="N",
         help="cascades in flight at once (default 4, FLOW_MAX_CASCADES)",
     )
     serve.add_argument(
         "--max-workers",
-        type=int,
+        type=_at_least(1),
         default=None,
         metavar="N",
         help="python worker processes (default 4, FLOW_MAX_WORKERS)",
     )
+    serve.add_argument(
+        "--plain",
+        action="store_true",
+        help="the log stream rather than the dashboard (the default with no terminal)",
+    )
+    serve.add_argument(
+        "--gpus",
+        type=_at_least(0),
+        default=None,
+        metavar="N",
+        help="GPUs on this machine a node may be given (default 0, FLOW_GPUS)",
+    )
     serve.set_defaults(func=cmd_serve)
 
     enroll = subparsers.add_parser(

backend/fluksio/core/config.py

@@ -9,6 +9,7 @@ from pydantic import (
     BeforeValidator,
     EmailStr,
     HttpUrl,
+    PositiveInt,
     computed_field,
     model_validator,
 )
@@ -97,15 +98,19 @@ class Settings(BaseSettings):
     # rather than a person, and it can refresh unattended.
     MCP_TOKEN_EXPIRE_MINUTES: int = 60
     MCP_REFRESH_EXPIRE_DAYS: int = 30
-    FLOW_MAX_WORKERS: int = 4
+    # The three below are all pool sizes, so 0 says neither "none" nor
+    # "unlimited" — it is a pool that cannot be built and an engine that would
+    # accept no work. Rejected here rather than quietly read as the default,
+    # because a limit somebody set and did not get is the worse surprise.
+    FLOW_MAX_WORKERS: PositiveInt = 4
     # How many cascades may be in flight at once. Sustained throughput is this
     # over the mean cascade time, so an installation whose nodes wait on the
     # network rather than on a CPU wants it higher than the core count.
-    FLOW_MAX_CASCADES: int = 4
+    FLOW_MAX_CASCADES: PositiveInt = 4
     # How many batch runs are driven at once. A different limit from the one
     # above: a run drives a whole graph, and its nodes are bounded by the worker
     # pool rather than by cascade slots. A sweep is what this governs.
-    FLOW_MAX_RUNS: int = 4
+    FLOW_MAX_RUNS: PositiveInt = 4
     # How long a python node may be silent before its worker is killed, unless
     # the node sets its own. 0, the default, disables it: a dead worker still
     # fails fast, and a slow one is left to finish. Set it where silence means

backend/fluksio/flow/connector.py

@@ -86,7 +86,14 @@ class ConnectorNode(Node):
             description="Seconds between polls; 0 polls never.",
         )
 
-    __slots__ = ("config", "_poll_task", "_stop_event", "_last_published", "_artifacts")
+    __slots__ = (
+        "config",
+        "_poll_task",
+        "_stop_event",
+        "_last_published",
+        "_artifacts",
+        "_down",
+    )
 
     def __init__(self, **kwargs: Any) -> None:
         super().__init__(f=self._dispatch, **kwargs)
@@ -95,6 +102,7 @@ class ConnectorNode(Node):
         self._stop_event: asyncio.Event | None = None
         self._last_published: dict[str, Any] = {}
         self._artifacts: ArtifactStore | None = None
+        self._down = False
 
     def _dispatch(self, params: dict[str, Any], **ports: Any) -> dict[str, Any] | None:
         """The scheduler's entry point. Settings are already on ``self.config``."""
@@ -161,11 +169,7 @@ class ConnectorNode(Node):
             return
         self._stop_event.set()
         if self._poll_task is not None:
-            self._poll_task.cancel()
-            try:
-                await self._poll_task
-            except (asyncio.CancelledError, Exception):  # noqa: B014 - shutting down
-                pass
+            await self._cancel_task(self._poll_task)
         self._poll_task = None
         self._stop_event = None
         self._last_published = {}
@@ -175,24 +179,29 @@ class ConnectorNode(Node):
 
         Only changed ports are published: a device polled every few seconds is
         usually saying the same thing, and every publication wakes everything
-        downstream of it.
+        downstream of it. What is remembered is what was *published*, not what
+        the poll returned — a publication that raised is retried next tick
+        rather than counting as said.
         """
         while not (self._stop_event and self._stop_event.is_set()):
             try:
                 values = await self.poll()
-                self.report_health("ok")
                 changed = {
                     port: value
                     for port, value in (values or {}).items()
                     if self._last_published.get(port, object()) != value
                 }
                 if changed:
-                    self._last_published.update(changed)
                     # inject runs the graph, which is blocking work.
                     await asyncio.to_thread(self.inject, changed)
+                    self._last_published.update(changed)
+                self.report_health("ok")
+                self._down = False
             except asyncio.CancelledError:
                 break
             except Exception as exc:
-                logger.warning("Connector '%s' failed to poll: %s", self.id, exc)
+                if not self._down:
+                    logger.warning("Connector '%s' failed to poll: %s", self.id, exc)
+                self._down = True
                 self.report_health("down", f"{type(exc).__name__}: {exc}")
             await asyncio.sleep(self.config.poll_interval)

backend/fluksio/flow/controller.py

@@ -916,12 +916,12 @@ class FlowController:
         placer, pool = self.placer, self.workers
 
         def call(**kwargs: Any) -> Any:
-            with placer.claim(
+            with placer.claim(  # type: ignore[union-attr]
                 wanted, device=device, policy=policy, node=node_id, run=run_id
             ) as (target, allocation):
                 env = derive_env(wanted, allocation)
                 if target.worker is None:
-                    return pool.for_env(env).run(
+                    return pool.for_env(env).run(  # type: ignore[union-attr]
                         owner,
                         local,
                         code,
@@ -1224,8 +1224,31 @@ class FlowController:
             }
         )
 
+    def _health_issues(self, flow: str | None = None) -> list[ValidationIssue]:
+        """Nodes that are running but not working, as issues on their flow.
+
+        Not part of `self.issues`: that list is what a build found, and this is
+        what is happening now. Derived on read, so a node reporting itself well
+        again clears it with nothing to remember.
+        """
+        return [
+            ValidationIssue(
+                code="node_unhealthy",
+                message=(
+                    f"Node '{entry.id.rpartition('.')[2]}' is down: "
+                    f"{entry.health_detail or 'no detail given'}"
+                ),
+                flow=entry.flow,
+                node=entry.id,
+            )
+            for entry in self.loaded.values()
+            if entry.health == "down" and (flow is None or entry.flow == flow)
+        ]
+
     def flow_issues(self, flow: str) -> list[ValidationIssue]:
-        return [issue for issue in self.issues if not issue.flow or issue.flow == flow]
+        return [
+            issue for issue in self.issues if not issue.flow or issue.flow == flow
+        ] + self._health_issues(flow)
 
     def preview(self, name: str) -> Preview:
         """Build a flow's unpublished draft without deploying it.

backend/fluksio/flow/executor.py

@@ -46,6 +46,11 @@ DELAYED_INTERVAL_S = 1.0
 #: whose nodes wait on a network rather than a CPU may want more of them —
 #: `FLOW_MAX_CASCADES` is where that is said.
 MAX_CASCADES = 4
+#: Node threads, unless the service is given a number. Both this and the one
+#: above are taken as written: only ``None`` means "nobody said", so a number
+#: that reached here is one somebody chose, and an unusable one is the pool's
+#: ``ValueError`` rather than a silent 4.
+MAX_WORKERS = 4
 # How long a reload waits for claimed work to finish before rebuilding anyway.
 DRAIN_TIMEOUT_S = 10.0
 # Work waiting in the stream, undelivered. A burst is normal — the pool claims
@@ -68,7 +73,7 @@ class ExecutionService:
     ) -> None:
         self.queue = queue
         self._events = events
-        self.max_cascades = max_cascades or MAX_CASCADES
+        self.max_cascades = MAX_CASCADES if max_cascades is None else max_cascades
         self._pipeline: Pipeline | None = None
         self._stop = threading.Event()
         # Set when a deadline moves closer, so the timer thread stops waiting
@@ -82,7 +87,8 @@ class ExecutionService:
         # Entry ids claimed and still running, under _inflight_lock.
         self._active: set[str] = set()
         self.node_pool = ThreadPoolExecutor(
-            max_workers=max_workers or 4, thread_name_prefix="node"
+            max_workers=MAX_WORKERS if max_workers is None else max_workers,
+            thread_name_prefix="node",
         )
         self._cascade_pool = ThreadPoolExecutor(
             max_workers=self.max_cascades, thread_name_prefix="cascade"

backend/fluksio/flow/nodes/base.py

@@ -212,6 +212,22 @@ class Node:
             return None
         return asyncio.create_task(factory())
 
+    @staticmethod
+    async def _cancel_task(task: asyncio.Task[None]) -> None:
+        """Stop an unsupervised loop and wait for it to be gone.
+
+        `wait` keeps whatever the task raises on its way out to itself, and
+        lets a cancellation aimed at *this* coroutine through — the
+        `except CancelledError` around `await task` it replaces swallowed that,
+        which left whoever asked for the teardown unkillable. The same trap
+        `Supervisor._cancel` documents.
+        """
+        task.cancel()
+        await asyncio.wait([task])
+        if not task.cancelled():
+            # Retrieved so a crash on the way out is not reported at exit.
+            task.exception()
+
     def report_health(self, status: str, detail: str | None = None) -> None:
         """Say how this node's connection is doing: ok, degraded or down."""
         if self._on_health is not None:

backend/fluksio/flow/nodes/delay.py

@@ -211,11 +211,7 @@ class DelayNode(Node):
         self._stop_cron.set()
 
         if self._cron_task is not None:
-            self._cron_task.cancel()
-            try:
-                await self._cron_task
-            except asyncio.CancelledError:
-                pass
+            await self._cancel_task(self._cron_task)
 
         self._cron_task = None
         self._stop_cron = None

backend/fluksio/flow/nodes/influx.py

@@ -14,6 +14,22 @@ from fluksio.flow.nodes.base import Node, NodeResult
 logger = logging.getLogger(__name__)
 
 
+def _influxdb() -> Any:
+    """The client library, which is a `fluksio[server]` extra.
+
+    Imported per use rather than at module level, because the node type is
+    registered at boot and an installation with no InfluxDB behind it should
+    not have to carry the library to start.
+    """
+    try:
+        import influxdb_client
+    except ImportError:
+        raise RuntimeError(
+            "the influxdb node needs the server extra: pip install 'fluksio[server]'"
+        ) from None
+    return influxdb_client
+
+
 class InfluxDbNode(Node):
     """
     InfluxDB node for writing to and reading from InfluxDB.
@@ -58,6 +74,7 @@ class InfluxDbNode(Node):
         - ``bucket`` (str): Bucket name (required)
         - ``write_precision`` (str): Write precision ("ns", "us", "ms", "s"), default "ms"
         - ``query_range`` (str): Default time range for queries, e.g., "-1h", "-24h"
+        - ``timeout`` (float): Deadline for a request in seconds (default: 10.0)
         - ``writes`` (dict): Write configurations keyed by message name, each with:
             - ``measurement`` (str): Measurement name to write to
             - ``field`` (str): Field name to write (default: "value")
@@ -154,6 +171,7 @@ class InfluxDbNode(Node):
         "bucket",
         "write_precision",
         "query_range",
+        "timeout",
         "writes",
         "queries",
         "_write_client",
@@ -169,6 +187,14 @@ class InfluxDbNode(Node):
         bucket: str
         write_precision: str = "ms"
         query_range: str = "-1h"
+        # Bounds every request to the server. Without one the client falls
+        # back to its own default, which no flow can see or change. Held in
+        # seconds like every other node; the client counts in milliseconds.
+        timeout: float = Field(
+            default=10.0,
+            gt=0,
+            description="Give up on a query or write after this many seconds.",
+        )
         # Per-port write and query configuration.
         writes: dict[str, dict[str, Any]] = {}
         queries: dict[str, dict[str, Any]] = {}
@@ -201,6 +227,7 @@ class InfluxDbNode(Node):
         self.bucket = cfg.bucket
         self.write_precision = cfg.write_precision
         self.query_range = cfg.query_range
+        self.timeout = cfg.timeout
         self.writes = cfg.writes
         self.queries = cfg.queries
 
@@ -279,13 +306,18 @@ class InfluxDbNode(Node):
         :returns: ``{"rows": [...], **echo}``.
         :rtype: dict
         """
-        from influxdb_client import InfluxDBClient
+        InfluxDBClient = _influxdb().InfluxDBClient
 
         flux = str(request["flux"])
         echo = {key: value for key, value in request.items() if key != "flux"}
 
         logger.info("Running Flux for node '%s': %s", self.name, flux)
-        with InfluxDBClient(url=self.url, token=self.token, org=self.org) as client:
+        with InfluxDBClient(
+            url=self.url,
+            token=self.token,
+            org=self.org,
+            timeout=int(self.timeout * 1000),
+        ) as client:
             tables = client.query_api().query(flux, org=self.org)
 
         rows = [
@@ -322,11 +354,18 @@ class InfluxDbNode(Node):
             - Dict with "value" and "tags" keys: Value written with merged tags
         :type data: dict
         """
-        from influxdb_client import InfluxDBClient, Point, WritePrecision
-        from influxdb_client.client.write_api import SYNCHRONOUS
+        influxdb_client = _influxdb()
+        InfluxDBClient = influxdb_client.InfluxDBClient
+        Point, WritePrecision = influxdb_client.Point, influxdb_client.WritePrecision
+        SYNCHRONOUS = influxdb_client.client.write_api.SYNCHRONOUS
 
         try:
-            with InfluxDBClient(url=self.url, token=self.token, org=self.org) as client:
+            with InfluxDBClient(
+                url=self.url,
+                token=self.token,
+                org=self.org,
+                timeout=int(self.timeout * 1000),
+            ) as client:
                 write_api = client.write_api(write_options=SYNCHRONOUS)
 
                 precision_map = {
@@ -417,12 +456,17 @@ class InfluxDbNode(Node):
         :returns: Dict of port name to queried value.
         :rtype: dict
         """
-        from influxdb_client import InfluxDBClient
+        InfluxDBClient = _influxdb().InfluxDBClient
 
         results = {}
 
         try:
-            with InfluxDBClient(url=self.url, token=self.token, org=self.org) as client:
+            with InfluxDBClient(
+                url=self.url,
+                token=self.token,
+                org=self.org,
+                timeout=int(self.timeout * 1000),
+            ) as client:
                 query_api = client.query_api()
 
                 for spec in self.output_ports:

backend/fluksio/flow/nodes/mqtt.py

@@ -19,9 +19,21 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-# Deep enough to ride out a broker hiccup, shallow enough that a publisher
-# which cannot keep up drops old values instead of growing without bound.
-PUBLISH_QUEUE_SIZE = 256
+
+def _aiomqtt() -> Any:
+    """The client library, which is a `fluksio[server]` extra.
+
+    Imported per use rather than at module level, because the node type is
+    registered at boot and an installation that talks to no broker should not
+    have to carry the library to start.
+    """
+    try:
+        import aiomqtt
+    except ImportError:
+        raise RuntimeError(
+            "the mqtt node needs the server extra: pip install 'fluksio[server]'"
+        ) from None
+    return aiomqtt
 
 
 def topic_matches(filter_: str, topic: str) -> bool:
@@ -83,6 +95,9 @@ class MqttNode(Node):
         - ``qos`` (int): Quality of Service level 0, 1, or 2 (default: 0)
         - ``retain`` (bool): Retain flag for published messages (default: False)
         - ``keepalive`` (int): Keepalive interval in seconds (default: 60)
+        - ``timeout`` (float): Deadline for a broker operation in seconds
+          (default: 10.0)
+        - ``publish_queue_size`` (int): Publisher backlog depth (default: 256)
     :type params: dict
     :param name: Optional name for the node.
     :type name: str | None
@@ -151,6 +166,8 @@ class MqttNode(Node):
         "qos",
         "retain",
         "keepalive",
+        "timeout",
+        "publish_queue_size",
         "json_keys",
         "_topic_to_ports",
         "_wildcards",
@@ -175,6 +192,26 @@ class MqttNode(Node):
         qos: int = 0
         retain: bool = False
         keepalive: int = 60
+        # Bounds every broker operation: subscribe, publish, and the
+        # disconnect acknowledgement on the way out. Without one a client
+        # whose socket died waits for that ack forever, and the task never
+        # finishes unwinding. Brokers differ, so it is per node.
+        timeout: float = Field(
+            default=10.0,
+            gt=0,
+            description="Give up on a broker operation after this many seconds.",
+        )
+        # Deep enough to ride out a broker hiccup, shallow enough that a
+        # publisher which cannot keep up drops old values instead of growing
+        # without bound. A node that bursts wants more than one that trickles.
+        publish_queue_size: int = Field(
+            default=256,
+            gt=0,
+            description=(
+                "How many payloads may wait for the broker. Past this the oldest "
+                "is dropped and the node reports degraded."
+            ),
+        )
         # Which key to lift out of a JSON object payload. A device that wraps
         # its reading — Victron's ``{"value": 5}`` — is otherwise a Python node
         # per port. One key for every port, or a per-port mapping.
@@ -247,6 +284,8 @@ class MqttNode(Node):
         self.qos = cfg.qos
         self.retain = cfg.retain
         self.keepalive = cfg.keepalive
+        self.timeout = cfg.timeout
+        self.publish_queue_size = cfg.publish_queue_size
 
         # Runtime state
         self._subscription_task: asyncio.Task[None] | None = None
@@ -359,7 +398,7 @@ class MqttNode(Node):
         A dropped connection raises, and the supervisor decides when to
         reconnect — the same arrangement the subscriber uses.
         """
-        import aiomqtt
+        aiomqtt = _aiomqtt()
 
         queue = self._publish_queue
         if queue is None:
@@ -372,6 +411,7 @@ class MqttNode(Node):
             password=self.password,
             identifier=self.client_id,
             keepalive=self.keepalive,
+            timeout=self.timeout,
         ) as client:
             self.report_health("ok")
             while True:
@@ -384,7 +424,7 @@ class MqttNode(Node):
 
     async def _publish_once(self, data: dict[str, Any]) -> None:
         """Connect, publish, disconnect — the unstarted node's path."""
-        import aiomqtt
+        aiomqtt = _aiomqtt()
 
         async with aiomqtt.Client(
             hostname=self.broker_host,
@@ -393,6 +433,7 @@ class MqttNode(Node):
             password=self.password,
             identifier=self.client_id,
             keepalive=self.keepalive,
+            timeout=self.timeout,
         ) as client:
             await self._publish_with(client, data)
 
@@ -452,7 +493,7 @@ class MqttNode(Node):
         """Run the task that owns this node's connection to the broker."""
         if self._publish_queue is not None:
             return
-        self._publish_queue = asyncio.Queue(maxsize=PUBLISH_QUEUE_SIZE)
+        self._publish_queue = asyncio.Queue(maxsize=self.publish_queue_size)
         self._loop = asyncio.get_running_loop()
         self._publisher_task = self._run_supervised("mqtt-out", self._publisher_loop)
 
@@ -461,11 +502,7 @@ class MqttNode(Node):
         if self._publish_queue is None:
             return
         if self._publisher_task is not None:
-            self._publisher_task.cancel()
-            try:
-                await self._publisher_task
-            except (asyncio.CancelledError, Exception):  # noqa: B014 - shutting down
-                pass
+            await self._cancel_task(self._publisher_task)
         self._publisher_task = None
         self._publish_queue = None
         self._loop = None
@@ -515,11 +552,7 @@ class MqttNode(Node):
         self._stop_event.set()
 
         if self._subscription_task is not None:
-            self._subscription_task.cancel()
-            try:
-                await self._subscription_task
-            except asyncio.CancelledError:
-                pass
+            await self._cancel_task(self._subscription_task)
 
         self._subscription_task = None
         self._stop_event = None
@@ -553,7 +586,7 @@ class MqttNode(Node):
         """
         import json
 
-        import aiomqtt
+        aiomqtt = _aiomqtt()
 
         if not (self._stop_event and self._stop_event.is_set()):
             try:
@@ -564,6 +597,7 @@ class MqttNode(Node):
                     password=self.password,
                     identifier=self.client_id,
                     keepalive=self.keepalive,
+                    timeout=self.timeout,
                 ) as client:
                     # Subscribe to every unique topic
                     for topic in self._topic_to_ports:

backend/fluksio/flow/pipeline.py

@@ -59,6 +59,7 @@ class ValidationIssue(BaseModel):
         "unauthenticated_hook",
         "self_loop_needs_initial",
         "missing_source",
+        "node_unhealthy",
     ]
     message: str
     flow: str = ""

backend/fluksio/flow/placement.py

@@ -215,9 +215,18 @@ class Placer:
         said = [shape[2] for shape in capable if shape[2]]
         ram = min(ram, max(said)) if said and ram else ram
         if (cpus, gpus, ram) != (wanted.cpus, wanted.gpus, wanted.ram or 0):
+            # Cards are not detected, so a machine that has one still reports
+            # none until it is told — which reads as "no GPU here" to a node
+            # that then runs unserialised beside every other one.
+            hint = (
+                "; no machine here declares a GPU — `fluksio serve --gpus N` "
+                "(or FLOW_GPUS) says how many this one has"
+                if wanted.gpus and not gpus
+                else ""
+            )
             logger.warning(
                 "%s asked for %d cpu(s), %d gpu(s) and %s MB; "
-                "the largest machine here can give %d, %d and %s",
+                "the largest machine here can give %d, %d and %s%s",
                 node or "a node",
                 wanted.cpus,
                 wanted.gpus,
@@ -225,6 +234,7 @@ class Placer:
                 cpus,
                 gpus,
                 ram or "no stated",
+                hint,
             )
         return cpus, gpus, ram
 

backend/fluksio/flow/runs.py

@@ -769,7 +769,9 @@ class RunService:
         # the isolation it wants, minus surviving the process.
         self._state_factory = state_factory or (lambda _ns: MemoryState())
         self.engine_name = f"{socket.gethostname()}-{os.getpid()}"[:64]
-        self.parallel = max(1, parallel)
+        # Taken as written: clamping a 0 up to 1 would hide a limit somebody
+        # set, and the pool below rejects an unusable one loudly anyway.
+        self.parallel = parallel
         self._pool = ThreadPoolExecutor(
             max_workers=self.parallel, thread_name_prefix="run"
         )
@@ -844,6 +846,26 @@ class RunService:
         # reference a python caller would have passed and every later reader —
         # the digest, the cache, the run detail — sees one spelling.
         params = resolve_references(flow, params, self._artifacts)
+        # What the run actually starts from, not only what was passed: an input
+        # left out takes its declared value, and a row that records `{}` cannot
+        # say which. Folded literally — an initial is a value from the
+        # definition, never a reference to resolve.
+        declared = {
+            one.spec.name: one.initial for one in flow.inputs if one.initial is not None
+        }
+        # The run's own seed fills an input of that name, outranking what the
+        # flow declares and outranked by one passed as a parameter — the order
+        # `seed_values` applies, moved to where the record is written.
+        if seed is not None and any(one.spec.name == "seed" for one in flow.inputs):
+            declared["seed"] = seed
+        params = {**declared, **params}
+        # And back the other way, so the run-level column holds the seed the
+        # run actually used however it arrived. Otherwise `--seed 1` fills one
+        # column and a declared seed the other, and that is the single field an
+        # export still has to coalesce.
+        resolved_seed = params.get("seed")
+        if isinstance(resolved_seed, int) and not isinstance(resolved_seed, bool):
+            seed = resolved_seed
         # Checked here rather than in the driver: a caller who mistyped a
         # parameter should be told now, not by a run that fails in a minute.
         seed_values(flow, params, seed)
@@ -1174,6 +1196,7 @@ class RunService:
             self._finish(run_id, status, reason, result, duration)
             run.status = status
             self._publish(run, "run_finished")
+            self._release_cards(run)
             # Its values were only ever this run's; nothing reads them once it
             # has a result. On Redis the namespace would expire anyway.
             if state is not None and status != "error":
@@ -1182,6 +1205,29 @@ class RunService:
                 except Exception:
                     logger.warning("Could not clear state of run %s", run_id)
 
+    def _release_cards(self, run: Run) -> None:
+        """Hand a GPU run's device memory back when the run is over.
+
+        The accountant frees the card the moment the node returns, but the
+        worker that ran on it is kept warm and a library that preallocated
+        most of the VRAM never gives it up — so the next process to want the
+        card found it taken by one sitting idle.
+        """
+        # ponytail: retires every CUDA pool rather than the ones this run used,
+        # which needs no bookkeeping — a concurrent GPU run's busy worker only
+        # dies when it returns, which is when its own memory should go back
+        # anyway. What it costs is the warm worker of a *live* flow's GPU node.
+        # Track the pools per run if that ever matters.
+        if not (run.needs or {}).get("gpus"):
+            return
+        pool = getattr(self.controller, "workers", None)
+        if pool is None:
+            return
+        try:
+            pool.retire_gpu_children()
+        except Exception:
+            logger.warning("Could not retire the GPU workers of run %s", run.id)
+
     def _record_node(self, run_id: str, outcome: NodeOutcome) -> None:
         outputs = _cacheable(outcome)
         row = RunNode(

backend/fluksio/flow/workers.py

@@ -321,6 +321,24 @@ class PythonWorkerPool:
         for child in children:
             child.respawn_all()
 
+    def retire_gpu_children(self) -> None:
+        """Retire the pools holding a card, so the VRAM goes back.
+
+        A library like JAX takes most of the device when it imports and never
+        releases it, so a warm worker that has run one such node is a held
+        card — and warm is the point of a pool, so nothing retires it. At the
+        end of a run there is something to key on: the environments carrying a
+        GPU assignment are exactly the pools that ran on one.
+        """
+        with self._lock:
+            children = [
+                child
+                for key, child in self._children.items()
+                if any(name == "CUDA_VISIBLE_DEVICES" for name, _ in key)
+            ]
+        for child in children:
+            child.respawn_all()
+
     def _drain(self) -> list[_Worker | None]:
         slots = []
         while True:

backend/fluksio/main.py

@@ -4,7 +4,6 @@ import logging
 from collections.abc import AsyncIterator
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
 
-import sentry_sdk
 from fastapi import FastAPI, Request
 from fastapi.concurrency import run_in_threadpool
 from fastapi.responses import JSONResponse
@@ -51,6 +50,10 @@ def custom_generate_unique_id(route: APIRoute) -> str:
 
 
 if settings.SENTRY_DSN and settings.ENVIRONMENT != "local":
+    # Imported here rather than at the top: it is a `fluksio[server]` extra, so
+    # a pip install without one has no sentry to import — and no DSN either.
+    import sentry_sdk
+
     # `enable_tracing` was removed in sentry-sdk 2.x; this is what it meant.
     sentry_sdk.init(dsn=str(settings.SENTRY_DSN), traces_sample_rate=1.0)
 

backend/fluksio/sdk/__init__.py

@@ -388,11 +388,6 @@ def _check_signature(spec: NodeSpec) -> None:
     fn, where = spec.fn, f"node '{spec.id}'"
     if inspect.iscoroutinefunction(fn):
         raise SyncError(f"{where}: async functions cannot be nodes")
-    if fn.__module__ == "__main__":
-        raise SyncError(
-            f"{where}: {fn.__name__}() is defined in a script run directly, so the "
-            "generated node could not import it — put it in an importable module"
-        )
     generator = inspect.isgeneratorfunction(fn)
     if generator and spec.single:
         raise SyncError(
@@ -551,6 +546,21 @@ def _check(spec: NodeSpec, mode: str) -> dict[str, Any]:
     }
 
 
+def _refuse_main(spec: NodeSpec) -> None:
+    """A node the generated body would have no way to import.
+
+    Refused where the body is written rather than where the decorator is: a
+    module defining nodes is then still runnable as a script, which is what a
+    `__main__` self-check beside them needs.
+    """
+    if spec.fn.__module__ == "__main__":
+        raise SyncError(
+            f"node '{spec.id}': {spec.fn.__name__}() is defined in a script run "
+            "directly, so the generated node could not import it — put it in an "
+            "importable module"
+        )
+
+
 def _code_of(spec: NodeSpec) -> dict[str, Any]:
     """What this node's function calls into, which its shim does not say.
 
@@ -559,6 +569,7 @@ def _code_of(spec: NodeSpec) -> dict[str, Any]:
     side is the only one that can work it out at all: it has imported the
     code, and the engine never does.
     """
+    _refuse_main(spec)
     files = reached(spec.fn)
     return {"code_files": files, "code_digest": digest_of(files)}
 
@@ -762,6 +773,7 @@ def _shim(spec: NodeSpec) -> str:
     simply happens to be generated, which is why it says so and says where the
     real thing is.
     """
+    _refuse_main(spec)
     fn = spec.fn
     where = inspect.getsourcefile(fn) or fn.__module__
     repo = import_root(fn)

backend/fluksio/sdk/cli.py

@@ -19,6 +19,7 @@ import sys
 import time
 from collections.abc import Callable, Iterable, Iterator
 from contextlib import contextmanager, nullcontext
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
@@ -136,8 +137,8 @@ def _import(root: str, dotted: str, expect: Path | None = None) -> None:
             f"  {actual}\n  {expect}\n"
             "Python keeps one module per name, and a node's generated body "
             "imports by that name, so the second would run the first's code. "
-            "Put an `__init__.py` in each directory — they become "
-            f"'<dir>.{dotted}' and stop colliding — or rename one of the files."
+            "Sync the directory they are both under — each then imports as "
+            "'<dir>.<file>' — or rename one of the files."
         )
 
 
@@ -211,7 +212,14 @@ def discover(targets: list[str], keep_going: bool = False) -> list[Flow]:
                 if entry.is_dir():
                     _import_package(entry)
                 else:
-                    _import(*_module_of(entry), entry)
+                    # Named for where it sits under the directory being synced,
+                    # so `dev/s1/study.py` imports as `s1.study` and one
+                    # `study.py` per study collides with nothing. No
+                    # `__init__.py` needed: the directories in between are
+                    # namespace packages. A file at the root keeps its bare
+                    # name, which is what it has always had.
+                    dotted = ".".join(entry.relative_to(path).with_suffix("").parts)
+                    _import(str(path), dotted, entry)
             except SyncError:
                 # A name collision is never somebody else's problem: it would
                 # put the wrong file behind a node.
@@ -439,33 +447,52 @@ def _ask_params(definition: dict[str, Any]) -> dict[str, Any]:
 def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
     """Turn `--lr 0.05` into a typed parameter, using the flow's own inputs."""
     types = _input_types(definition)
-    params: dict[str, Any] = {}
+    # Collected as written and typed afterwards, so a name this flow does not
+    # have is refused by name rather than by whatever its value failed to parse
+    # as: `--param lr=0.002` is a sweep's spelling, and said so it reads as an
+    # input called `param` holding unparseable json.
+    raw: dict[str, str | bool] = {}
     pending: str | None = None
     for token in rest:
         if token.startswith("--"):
             if pending is not None:
                 # A flag with no value is a flag: `--resume` means true.
-                params[pending] = True
+                raw[pending] = True
             name, sep, value = token[2:].partition("=")
             # Only the name is spelled with dashes; a value may hold one, and
             # `--lr=1e-4` is the case that says so.
             pending = name.replace("-", "_")
             if sep:
-                params[pending] = _coerce(value, types.get(pending, "json"))
+                raw[pending] = value
                 pending = None
             continue
         if pending is None:
             raise SyncError(f"unexpected argument '{token}'")
-        params[pending] = _coerce(token, types.get(pending, "json"))
+        raw[pending] = token
         pending = None
     if pending is not None:
-        params[pending] = True
-    unknown = sorted(set(params) - set(types))
+        raw[pending] = True
+    unknown = sorted(set(raw) - set(types))
     if unknown:
+        hint = ""
+        if unknown[0] == "param":
+            hint = (
+                " — one value is `--<name> <value>`; several is a sweep: "
+                "`fluksio sweep --param name=v1,v2`"
+            )
         raise SyncError(
             f"'{unknown[0]}' is not an input of this flow (it takes "
-            f"{', '.join(sorted(types)) or 'none'})"
+            f"{', '.join(sorted(types)) or 'none'}){hint}"
         )
+    params: dict[str, Any] = {}
+    for key, written in raw.items():
+        if isinstance(written, bool):
+            params[key] = written
+            continue
+        try:
+            params[key] = _coerce(written, types[key])
+        except ValueError as exc:
+            raise SyncError(f"'{key}' takes {types[key]}: {exc}") from exc
     return params
 
 
@@ -778,7 +805,8 @@ def _status_screen(client: Client) -> Any:
                 )
                 + Text(
                     f"{str(row.get('flow', '')):<16}"
-                    f"{(row.get('duration_ms') or 0) / 1000:7.1f}s",
+                    f"{(row.get('duration_ms') or 0) / 1000:7.1f}s"
+                    f"  {_ago(row.get('finished_at') or row.get('created_at')):>9}",
                     style="dim",
                 )
             )
@@ -790,6 +818,7 @@ def _status_screen(client: Client) -> Any:
             )
             parts.append(
                 Text(f"  {where}  ", style="red")
+                + Text(f"{_ago(event.get('ts')):>9}  ", style="dim")
                 + Text(str(event.get("detail", ""))[:100], style="dim")
             )
     return Group(*parts)
@@ -831,9 +860,9 @@ def cmd_status(args: argparse.Namespace) -> int:
 #: value is read.
 PARAMS_WIDTH = 80
 
-#: What the columns before the inputs take: the id, status, flow, duration and
-#: stamp, with their spacing.
-LISTING_WIDTH = 84
+#: What the columns before the inputs take: the id, status, flow, duration,
+#: age and stamp, with their spacing.
+LISTING_WIDTH = 95
 
 
 def _declared(client: Client, flows: Iterable[str]) -> dict[str, dict[str, Any]]:
@@ -857,6 +886,28 @@ def _declared(client: Client, flows: Iterable[str]) -> dict[str, dict[str, Any]]
     return known
 
 
+def _ago(stamp: Any) -> str:
+    """How long ago something happened, in the notation the screens use.
+
+    A duration and an age read together, so they are spelled the same way:
+    seconds, then minutes, hours, days.
+    """
+    if not stamp:
+        return ""
+    try:
+        then = datetime.fromisoformat(str(stamp))
+    except ValueError:
+        return ""
+    if then.tzinfo is None:
+        # Everything the engine records is UTC; only some spellings say so.
+        then = then.replace(tzinfo=UTC)
+    seconds = max((datetime.now(UTC) - then).total_seconds(), 0)
+    for span, unit in ((86400, "d"), (3600, "h"), (60, "min")):
+        if seconds >= span:
+            return f"{seconds / span:.0f}{unit} ago"
+    return f"{seconds:.0f}s ago"
+
+
 def _stamp(row: dict[str, Any]) -> str:
     """What code a run ran: the commit, whether it was dirty, and the digest.
 
@@ -899,11 +950,49 @@ def cmd_runs(args: argparse.Namespace) -> int:
             params = params[: room - 3] + "..."
         _say(
             f"{row['id']}  {_status(row['status'], 9)} {row['flow']:<16} "
-            f"{row['duration_ms'] / 1000:7.1f}s  {_stamp(row):<22} {params}"
+            f"{row['duration_ms'] / 1000:7.1f}s {_ago(row.get('created_at')):>9}  "
+            f"{_stamp(row):<22} {params}"
         )
     return 0
 
 
+def _artifacts(client: Client, args: argparse.Namespace) -> int:
+    """List a run's files, or write one of them here."""
+    handle = RunHandle(client, args.run_id, client.run(args.run_id))
+    rows = handle.artifacts
+    if not args.name:
+        if not rows:
+            _say("This run produced no artifacts.")
+            return 0
+        for row in rows:
+            named = row.get("filename") or ""
+            _say(
+                f"{str(row['name']):<24} {row['size']:>10} B  "
+                f"{row.get('media_type', ''):<24} {named}"
+            )
+        return 0
+    data = handle.download(args.name)
+    # The name it was written under reads better than the message's, which is
+    # chosen for the graph; `--out` beats both.
+    match = next((row for row in rows if row.get("name") == args.name), {})
+    out = Path(args.out or match.get("filename") or args.name)
+    out.write_bytes(data)
+    _say(f"{out}  {len(data)} bytes")
+    return 0
+
+
+def cmd_artifacts(args: argparse.Namespace) -> int:
+    try:
+        with _client_for(args, retries=0) as client:
+            return _artifacts(client, args)
+    except KeyError as exc:
+        return _fail(str(exc.args[0]))
+    except (SyncError, ApiError) as exc:
+        return _fail(str(exc))
+    except httpx.HTTPError as exc:
+        return _unreachable(exc)
+
+
 def cmd_flavors(args: argparse.Namespace) -> int:
     """The named sizes a node can ask for."""
     try:
@@ -1267,6 +1356,21 @@ def add_parsers(subparsers: Any) -> None:
     with_engine(parser, local=True)
     parser.set_defaults(func=cmd_runs)
 
+    parser = subparsers.add_parser(
+        "artifacts", help="the files a run produced; name one to download it"
+    )
+    parser.add_argument("run_id")
+    parser.add_argument("name", nargs="?", default="")
+    parser.add_argument(
+        "-o",
+        "--out",
+        default="",
+        metavar="PATH",
+        help="where to write it (default: the name it was saved under)",
+    )
+    with_engine(parser, local=True)
+    parser.set_defaults(func=cmd_artifacts)
+
     parser = subparsers.add_parser(
         "flavors", help="the named resource sizes a node can ask for"
     )
@@ -1375,7 +1479,7 @@ def add_parsers(subparsers: Any) -> None:
         metavar="A,B",
         help=(
             "the inputs to put in columns, dotted into a record "
-            "(default: the ones that vary)"
+            "(default: every input the runs recorded)"
         ),
     )
     sub.add_argument(

backend/fluksio/sdk/client.py

@@ -428,9 +428,9 @@ class Client:
     ) -> list[dict[str, Any]]:
         """One row per run: its inputs as columns, its final numbers, its code.
 
-        The arm-comparison table. The inputs kept are the ones that vary
-        across the selection unless ``params`` names them, which is the axis a
-        sweep is read along.
+        The arm-comparison table. Every input the selection recorded is a
+        column, so the schema does not depend on which runs were asked for;
+        ``params`` narrows it to the axis a sweep is read along.
         """
         query: dict[str, Any] = {}
         if params:

backend/fluksio/tui.py

@@ -0,0 +1,342 @@
+"""The dashboard `fluksio serve` opens at a terminal.
+
+A supervisor, not an engine: it starts `serve --plain` as a child and talks to
+it over the same HTTP API every other client uses. So the engine is a process
+that can be stopped and started under the screen watching it, an engine
+somebody else started can be adopted rather than duplicated, and quitting the
+dashboard is not the same as stopping the engine.
+
+Nothing of the engine is imported here — that lives in the child.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import signal
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+from textual import work
+from textual.app import App, ComposeResult
+from textual.containers import Horizontal, Vertical
+from textual.screen import ModalScreen
+from textual.widgets import Button, DataTable, Footer, Header, Input, Label, RichLog
+
+from fluksio.cli import (
+    DEFAULT_PORT,
+    DEFAULT_PORTAL,
+    _data_dir,
+    _token_for,
+    probe_engine,
+    read_pidfile,
+)
+from fluksio.sdk.cli import WATCH_INTERVAL_S, _status_screen
+from fluksio.sdk.client import Client, config_path
+
+#: How long to keep asking the child for the credential it writes on the way
+#: up, before giving up and letting the panels say so.
+STARTUP_TRIES = 60
+
+
+def child_argv(argv: list[str]) -> list[str]:
+    """The same command, told to serve without a dashboard of its own.
+
+    Every flag is passed through: whatever `serve` was asked for is what the
+    engine under this screen is running with.
+    """
+    passed = [flag for flag in argv if flag != "--plain"]
+    return [sys.executable, "-m", "fluksio.cli", *passed, "--plain"]
+
+
+class Enroll(ModalScreen[tuple[str, str] | None]):
+    """The claim code a portal minted, and which portal minted it."""
+
+    BINDINGS = [("escape", "dismiss(None)", "cancel")]
+
+    def compose(self) -> ComposeResult:
+        with Vertical(id="enroll"):
+            yield Label("Pair this installation with a portal")
+            yield Input(placeholder="claim code", id="code")
+            yield Input(value=DEFAULT_PORTAL, id="portal")
+            with Horizontal():
+                yield Button("Enroll", variant="primary", id="go")
+                yield Button("Cancel", id="cancel")
+
+    def on_button_pressed(self, event: Button.Pressed) -> None:
+        if event.button.id == "cancel":
+            self.dismiss(None)
+            return
+        code = self.query_one("#code", Input).value.strip()
+        portal = self.query_one("#portal", Input).value.strip()
+        self.dismiss((code, portal or DEFAULT_PORTAL) if code else None)
+
+
+class ServeApp(App[int]):
+    """One screen: how the engine is, what has run, and what it is saying."""
+
+    CSS = """
+    #status { height: auto; padding: 0 1; }
+    #runs { height: 2fr; }
+    #log { height: 1fr; border-top: solid $panel; }
+    #enroll { width: 60; height: auto; padding: 1 2; background: $surface; }
+    """
+
+    BINDINGS = [
+        ("q", "quit", "quit (engine keeps running)"),
+        ("s", "stop_start", "stop/start"),
+        ("r", "restart", "restart"),
+        ("c", "cancel_run", "cancel run"),
+        ("e", "enroll", "enroll"),
+    ]
+
+    def __init__(self, args: argparse.Namespace) -> None:
+        super().__init__()
+        self.args = args
+        self.data_dir: Path = _data_dir(args.data_dir, args.shared)
+        self.child: subprocess.Popen[str] | None = None
+        #: The pid of an engine this dashboard did not start. Only ever one
+        #: whose data directory is this one — the probe is what proves it.
+        self.adopted: int | None = None
+        self.client: Client | None = None
+        self.url = ""
+
+    # -- layout ---------------------------------------------------------------
+
+    def compose(self) -> ComposeResult:
+        yield Header()
+        yield RichLog(id="status", markup=True, wrap=True)
+        table: DataTable[str] = DataTable(id="runs", cursor_type="row")
+        yield table
+        yield RichLog(id="log", markup=False, highlight=False, max_lines=2000)
+        yield Footer()
+
+    def on_mount(self) -> None:
+        self.title = f"fluksio — {self.data_dir}"
+        table = self.query_one("#runs", DataTable)
+        table.add_columns("run", "status", "flow", "took", "params")
+        self.start_engine(first=True)
+        self.set_interval(WATCH_INTERVAL_S, self.refresh_panels)
+
+    # -- the engine under the screen ------------------------------------------
+
+    def note(self, message: str) -> None:
+        self.query_one("#log", RichLog).write(message)
+
+    def start_engine(self, first: bool = False) -> None:
+        """Adopt whatever is already serving this directory, or start one."""
+        host = self.args.host
+        reachable = "127.0.0.1" if host in ("0.0.0.0", "::", "") else host
+        wanted = self.args.port or DEFAULT_PORT
+        url = f"http://{reachable}:{wanted}"
+        who = probe_engine(url, _token_for(self.data_dir)) if first else "other"
+
+        if who == "ours":
+            running = read_pidfile(self.data_dir)
+            self.adopted = running["pid"] if running else None
+            self.url = url
+            named = f" (pid {self.adopted})" if self.adopted else ""
+            self.note(f"Adopted the engine already serving this directory{named}.")
+            if self.adopted is None:
+                self.note("It wrote no pidfile, so this screen cannot stop it.")
+            self.connect()
+            return
+        if who == "foreign":
+            self.note(f"Port {wanted} holds another installation's Fluksio.")
+
+        self.child = subprocess.Popen(  # noqa: S603
+            child_argv(sys.argv[1:]),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+            env={**os.environ, "PYTHONUNBUFFERED": "1"},
+        )
+        self.tail_child(self.child)
+        self.await_engine()
+
+    @work(thread=True, exclusive=False)
+    def tail_child(self, child: subprocess.Popen[str]) -> None:
+        """The engine's own output, which is why a second terminal was needed."""
+        if child.stdout is None:
+            return
+        for line in child.stdout:
+            self.call_from_thread(self.note, line.rstrip())
+        self.call_from_thread(self.note, f"The engine stopped ({child.wait()}).")
+
+    @work(thread=True, exclusive=True, group="startup")
+    def await_engine(self) -> None:
+        """Wait for the child to say where it landed, then talk to it.
+
+        The port is the child's to choose — it moves off a taken one — and
+        `client.json` is where it says which it took.
+        """
+        import time
+
+        for _ in range(STARTUP_TRIES):
+            if self.child is not None and self.child.poll() is not None:
+                return
+            try:
+                stored = json.loads(config_path(self.data_dir).read_text())
+            except (OSError, ValueError):
+                stored = {}
+            if stored.get("url") and stored.get("token"):
+                self.url = str(stored["url"])
+                self.call_from_thread(self.connect)
+                return
+            time.sleep(0.5)
+        self.call_from_thread(self.note, "The engine did not come up.")
+
+    def connect(self) -> None:
+        stored = config_path(self.data_dir)
+        try:
+            config = json.loads(stored.read_text())
+            self.client = Client(
+                url=str(config["url"]), token=str(config["token"]), retries=0
+            )
+        except (OSError, ValueError, KeyError) as exc:
+            self.note(f"No credential to talk to the engine with: {exc}")
+            return
+        self.url = self.client.url
+        self.refresh_panels()
+
+    def engine_pid(self) -> int | None:
+        return self.child.pid if self.child is not None else self.adopted
+
+    def stop_engine(self) -> None:
+        if self.child is not None:
+            self.note(f"Stopping the engine (pid {self.child.pid}).")
+            self.child.terminate()
+            self.child = None
+        elif self.adopted is not None:
+            self.note(f"Stopping the adopted engine (pid {self.adopted}).")
+            try:
+                os.kill(self.adopted, signal.SIGTERM)
+            except OSError as exc:
+                self.note(f"Could not stop it: {exc}")
+            self.adopted = None
+        self.client = None
+
+    # -- what the panels show -------------------------------------------------
+
+    def refresh_panels(self) -> None:
+        if self.client is not None:
+            self.read_engine()
+
+    @work(thread=True, exclusive=True, group="poll")
+    def read_engine(self) -> None:
+        client = self.client
+        if client is None:
+            return
+        try:
+            screen = _status_screen(client)
+            rows = client.runs(limit=20)
+        except Exception as exc:
+            self.call_from_thread(self.show_offline, exc)
+            return
+        self.call_from_thread(self.show, screen, rows)
+
+    def show(self, screen: Any, rows: list[dict[str, Any]]) -> None:
+        status = self.query_one("#status", RichLog)
+        status.clear()
+        status.write(screen)
+        table = self.query_one("#runs", DataTable)
+        table.clear()
+        for row in rows:
+            table.add_row(
+                str(row.get("id", ""))[-8:],
+                str(row.get("status", "")),
+                str(row.get("flow", "")),
+                f"{(row.get('duration_ms') or 0) / 1000:.1f}s",
+                json.dumps(row.get("params") or {})[:60],
+                key=str(row.get("id", "")),
+            )
+
+    def show_offline(self, exc: Exception) -> None:
+        status = self.query_one("#status", RichLog)
+        status.clear()
+        which = "starting" if self.engine_pid() is not None else "not running"
+        status.write(f"[yellow]The engine is {which}.[/] ({type(exc).__name__})")
+
+    # -- keys -----------------------------------------------------------------
+
+    def action_stop_start(self) -> None:
+        if self.engine_pid() is not None:
+            self.stop_engine()
+        else:
+            self.start_engine()
+
+    def action_restart(self) -> None:
+        self.stop_engine()
+        self.start_engine()
+
+    def action_cancel_run(self) -> None:
+        table = self.query_one("#runs", DataTable)
+        if self.client is None or not table.row_count:
+            return
+        # The cell holds the tail of the id, which is what fits; the row's key
+        # is the whole of it, which is what the engine is asked about.
+        key = table.coordinate_to_cell_key(table.cursor_coordinate).row_key
+        if key.value:
+            self.cancel_run(str(key.value))
+
+    @work(thread=True)
+    def cancel_run(self, run_id: str) -> None:
+        client = self.client
+        if client is None:
+            return
+        try:
+            client.cancel(run_id)
+        except Exception as exc:
+            self.call_from_thread(self.note, f"Could not cancel {run_id}: {exc}")
+            return
+        self.call_from_thread(self.note, f"Cancelled {run_id}.")
+        self.call_from_thread(self.refresh_panels)
+
+    def action_enroll(self) -> None:
+        self.push_screen(Enroll(), self.enrolled)
+
+    def enrolled(self, answer: tuple[str, str] | None) -> None:
+        if answer is None:
+            return
+        code, portal = answer
+        self.run_enroll(code, portal)
+
+    @work(thread=True)
+    def run_enroll(self, code: str, portal: str) -> None:
+        """`fluksio enroll`, as its own process for the same reason serve is.
+
+        A running engine picks the configuration up on its own; this screen
+        only has to report what the command said.
+        """
+        done = subprocess.run(  # noqa: S603
+            [
+                sys.executable,
+                "-m",
+                "fluksio.cli",
+                "enroll",
+                code,
+                "--portal",
+                portal,
+                "--data-dir",
+                str(self.data_dir),
+            ],
+            capture_output=True,
+            text=True,
+        )
+        for line in (done.stdout + done.stderr).splitlines():
+            self.call_from_thread(self.note, line)
+
+
+def run_tui(args: argparse.Namespace) -> int:
+    """Open the dashboard, and say what is still running when it closes."""
+    app = ServeApp(args)
+    app.run()
+    pid, url = app.engine_pid(), app.url
+    if pid is not None:
+        print(f"The engine is still running: pid {pid} at {url or 'its port'}.")
+        print(f"  fluksio serve reattaches to it; kill {pid} stops it.")
+    return 0

backend/fluksio/utils.py

@@ -4,7 +4,6 @@ from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from typing import Any
 
-import emails  # type: ignore
 import jwt
 from jinja2 import Template
 from jwt.exceptions import InvalidTokenError
@@ -37,6 +36,12 @@ def send_email(
     html_content: str = "",
 ) -> None:
     assert settings.emails_enabled, "no provided configuration for email variables"
+    try:
+        import emails  # type: ignore
+    except ImportError:
+        raise RuntimeError(
+            "sending mail needs the server extra: pip install 'fluksio[server]'"
+        ) from None
     message = emails.Message(
         subject=subject,
         html=html_content,