Merge branch 'main' of git.stroblme.de:Fluksio/app

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01C5H4uLCCpsbipL1R7WKCee

backend/Dockerfile

@@ -31,7 +31,7 @@ ENV NODE_VENV=managed
 RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,source=uv.lock,target=uv.lock \
     --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    uv sync --frozen --no-install-workspace --package fluksio
+    uv sync --frozen --no-install-workspace --package fluksio --extra server
 
 COPY ./backend/scripts /app/backend/scripts
 
@@ -52,7 +52,7 @@ COPY ./worker /app/worker
 RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,source=uv.lock,target=uv.lock \
     --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    uv sync --frozen --package fluksio
+    uv sync --frozen --package fluksio --extra server
 
 # Connectors are ordinary installed packages found through the
 # `fluksio.node_types` entry point. `make connectors` builds them into here;

backend/fluksio/api/main.py

@@ -15,6 +15,7 @@ from fluksio.api.routes import (
     panels,
     private,
     runs,
+    search,
     secrets,
     users,
     utils,
@@ -38,6 +39,7 @@ api_router.include_router(observability.router)
 api_router.include_router(runs.router)
 api_router.include_router(artifacts.router)
 api_router.include_router(workers.router)
+api_router.include_router(search.router)
 # Remote access through a portal. Always mounted; with no enrolment the
 # endpoints only ever report that there is none.
 api_router.include_router(cloud.router)

backend/fluksio/api/routes/observability.py

@@ -176,10 +176,16 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
     paused = set(controller.paused_flows())
     entries = list(controller.loaded.values())
     errored = [e for e in entries if e.status is NodeStatus.ERROR]
+    unhealthy = [e for e in entries if e.health == "down"]
     if quarantined:
         problems.append(f"{len(quarantined)} flow(s) quarantined")
     if errored:
         problems.append(f"{len(errored)} node(s) failed to load")
+    if unhealthy:
+        problems.append(
+            f"{len(unhealthy)} node(s) down: "
+            f"{', '.join(sorted(e.id for e in unhealthy))}"
+        )
 
     # What the canvas flags on a flow — a dependency loop, an input nothing
     # feeds — stops that flow running just as surely as a node that will not
@@ -219,7 +225,11 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
             "quarantined": len(quarantined),
             "invalid": len(invalid),
         },
-        nodes={"total": len(entries), "error": len(errored)},
+        nodes={
+            "total": len(entries),
+            "error": len(errored),
+            "unhealthy": len(unhealthy),
+        },
         queue=queue,
         loop_lag=(
             watchdog.snapshot()

backend/fluksio/api/routes/runs.py

@@ -8,20 +8,22 @@ or to listen on the flow socket, which carries its start and finish.
 import csv
 import io
 import json
+import time
 from collections.abc import Iterator
 from datetime import UTC, datetime
 from itertools import groupby
 from typing import Any, Literal
 
-from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi import APIRouter, Depends, HTTPException, Request, Response
 from fastapi.concurrency import run_in_threadpool
 from fastapi.responses import StreamingResponse
 from pydantic import BaseModel, Field, model_validator
-from sqlalchemy import func
+from sqlalchemy import delete, func
 from sqlalchemy import select as sa_select
 from sqlmodel import Session, col, select
 
 from fluksio.api.deps import CurrentUser, SessionDep, get_current_user
+from fluksio.flow.events import event_bus
 from fluksio.flow.messages import requalify
 from fluksio.flow.runs import RunRejected, RunService, new_run_id
 from fluksio.flow.store import FlowNotFound
@@ -102,6 +104,10 @@ class ArtifactRow(BaseModel):
     digest: str
     size: int
     media_type: str
+    #: What it was called where it was written, so something downloading it can
+    #: give it that name back rather than the message's. Absent when the node
+    #: never said one.
+    filename: str | None = None
 
 
 class RunRow(BaseModel):
@@ -420,25 +426,6 @@ def _dig(record: dict[str, Any], path: str) -> Any:
     return value
 
 
-def _varying(runs: list[Run]) -> list[str]:
-    """The inputs that differ across these runs — the axis of a sweep.
-
-    What a reader comparing arms wants as columns, compared leaf by leaf: two
-    configurations differing in one field give that field as a column rather
-    than two blobs that are not the same. Under two runs nothing can differ,
-    and a table of one run with none of its inputs in it is not worth reading,
-    so all of them are kept.
-    """
-    keys = sorted({path for run in runs for path, _ in _leaves(run.params)})
-    if len(runs) < 2:
-        return keys
-    return [
-        key
-        for key in keys
-        if len({json.dumps(_dig(run.params, key), sort_keys=True) for run in runs}) > 1
-    ]
-
-
 def _scored(runs: list[Run]) -> list[str]:
     """A run's final numbers: every number its declared outputs carry, however
     deep it sits. A flag is not a number, and neither is a label."""
@@ -555,9 +542,9 @@ def export_runs(
     """One row per run: what it was given, what it scored, what code it ran.
 
     The arm-comparison table. Inputs are columns rather than one JSON blob —
-    by default the ones that vary across the selection, which is the sweep
-    axis; ``params`` names them instead. ``metrics`` narrows the final numbers
-    to a few of a run's declared outputs.
+    every input the selection recorded, so the schema is the same whichever
+    runs are asked for; ``params`` narrows it. ``metrics`` narrows the final
+    numbers to a few of a run's declared outputs.
 
     Both take dotted paths into a record a node returned:
     ``metrics=final_metrics.train_loss,test_metrics.known.perfect`` selects
@@ -565,7 +552,9 @@ def export_runs(
     depth.
     """
     runs = _selected(session, flow, status, group, ids, since, until)
-    inputs = [part for part in params.split(",") if part] or _varying(runs)
+    inputs = [part for part in params.split(",") if part] or sorted(
+        {path for run in runs for path, _ in _leaves(run.params)}
+    )
     scores = [part for part in metrics.split(",") if part] or _scored(runs)
     columns = [
         *RUN_COLUMNS,
@@ -613,6 +602,56 @@ async def cancel_run(run_id: str, request: Request, session: SessionDep) -> Any:
     return run
 
 
+@router.delete("/{run_id}", status_code=204)
+def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response:
+    """Forget a run and everything hanging off it.
+
+    The same four statements ``_forget_runs`` uses when a flow goes: the run
+    tables carry a plain string ``run_id`` and no foreign key, so nothing
+    cascades on its own. ``flow_run``, ``metric_minute`` and ``engine_event``
+    stay — they are the observability record and are pruned on their own window.
+
+    A live run is refused rather than raced: the driver writes its nodes back
+    when it finishes, and those rows would arrive for a run that no longer
+    exists. Cancel it first.
+
+    Two things this costs, both deliberate. ``RunNode.outputs`` *is* the stage
+    cache, so a later run loses hits this one would have served. And a node
+    restored from this run points here through ``cached_from`` — ``_series``
+    already reads a missing source as an empty curve, which is what ``NO_CURVE``
+    explains on the screen. The artifact bytes need no help: ``sweep_artifacts``
+    keeps whatever a ``run_artifact`` row or a live message still names, so
+    dropping the rows is enough and the hourly sweep reclaims the blobs.
+    """
+    run = session.get(Run, run_id)
+    if run is None:
+        raise HTTPException(status_code=404, detail="No such run")
+    if run.status in ("running", "queued"):
+        raise HTTPException(
+            status_code=409,
+            detail=(
+                f"Run {run_id} is {run.status}. Cancel it, or wait for it to "
+                "finish, before deleting it."
+            ),
+        )
+    flow = run.flow
+    session.execute(delete(RunNode).where(col(RunNode.run_id) == run_id))
+    session.execute(delete(RunMetric).where(col(RunMetric.run_id) == run_id))
+    session.execute(delete(RunArtifact).where(col(RunArtifact.run_id) == run_id))
+    session.execute(delete(Run).where(col(Run.id) == run_id))
+    session.commit()
+    event_bus.publish(
+        {
+            "type": "audit",
+            "action": f"deleted run {run_id}",
+            "flow": flow,
+            "user": user.email,
+            "ts": time.time(),
+        }
+    )
+    return Response(status_code=204)
+
+
 def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
     """A run's numbers, including the ones a cached node points at.
 

backend/fluksio/api/routes/search.py

@@ -0,0 +1,156 @@
+"""One index of everything in this installation worth jumping to by name."""
+
+from typing import Any, Literal
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.concurrency import run_in_threadpool
+from pydantic import BaseModel
+
+from fluksio.api.deps import (
+    CurrentUser,
+    DashboardStoreDep,
+    FlowControllerDep,
+    get_current_user,
+)
+from fluksio.api.routes.alerts import read_config as read_alerts_config
+from fluksio.flow import modules, panels
+from fluksio.flow.controller import FlowController
+from fluksio.flow.dashboards import DashboardNotFound, DashboardStore
+from fluksio.flow.secrets import get_secrets
+from fluksio.flow.store import FlowNotFound
+
+# A wall panel never reaches this route: ``deps._panel_may`` is a whitelist that
+# ends in a 403, and a whole-installation index is the opposite of what a screen
+# on a wall is allowed to read.
+router = APIRouter(
+    prefix="/search", tags=["search"], dependencies=[Depends(get_current_user)]
+)
+
+Category = Literal[
+    "flow",
+    "node",
+    "dashboard",
+    "widget",
+    "panel",
+    "secret",
+    "module",
+    "worker",
+    "alert",
+]
+
+
+class SearchEntry(BaseModel):
+    """One thing somebody might be looking for.
+
+    Deliberately not a route: where a category lands is the frontend's business,
+    and it already owns the router. This says what the thing is and what it is
+    called, which is all the matching needs.
+    """
+
+    category: Category
+    #: The id the frontend routes on.
+    name: str
+    #: Human title, often empty — a flow is usually only its name.
+    title: str = ""
+    #: The flow a node sits in, or the dashboard a widget sits on.
+    parent: str = ""
+    #: Node type, widget type, channel kind.
+    kind: str = ""
+
+
+def _build(
+    controller: FlowController, dashboards: DashboardStore, hub: Any, secrets: bool
+) -> list[SearchEntry]:
+    """Read every store once. Blocking: disk and git throughout.
+
+    # ponytail: rebuilt per call. Key it on ``controller.store.revision`` if a
+    # store large enough to feel it ever shows up in a profile.
+    """
+    entries: list[SearchEntry] = []
+
+    for name in controller.store.list_flows():
+        try:
+            flow = controller.store.read_flow(name, draft=True)
+        except FlowNotFound:
+            continue
+        entries.append(SearchEntry(category="flow", name=flow.name, title=flow.title))
+        entries.extend(
+            SearchEntry(
+                category="node",
+                name=node.id,
+                title=node.title,
+                parent=flow.name,
+                kind=node.type,
+            )
+            for node in flow.nodes
+        )
+
+    for summary in dashboards.list():
+        try:
+            dashboard = dashboards.read(summary.name, draft=True)
+        except DashboardNotFound:
+            continue
+        entries.append(
+            SearchEntry(
+                category="dashboard", name=dashboard.name, title=dashboard.title
+            )
+        )
+        entries.extend(
+            SearchEntry(
+                category="widget",
+                name=widget.id,
+                title=widget.title,
+                parent=dashboard.name,
+                kind=widget.type,
+            )
+            for widget in dashboard.widgets
+        )
+
+    entries.extend(
+        SearchEntry(category="panel", name=panel.id, title=panel.title)
+        for panel in panels.read_config().panels
+    )
+
+    if secrets:
+        entries.extend(
+            SearchEntry(category="secret", name=name) for name in get_secrets().list()
+        )
+
+    entries.extend(
+        SearchEntry(category="module", name=package.name, kind=package.version)
+        for package in modules.info(controller.store).packages
+    )
+
+    entries.extend(
+        SearchEntry(category="worker", name=worker.name)
+        for worker in (hub.workers() if hub is not None else [])
+    )
+
+    entries.extend(
+        SearchEntry(category="alert", name=channel.name, kind=channel.kind)
+        for channel in read_alerts_config().channels
+    )
+
+    return entries
+
+
+@router.get("/", response_model=list[SearchEntry])
+async def read_search_index(
+    current_user: CurrentUser,
+    request: Request,
+    controller: FlowControllerDep,
+    dashboards: DashboardStoreDep,
+) -> Any:
+    """Everything searchable, for the client to match against as it is typed.
+
+    The whole index rather than a query: it is a few hundred short rows for an
+    installation of any ordinary size, so one fetch when the panel opens beats a
+    round trip per keystroke — and the client already has a matcher.
+
+    Secrets are named only to a superuser, which is who ``/secrets`` answers to.
+    """
+    # Absent when remote workers are switched off — not a reason to fail a search.
+    hub = getattr(request.app.state, "worker_hub", None)
+    return await run_in_threadpool(
+        _build, controller, dashboards, hub, current_user.is_superuser
+    )

backend/fluksio/cli.py

@@ -17,10 +17,12 @@ from __future__ import annotations
 
 import argparse
 import copy
+import json
 import os
 import secrets
 import socket
 import sys
+from collections.abc import Callable
 from pathlib import Path
 from typing import Any
 
@@ -80,6 +82,39 @@ def _mention_other_installation(data_dir: Path) -> None:
     _say("           `fluksio serve --global` runs that one instead.")
 
 
+def _mention_undeclared_cards() -> None:
+    """Say when a stored flow asks for a card this engine does not have.
+
+    Cards are declared rather than detected, so an engine told nothing has
+    none — and a node asking for one is clamped to zero and runs beside every
+    other, which on a GPU is the deadlock the declaration exists to prevent.
+    The placer says so once it happens, into the log; this says it while
+    somebody is still reading the terminal.
+    """
+    from fluksio.core.config import settings
+
+    if settings.FLOW_GPUS:
+        return
+    from fluksio.flow.runs import required_resources
+    from fluksio.flow.store import FlowStore
+
+    store = FlowStore(settings.FLOWS_DIR)
+    asking = []
+    for name in store.list_flows():
+        try:
+            needs = required_resources(store.read_flow(name))
+        except Exception:
+            # A flow that will not parse is the engine's to complain about.
+            continue
+        if (needs or {}).get("gpus"):
+            asking.append(name)
+    if not asking:
+        return
+    named = ", ".join(sorted(asking)[:3]) + (" …" if len(asking) > 3 else "")
+    _say(f"  Cards    0 declared, but {named} asks for one.")
+    _say("           Nothing detects them: `--gpus N` says how many are here.")
+
+
 def _warn_if_networked(path: Path) -> None:
     """A cluster's $HOME is often NFS, and SQLite's WAL does not work there."""
     try:
@@ -228,9 +263,26 @@ CONCURRENCY_FLAGS = {
     "max_workers": "FLOW_MAX_WORKERS",
     "max_cascades": "FLOW_MAX_CASCADES",
     "max_runs": "FLOW_MAX_RUNS",
+    "gpus": "FLOW_GPUS",
 }
 
 
+def _at_least(minimum: int) -> Callable[[str], int]:
+    """A flag's value, checked here rather than by the settings.
+
+    These are written into the environment before the settings are built, so a
+    number they refuse dies inside a pydantic import with no flag named in it.
+    """
+
+    def parse(text: str) -> int:
+        value = int(text)
+        if value < minimum:
+            raise argparse.ArgumentTypeError(f"is {value}, needs at least {minimum}")
+        return value
+
+    return parse
+
+
 #: What `serve` listens on when nobody says. Taken often enough — another
 #: engine, another framework's dev server — that dying on it is the first
 #: thing a zero-config start would hit.
@@ -240,6 +292,85 @@ DEFAULT_PORT = 8000
 PORT_TRIES = 20
 
 
+#: Where a serving engine records itself, beside the data it is serving. Read
+#: to tell "this directory's engine is already up" from "something else has the
+#: port", which are the two ways a second `serve` fails to be what was wanted.
+PIDFILE = "serve.pid"
+
+
+def write_pidfile(data_dir: Path, port: int) -> Path:
+    """Record which process is serving this directory, and where."""
+    path = data_dir / PIDFILE
+    path.write_text(json.dumps({"pid": os.getpid(), "port": port}))
+    return path
+
+
+def read_pidfile(data_dir: Path) -> dict[str, int] | None:
+    """The engine serving this directory, if one still is.
+
+    A process killed outright leaves the file behind, so the pid is checked
+    rather than believed — a stale file is the same as no file.
+    """
+    try:
+        record = json.loads((data_dir / PIDFILE).read_text())
+        pid, port = int(record["pid"]), int(record["port"])
+    except (OSError, ValueError, KeyError, TypeError):
+        return None
+    try:
+        os.kill(pid, 0)
+    except (OSError, ProcessLookupError):
+        return None
+    return {"pid": pid, "port": port}
+
+
+def _token_for(data_dir: Path) -> str:
+    """The credential this directory's last engine wrote, if it wrote one."""
+    from fluksio.sdk.client import config_path
+
+    try:
+        return str(json.loads(config_path(data_dir).read_text()).get("token", ""))
+    except (OSError, ValueError, AttributeError):
+        return ""
+
+
+def probe_engine(url: str, token: str, client: Any = None) -> str:
+    """Who is on this port: ``ours``, ``foreign``, or ``other``.
+
+    ``ours`` means an engine serving *this* data directory, which is what
+    makes stopping it something this command may offer. The proof is the
+    token: it is signed with this directory's secret key, so an engine that
+    accepts it is one reading this directory's database. A Fluksio belonging
+    to another installation answers the health check and refuses the token,
+    and is only ever named — never stopped from here.
+    """
+    import logging
+
+    import httpx
+
+    # Two requests nobody asked for, on a start that is otherwise quiet until
+    # uvicorn's own banner. httpx logs every one of them at INFO.
+    noisy = logging.getLogger("httpx")
+    was = noisy.level
+    noisy.setLevel(logging.WARNING)
+    http = client or httpx.Client(timeout=2.0)
+    try:
+        health = http.get(f"{url}/api/v1/utils/health-check/")
+        if health.status_code != 200:
+            return "other"
+        # No token at all is asked unauthenticated: `Bearer ` is not a legal
+        # header value, and a Fluksio this directory cannot prove is its own
+        # is foreign — which is the answer that never stops anything.
+        auth = {"Authorization": f"Bearer {token}"} if token else {}
+        answer = http.get(f"{url}/api/v1/observability/summary", headers=auth)
+        return "ours" if answer.status_code == 200 else "foreign"
+    except Exception:
+        return "other"
+    finally:
+        noisy.setLevel(was)
+        if client is None:
+            http.close()
+
+
 def _free_port(host: str, start: int) -> int:
     """The first port from ``start`` that nothing is listening on.
 
@@ -258,6 +389,14 @@ def _free_port(host: str, start: int) -> int:
 
 
 def cmd_serve(args: argparse.Namespace) -> int:
+    # At a terminal this is a dashboard with the engine as a child of it. The
+    # import is here rather than at the top because it is only ever needed on
+    # that path, and `serve` in a container must not pay for it.
+    if not args.plain and sys.stdout.isatty() and sys.stdin.isatty():
+        from fluksio.tui import run_tui
+
+        return run_tui(args)
+
     data_dir = _data_dir(args.data_dir, args.shared)
     for flag, name in CONCURRENCY_FLAGS.items():
         value = getattr(args, flag, None)
@@ -294,15 +433,34 @@ def cmd_serve(args: argparse.Namespace) -> int:
     from fluksio.flow import modules
     from fluksio.main import app
 
+    # The client talks to this engine, and 0.0.0.0 is not an address to talk
+    # to — it is a statement about which interfaces to listen on.
+    reachable = "127.0.0.1" if args.host in ("0.0.0.0", "::", "") else args.host
+
     port = args.port
     if port is None:
         port = _free_port(args.host, DEFAULT_PORT)
         if port != DEFAULT_PORT:
-            _say(f"Port {DEFAULT_PORT} is in use; serving on {port} instead.")
+            # Moving off the port quietly makes starting a second engine for
+            # one directory look like it worked. Two of them on one SQLite
+            # file is not a supported shape — two *installations* on one
+            # machine is — so the one already up is named instead.
+            url = f"http://{reachable}:{DEFAULT_PORT}"
+            who = probe_engine(url, _token_for(data_dir))
+            if who == "ours":
+                running = read_pidfile(data_dir)
+                where = f" (pid {running['pid']})" if running else ""
+                _say(f"An engine for {data_dir} is already serving at {url}{where}.")
+                _say("  fluksio status talks to it; stop it to start another.")
+                return 0
+            if who == "foreign":
+                _say(
+                    f"Port {DEFAULT_PORT} holds another installation's Fluksio; "
+                    f"serving on {port} instead."
+                )
+            else:
+                _say(f"Port {DEFAULT_PORT} is in use; serving on {port} instead.")
 
-    # The client talks to this engine, and 0.0.0.0 is not an address to talk
-    # to — it is a statement about which interfaces to listen on.
-    reachable = "127.0.0.1" if args.host in ("0.0.0.0", "::", "") else args.host
     url = f"http://{reachable}:{port}"
     token_path = _sign_in(admin_id, url, data_dir)
 
@@ -329,17 +487,22 @@ def cmd_serve(args: argparse.Namespace) -> int:
         _say("    fluksio enroll <code>")
     _say(f"  Signed in as {admin_email}")
     _say(f"           token in {token_path}")
+    _mention_undeclared_cards()
     _mention_other_installation(data_dir)
 
     # One process: it holds the flow engine, and a second worker would be a
     # second engine — duplicated subscriptions, cron ticks and webhooks.
-    uvicorn.run(
-        app,
-        host=args.host,
-        port=port,
-        log_level=args.log_level,
-        log_config=_log_config(args.log_level),
-    )
+    pidfile = write_pidfile(data_dir, port)
+    try:
+        uvicorn.run(
+            app,
+            host=args.host,
+            port=port,
+            log_level=args.log_level,
+            log_config=_log_config(args.log_level),
+        )
+    finally:
+        pidfile.unlink(missing_ok=True)
     return 0
 
 
@@ -426,25 +589,37 @@ def _parser() -> argparse.ArgumentParser:
     )
     serve.add_argument(
         "--max-runs",
-        type=int,
+        type=_at_least(1),
         default=None,
         metavar="N",
         help="batch runs driven at once (default 4, FLOW_MAX_RUNS)",
     )
     serve.add_argument(
         "--max-cascades",
-        type=int,
+        type=_at_least(1),
         default=None,
         metavar="N",
         help="cascades in flight at once (default 4, FLOW_MAX_CASCADES)",
     )
     serve.add_argument(
         "--max-workers",
-        type=int,
+        type=_at_least(1),
         default=None,
         metavar="N",
         help="python worker processes (default 4, FLOW_MAX_WORKERS)",
     )
+    serve.add_argument(
+        "--plain",
+        action="store_true",
+        help="the log stream rather than the dashboard (the default with no terminal)",
+    )
+    serve.add_argument(
+        "--gpus",
+        type=_at_least(0),
+        default=None,
+        metavar="N",
+        help="GPUs on this machine a node may be given (default 0, FLOW_GPUS)",
+    )
     serve.set_defaults(func=cmd_serve)
 
     enroll = subparsers.add_parser(

backend/fluksio/core/config.py

@@ -9,6 +9,7 @@ from pydantic import (
     BeforeValidator,
     EmailStr,
     HttpUrl,
+    PositiveInt,
     computed_field,
     model_validator,
 )
@@ -97,15 +98,19 @@ class Settings(BaseSettings):
     # rather than a person, and it can refresh unattended.
     MCP_TOKEN_EXPIRE_MINUTES: int = 60
     MCP_REFRESH_EXPIRE_DAYS: int = 30
-    FLOW_MAX_WORKERS: int = 4
+    # The three below are all pool sizes, so 0 says neither "none" nor
+    # "unlimited" — it is a pool that cannot be built and an engine that would
+    # accept no work. Rejected here rather than quietly read as the default,
+    # because a limit somebody set and did not get is the worse surprise.
+    FLOW_MAX_WORKERS: PositiveInt = 4
     # How many cascades may be in flight at once. Sustained throughput is this
     # over the mean cascade time, so an installation whose nodes wait on the
     # network rather than on a CPU wants it higher than the core count.
-    FLOW_MAX_CASCADES: int = 4
+    FLOW_MAX_CASCADES: PositiveInt = 4
     # How many batch runs are driven at once. A different limit from the one
     # above: a run drives a whole graph, and its nodes are bounded by the worker
     # pool rather than by cascade slots. A sweep is what this governs.
-    FLOW_MAX_RUNS: int = 4
+    FLOW_MAX_RUNS: PositiveInt = 4
     # How long a python node may be silent before its worker is killed, unless
     # the node sets its own. 0, the default, disables it: a dead worker still
     # fails fast, and a slow one is left to finish. Set it where silence means

backend/fluksio/flow/connector.py

@@ -86,7 +86,14 @@ class ConnectorNode(Node):
             description="Seconds between polls; 0 polls never.",
         )
 
-    __slots__ = ("config", "_poll_task", "_stop_event", "_last_published", "_artifacts")
+    __slots__ = (
+        "config",
+        "_poll_task",
+        "_stop_event",
+        "_last_published",
+        "_artifacts",
+        "_down",
+    )
 
     def __init__(self, **kwargs: Any) -> None:
         super().__init__(f=self._dispatch, **kwargs)
@@ -95,6 +102,7 @@ class ConnectorNode(Node):
         self._stop_event: asyncio.Event | None = None
         self._last_published: dict[str, Any] = {}
         self._artifacts: ArtifactStore | None = None
+        self._down = False
 
     def _dispatch(self, params: dict[str, Any], **ports: Any) -> dict[str, Any] | None:
         """The scheduler's entry point. Settings are already on ``self.config``."""
@@ -161,11 +169,7 @@ class ConnectorNode(Node):
             return
         self._stop_event.set()
         if self._poll_task is not None:
-            self._poll_task.cancel()
-            try:
-                await self._poll_task
-            except (asyncio.CancelledError, Exception):  # noqa: B014 - shutting down
-                pass
+            await self._cancel_task(self._poll_task)
         self._poll_task = None
         self._stop_event = None
         self._last_published = {}
@@ -175,24 +179,29 @@ class ConnectorNode(Node):
 
         Only changed ports are published: a device polled every few seconds is
         usually saying the same thing, and every publication wakes everything
-        downstream of it.
+        downstream of it. What is remembered is what was *published*, not what
+        the poll returned — a publication that raised is retried next tick
+        rather than counting as said.
         """
         while not (self._stop_event and self._stop_event.is_set()):
             try:
                 values = await self.poll()
-                self.report_health("ok")
                 changed = {
                     port: value
                     for port, value in (values or {}).items()
                     if self._last_published.get(port, object()) != value
                 }
                 if changed:
-                    self._last_published.update(changed)
                     # inject runs the graph, which is blocking work.
                     await asyncio.to_thread(self.inject, changed)
+                    self._last_published.update(changed)
+                self.report_health("ok")
+                self._down = False
             except asyncio.CancelledError:
                 break
             except Exception as exc:
-                logger.warning("Connector '%s' failed to poll: %s", self.id, exc)
+                if not self._down:
+                    logger.warning("Connector '%s' failed to poll: %s", self.id, exc)
+                self._down = True
                 self.report_health("down", f"{type(exc).__name__}: {exc}")
             await asyncio.sleep(self.config.poll_interval)

backend/fluksio/flow/controller.py

@@ -916,12 +916,12 @@ class FlowController:
         placer, pool = self.placer, self.workers
 
         def call(**kwargs: Any) -> Any:
-            with placer.claim(
+            with placer.claim(  # type: ignore[union-attr]
                 wanted, device=device, policy=policy, node=node_id, run=run_id
             ) as (target, allocation):
                 env = derive_env(wanted, allocation)
                 if target.worker is None:
-                    return pool.for_env(env).run(
+                    return pool.for_env(env).run(  # type: ignore[union-attr]
                         owner,
                         local,
                         code,
@@ -1224,8 +1224,31 @@ class FlowController:
             }
         )
 
+    def _health_issues(self, flow: str | None = None) -> list[ValidationIssue]:
+        """Nodes that are running but not working, as issues on their flow.
+
+        Not part of `self.issues`: that list is what a build found, and this is
+        what is happening now. Derived on read, so a node reporting itself well
+        again clears it with nothing to remember.
+        """
+        return [
+            ValidationIssue(
+                code="node_unhealthy",
+                message=(
+                    f"Node '{entry.id.rpartition('.')[2]}' is down: "
+                    f"{entry.health_detail or 'no detail given'}"
+                ),
+                flow=entry.flow,
+                node=entry.id,
+            )
+            for entry in self.loaded.values()
+            if entry.health == "down" and (flow is None or entry.flow == flow)
+        ]
+
     def flow_issues(self, flow: str) -> list[ValidationIssue]:
-        return [issue for issue in self.issues if not issue.flow or issue.flow == flow]
+        return [
+            issue for issue in self.issues if not issue.flow or issue.flow == flow
+        ] + self._health_issues(flow)
 
     def preview(self, name: str) -> Preview:
         """Build a flow's unpublished draft without deploying it.

backend/fluksio/flow/executor.py

@@ -46,6 +46,11 @@ DELAYED_INTERVAL_S = 1.0
 #: whose nodes wait on a network rather than a CPU may want more of them —
 #: `FLOW_MAX_CASCADES` is where that is said.
 MAX_CASCADES = 4
+#: Node threads, unless the service is given a number. Both this and the one
+#: above are taken as written: only ``None`` means "nobody said", so a number
+#: that reached here is one somebody chose, and an unusable one is the pool's
+#: ``ValueError`` rather than a silent 4.
+MAX_WORKERS = 4
 # How long a reload waits for claimed work to finish before rebuilding anyway.
 DRAIN_TIMEOUT_S = 10.0
 # Work waiting in the stream, undelivered. A burst is normal — the pool claims
@@ -68,7 +73,7 @@ class ExecutionService:
     ) -> None:
         self.queue = queue
         self._events = events
-        self.max_cascades = max_cascades or MAX_CASCADES
+        self.max_cascades = MAX_CASCADES if max_cascades is None else max_cascades
         self._pipeline: Pipeline | None = None
         self._stop = threading.Event()
         # Set when a deadline moves closer, so the timer thread stops waiting
@@ -82,7 +87,8 @@ class ExecutionService:
         # Entry ids claimed and still running, under _inflight_lock.
         self._active: set[str] = set()
         self.node_pool = ThreadPoolExecutor(
-            max_workers=max_workers or 4, thread_name_prefix="node"
+            max_workers=MAX_WORKERS if max_workers is None else max_workers,
+            thread_name_prefix="node",
         )
         self._cascade_pool = ThreadPoolExecutor(
             max_workers=self.max_cascades, thread_name_prefix="cascade"

backend/fluksio/flow/nodes/base.py

@@ -212,6 +212,22 @@ class Node:
             return None
         return asyncio.create_task(factory())
 
+    @staticmethod
+    async def _cancel_task(task: asyncio.Task[None]) -> None:
+        """Stop an unsupervised loop and wait for it to be gone.
+
+        `wait` keeps whatever the task raises on its way out to itself, and
+        lets a cancellation aimed at *this* coroutine through — the
+        `except CancelledError` around `await task` it replaces swallowed that,
+        which left whoever asked for the teardown unkillable. The same trap
+        `Supervisor._cancel` documents.
+        """
+        task.cancel()
+        await asyncio.wait([task])
+        if not task.cancelled():
+            # Retrieved so a crash on the way out is not reported at exit.
+            task.exception()
+
     def report_health(self, status: str, detail: str | None = None) -> None:
         """Say how this node's connection is doing: ok, degraded or down."""
         if self._on_health is not None:

backend/fluksio/flow/nodes/delay.py

@@ -211,11 +211,7 @@ class DelayNode(Node):
         self._stop_cron.set()
 
         if self._cron_task is not None:
-            self._cron_task.cancel()
-            try:
-                await self._cron_task
-            except asyncio.CancelledError:
-                pass
+            await self._cancel_task(self._cron_task)
 
         self._cron_task = None
         self._stop_cron = None

backend/fluksio/flow/nodes/influx.py

@@ -14,6 +14,22 @@ from fluksio.flow.nodes.base import Node, NodeResult
 logger = logging.getLogger(__name__)
 
 
+def _influxdb() -> Any:
+    """The client library, which is a `fluksio[server]` extra.
+
+    Imported per use rather than at module level, because the node type is
+    registered at boot and an installation with no InfluxDB behind it should
+    not have to carry the library to start.
+    """
+    try:
+        import influxdb_client
+    except ImportError:
+        raise RuntimeError(
+            "the influxdb node needs the server extra: pip install 'fluksio[server]'"
+        ) from None
+    return influxdb_client
+
+
 class InfluxDbNode(Node):
     """
     InfluxDB node for writing to and reading from InfluxDB.
@@ -58,6 +74,7 @@ class InfluxDbNode(Node):
         - ``bucket`` (str): Bucket name (required)
         - ``write_precision`` (str): Write precision ("ns", "us", "ms", "s"), default "ms"
         - ``query_range`` (str): Default time range for queries, e.g., "-1h", "-24h"
+        - ``timeout`` (float): Deadline for a request in seconds (default: 10.0)
         - ``writes`` (dict): Write configurations keyed by message name, each with:
             - ``measurement`` (str): Measurement name to write to
             - ``field`` (str): Field name to write (default: "value")
@@ -154,6 +171,7 @@ class InfluxDbNode(Node):
         "bucket",
         "write_precision",
         "query_range",
+        "timeout",
         "writes",
         "queries",
         "_write_client",
@@ -169,6 +187,14 @@ class InfluxDbNode(Node):
         bucket: str
         write_precision: str = "ms"
         query_range: str = "-1h"
+        # Bounds every request to the server. Without one the client falls
+        # back to its own default, which no flow can see or change. Held in
+        # seconds like every other node; the client counts in milliseconds.
+        timeout: float = Field(
+            default=10.0,
+            gt=0,
+            description="Give up on a query or write after this many seconds.",
+        )
         # Per-port write and query configuration.
         writes: dict[str, dict[str, Any]] = {}
         queries: dict[str, dict[str, Any]] = {}
@@ -201,6 +227,7 @@ class InfluxDbNode(Node):
         self.bucket = cfg.bucket
         self.write_precision = cfg.write_precision
         self.query_range = cfg.query_range
+        self.timeout = cfg.timeout
         self.writes = cfg.writes
         self.queries = cfg.queries
 
@@ -279,13 +306,18 @@ class InfluxDbNode(Node):
         :returns: ``{"rows": [...], **echo}``.
         :rtype: dict
         """
-        from influxdb_client import InfluxDBClient
+        InfluxDBClient = _influxdb().InfluxDBClient
 
         flux = str(request["flux"])
         echo = {key: value for key, value in request.items() if key != "flux"}
 
         logger.info("Running Flux for node '%s': %s", self.name, flux)
-        with InfluxDBClient(url=self.url, token=self.token, org=self.org) as client:
+        with InfluxDBClient(
+            url=self.url,
+            token=self.token,
+            org=self.org,
+            timeout=int(self.timeout * 1000),
+        ) as client:
             tables = client.query_api().query(flux, org=self.org)
 
         rows = [
@@ -322,11 +354,18 @@ class InfluxDbNode(Node):
             - Dict with "value" and "tags" keys: Value written with merged tags
         :type data: dict
         """
-        from influxdb_client import InfluxDBClient, Point, WritePrecision
-        from influxdb_client.client.write_api import SYNCHRONOUS
+        influxdb_client = _influxdb()
+        InfluxDBClient = influxdb_client.InfluxDBClient
+        Point, WritePrecision = influxdb_client.Point, influxdb_client.WritePrecision
+        SYNCHRONOUS = influxdb_client.client.write_api.SYNCHRONOUS
 
         try:
-            with InfluxDBClient(url=self.url, token=self.token, org=self.org) as client:
+            with InfluxDBClient(
+                url=self.url,
+                token=self.token,
+                org=self.org,
+                timeout=int(self.timeout * 1000),
+            ) as client:
                 write_api = client.write_api(write_options=SYNCHRONOUS)
 
                 precision_map = {
@@ -417,12 +456,17 @@ class InfluxDbNode(Node):
         :returns: Dict of port name to queried value.
         :rtype: dict
         """
-        from influxdb_client import InfluxDBClient
+        InfluxDBClient = _influxdb().InfluxDBClient
 
         results = {}
 
         try:
-            with InfluxDBClient(url=self.url, token=self.token, org=self.org) as client:
+            with InfluxDBClient(
+                url=self.url,
+                token=self.token,
+                org=self.org,
+                timeout=int(self.timeout * 1000),
+            ) as client:
                 query_api = client.query_api()
 
                 for spec in self.output_ports:

backend/fluksio/flow/nodes/mqtt.py

@@ -19,9 +19,21 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-# Deep enough to ride out a broker hiccup, shallow enough that a publisher
-# which cannot keep up drops old values instead of growing without bound.
-PUBLISH_QUEUE_SIZE = 256
+
+def _aiomqtt() -> Any:
+    """The client library, which is a `fluksio[server]` extra.
+
+    Imported per use rather than at module level, because the node type is
+    registered at boot and an installation that talks to no broker should not
+    have to carry the library to start.
+    """
+    try:
+        import aiomqtt
+    except ImportError:
+        raise RuntimeError(
+            "the mqtt node needs the server extra: pip install 'fluksio[server]'"
+        ) from None
+    return aiomqtt
 
 
 def topic_matches(filter_: str, topic: str) -> bool:
@@ -83,6 +95,9 @@ class MqttNode(Node):
         - ``qos`` (int): Quality of Service level 0, 1, or 2 (default: 0)
         - ``retain`` (bool): Retain flag for published messages (default: False)
         - ``keepalive`` (int): Keepalive interval in seconds (default: 60)
+        - ``timeout`` (float): Deadline for a broker operation in seconds
+          (default: 10.0)
+        - ``publish_queue_size`` (int): Publisher backlog depth (default: 256)
     :type params: dict
     :param name: Optional name for the node.
     :type name: str | None
@@ -151,6 +166,8 @@ class MqttNode(Node):
         "qos",
         "retain",
         "keepalive",
+        "timeout",
+        "publish_queue_size",
         "json_keys",
         "_topic_to_ports",
         "_wildcards",
@@ -175,6 +192,26 @@ class MqttNode(Node):
         qos: int = 0
         retain: bool = False
         keepalive: int = 60
+        # Bounds every broker operation: subscribe, publish, and the
+        # disconnect acknowledgement on the way out. Without one a client
+        # whose socket died waits for that ack forever, and the task never
+        # finishes unwinding. Brokers differ, so it is per node.
+        timeout: float = Field(
+            default=10.0,
+            gt=0,
+            description="Give up on a broker operation after this many seconds.",
+        )
+        # Deep enough to ride out a broker hiccup, shallow enough that a
+        # publisher which cannot keep up drops old values instead of growing
+        # without bound. A node that bursts wants more than one that trickles.
+        publish_queue_size: int = Field(
+            default=256,
+            gt=0,
+            description=(
+                "How many payloads may wait for the broker. Past this the oldest "
+                "is dropped and the node reports degraded."
+            ),
+        )
         # Which key to lift out of a JSON object payload. A device that wraps
         # its reading — Victron's ``{"value": 5}`` — is otherwise a Python node
         # per port. One key for every port, or a per-port mapping.
@@ -247,6 +284,8 @@ class MqttNode(Node):
         self.qos = cfg.qos
         self.retain = cfg.retain
         self.keepalive = cfg.keepalive
+        self.timeout = cfg.timeout
+        self.publish_queue_size = cfg.publish_queue_size
 
         # Runtime state
         self._subscription_task: asyncio.Task[None] | None = None
@@ -359,7 +398,7 @@ class MqttNode(Node):
         A dropped connection raises, and the supervisor decides when to
         reconnect — the same arrangement the subscriber uses.
         """
-        import aiomqtt
+        aiomqtt = _aiomqtt()
 
         queue = self._publish_queue
         if queue is None:
@@ -372,6 +411,7 @@ class MqttNode(Node):
             password=self.password,
             identifier=self.client_id,
             keepalive=self.keepalive,
+            timeout=self.timeout,
         ) as client:
             self.report_health("ok")
             while True:
@@ -384,7 +424,7 @@ class MqttNode(Node):
 
     async def _publish_once(self, data: dict[str, Any]) -> None:
         """Connect, publish, disconnect — the unstarted node's path."""
-        import aiomqtt
+        aiomqtt = _aiomqtt()
 
         async with aiomqtt.Client(
             hostname=self.broker_host,
@@ -393,6 +433,7 @@ class MqttNode(Node):
             password=self.password,
             identifier=self.client_id,
             keepalive=self.keepalive,
+            timeout=self.timeout,
         ) as client:
             await self._publish_with(client, data)
 
@@ -452,7 +493,7 @@ class MqttNode(Node):
         """Run the task that owns this node's connection to the broker."""
         if self._publish_queue is not None:
             return
-        self._publish_queue = asyncio.Queue(maxsize=PUBLISH_QUEUE_SIZE)
+        self._publish_queue = asyncio.Queue(maxsize=self.publish_queue_size)
         self._loop = asyncio.get_running_loop()
         self._publisher_task = self._run_supervised("mqtt-out", self._publisher_loop)
 
@@ -461,11 +502,7 @@ class MqttNode(Node):
         if self._publish_queue is None:
             return
         if self._publisher_task is not None:
-            self._publisher_task.cancel()
-            try:
-                await self._publisher_task
-            except (asyncio.CancelledError, Exception):  # noqa: B014 - shutting down
-                pass
+            await self._cancel_task(self._publisher_task)
         self._publisher_task = None
         self._publish_queue = None
         self._loop = None
@@ -515,11 +552,7 @@ class MqttNode(Node):
         self._stop_event.set()
 
         if self._subscription_task is not None:
-            self._subscription_task.cancel()
-            try:
-                await self._subscription_task
-            except asyncio.CancelledError:
-                pass
+            await self._cancel_task(self._subscription_task)
 
         self._subscription_task = None
         self._stop_event = None
@@ -553,7 +586,7 @@ class MqttNode(Node):
         """
         import json
 
-        import aiomqtt
+        aiomqtt = _aiomqtt()
 
         if not (self._stop_event and self._stop_event.is_set()):
             try:
@@ -564,6 +597,7 @@ class MqttNode(Node):
                     password=self.password,
                     identifier=self.client_id,
                     keepalive=self.keepalive,
+                    timeout=self.timeout,
                 ) as client:
                     # Subscribe to every unique topic
                     for topic in self._topic_to_ports:

backend/fluksio/flow/pipeline.py

@@ -59,6 +59,7 @@ class ValidationIssue(BaseModel):
         "unauthenticated_hook",
         "self_loop_needs_initial",
         "missing_source",
+        "node_unhealthy",
     ]
     message: str
     flow: str = ""

backend/fluksio/flow/placement.py

@@ -215,9 +215,18 @@ class Placer:
         said = [shape[2] for shape in capable if shape[2]]
         ram = min(ram, max(said)) if said and ram else ram
         if (cpus, gpus, ram) != (wanted.cpus, wanted.gpus, wanted.ram or 0):
+            # Cards are not detected, so a machine that has one still reports
+            # none until it is told — which reads as "no GPU here" to a node
+            # that then runs unserialised beside every other one.
+            hint = (
+                "; no machine here declares a GPU — `fluksio serve --gpus N` "
+                "(or FLOW_GPUS) says how many this one has"
+                if wanted.gpus and not gpus
+                else ""
+            )
             logger.warning(
                 "%s asked for %d cpu(s), %d gpu(s) and %s MB; "
-                "the largest machine here can give %d, %d and %s",
+                "the largest machine here can give %d, %d and %s%s",
                 node or "a node",
                 wanted.cpus,
                 wanted.gpus,
@@ -225,6 +234,7 @@ class Placer:
                 cpus,
                 gpus,
                 ram or "no stated",
+                hint,
             )
         return cpus, gpus, ram
 

backend/fluksio/flow/runs.py

@@ -769,7 +769,9 @@ class RunService:
         # the isolation it wants, minus surviving the process.
         self._state_factory = state_factory or (lambda _ns: MemoryState())
         self.engine_name = f"{socket.gethostname()}-{os.getpid()}"[:64]
-        self.parallel = max(1, parallel)
+        # Taken as written: clamping a 0 up to 1 would hide a limit somebody
+        # set, and the pool below rejects an unusable one loudly anyway.
+        self.parallel = parallel
         self._pool = ThreadPoolExecutor(
             max_workers=self.parallel, thread_name_prefix="run"
         )
@@ -844,6 +846,26 @@ class RunService:
         # reference a python caller would have passed and every later reader —
         # the digest, the cache, the run detail — sees one spelling.
         params = resolve_references(flow, params, self._artifacts)
+        # What the run actually starts from, not only what was passed: an input
+        # left out takes its declared value, and a row that records `{}` cannot
+        # say which. Folded literally — an initial is a value from the
+        # definition, never a reference to resolve.
+        declared = {
+            one.spec.name: one.initial for one in flow.inputs if one.initial is not None
+        }
+        # The run's own seed fills an input of that name, outranking what the
+        # flow declares and outranked by one passed as a parameter — the order
+        # `seed_values` applies, moved to where the record is written.
+        if seed is not None and any(one.spec.name == "seed" for one in flow.inputs):
+            declared["seed"] = seed
+        params = {**declared, **params}
+        # And back the other way, so the run-level column holds the seed the
+        # run actually used however it arrived. Otherwise `--seed 1` fills one
+        # column and a declared seed the other, and that is the single field an
+        # export still has to coalesce.
+        resolved_seed = params.get("seed")
+        if isinstance(resolved_seed, int) and not isinstance(resolved_seed, bool):
+            seed = resolved_seed
         # Checked here rather than in the driver: a caller who mistyped a
         # parameter should be told now, not by a run that fails in a minute.
         seed_values(flow, params, seed)
@@ -1174,6 +1196,7 @@ class RunService:
             self._finish(run_id, status, reason, result, duration)
             run.status = status
             self._publish(run, "run_finished")
+            self._release_cards(run)
             # Its values were only ever this run's; nothing reads them once it
             # has a result. On Redis the namespace would expire anyway.
             if state is not None and status != "error":
@@ -1182,6 +1205,29 @@ class RunService:
                 except Exception:
                     logger.warning("Could not clear state of run %s", run_id)
 
+    def _release_cards(self, run: Run) -> None:
+        """Hand a GPU run's device memory back when the run is over.
+
+        The accountant frees the card the moment the node returns, but the
+        worker that ran on it is kept warm and a library that preallocated
+        most of the VRAM never gives it up — so the next process to want the
+        card found it taken by one sitting idle.
+        """
+        # ponytail: retires every CUDA pool rather than the ones this run used,
+        # which needs no bookkeeping — a concurrent GPU run's busy worker only
+        # dies when it returns, which is when its own memory should go back
+        # anyway. What it costs is the warm worker of a *live* flow's GPU node.
+        # Track the pools per run if that ever matters.
+        if not (run.needs or {}).get("gpus"):
+            return
+        pool = getattr(self.controller, "workers", None)
+        if pool is None:
+            return
+        try:
+            pool.retire_gpu_children()
+        except Exception:
+            logger.warning("Could not retire the GPU workers of run %s", run.id)
+
     def _record_node(self, run_id: str, outcome: NodeOutcome) -> None:
         outputs = _cacheable(outcome)
         row = RunNode(

backend/fluksio/flow/workers.py

@@ -321,6 +321,24 @@ class PythonWorkerPool:
         for child in children:
             child.respawn_all()
 
+    def retire_gpu_children(self) -> None:
+        """Retire the pools holding a card, so the VRAM goes back.
+
+        A library like JAX takes most of the device when it imports and never
+        releases it, so a warm worker that has run one such node is a held
+        card — and warm is the point of a pool, so nothing retires it. At the
+        end of a run there is something to key on: the environments carrying a
+        GPU assignment are exactly the pools that ran on one.
+        """
+        with self._lock:
+            children = [
+                child
+                for key, child in self._children.items()
+                if any(name == "CUDA_VISIBLE_DEVICES" for name, _ in key)
+            ]
+        for child in children:
+            child.respawn_all()
+
     def _drain(self) -> list[_Worker | None]:
         slots = []
         while True:

backend/fluksio/main.py

@@ -4,7 +4,6 @@ import logging
 from collections.abc import AsyncIterator
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
 
-import sentry_sdk
 from fastapi import FastAPI, Request
 from fastapi.concurrency import run_in_threadpool
 from fastapi.responses import JSONResponse
@@ -51,6 +50,10 @@ def custom_generate_unique_id(route: APIRoute) -> str:
 
 
 if settings.SENTRY_DSN and settings.ENVIRONMENT != "local":
+    # Imported here rather than at the top: it is a `fluksio[server]` extra, so
+    # a pip install without one has no sentry to import — and no DSN either.
+    import sentry_sdk
+
     # `enable_tracing` was removed in sentry-sdk 2.x; this is what it meant.
     sentry_sdk.init(dsn=str(settings.SENTRY_DSN), traces_sample_rate=1.0)
 

backend/fluksio/sdk/__init__.py

@@ -388,11 +388,6 @@ def _check_signature(spec: NodeSpec) -> None:
     fn, where = spec.fn, f"node '{spec.id}'"
     if inspect.iscoroutinefunction(fn):
         raise SyncError(f"{where}: async functions cannot be nodes")
-    if fn.__module__ == "__main__":
-        raise SyncError(
-            f"{where}: {fn.__name__}() is defined in a script run directly, so the "
-            "generated node could not import it — put it in an importable module"
-        )
     generator = inspect.isgeneratorfunction(fn)
     if generator and spec.single:
         raise SyncError(
@@ -551,6 +546,21 @@ def _check(spec: NodeSpec, mode: str) -> dict[str, Any]:
     }
 
 
+def _refuse_main(spec: NodeSpec) -> None:
+    """A node the generated body would have no way to import.
+
+    Refused where the body is written rather than where the decorator is: a
+    module defining nodes is then still runnable as a script, which is what a
+    `__main__` self-check beside them needs.
+    """
+    if spec.fn.__module__ == "__main__":
+        raise SyncError(
+            f"node '{spec.id}': {spec.fn.__name__}() is defined in a script run "
+            "directly, so the generated node could not import it — put it in an "
+            "importable module"
+        )
+
+
 def _code_of(spec: NodeSpec) -> dict[str, Any]:
     """What this node's function calls into, which its shim does not say.
 
@@ -559,6 +569,7 @@ def _code_of(spec: NodeSpec) -> dict[str, Any]:
     side is the only one that can work it out at all: it has imported the
     code, and the engine never does.
     """
+    _refuse_main(spec)
     files = reached(spec.fn)
     return {"code_files": files, "code_digest": digest_of(files)}
 
@@ -762,6 +773,7 @@ def _shim(spec: NodeSpec) -> str:
     simply happens to be generated, which is why it says so and says where the
     real thing is.
     """
+    _refuse_main(spec)
     fn = spec.fn
     where = inspect.getsourcefile(fn) or fn.__module__
     repo = import_root(fn)

backend/fluksio/sdk/cli.py

@@ -19,6 +19,7 @@ import sys
 import time
 from collections.abc import Callable, Iterable, Iterator
 from contextlib import contextmanager, nullcontext
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
@@ -136,8 +137,8 @@ def _import(root: str, dotted: str, expect: Path | None = None) -> None:
             f"  {actual}\n  {expect}\n"
             "Python keeps one module per name, and a node's generated body "
             "imports by that name, so the second would run the first's code. "
-            "Put an `__init__.py` in each directory — they become "
-            f"'<dir>.{dotted}' and stop colliding — or rename one of the files."
+            "Sync the directory they are both under — each then imports as "
+            "'<dir>.<file>' — or rename one of the files."
         )
 
 
@@ -211,7 +212,14 @@ def discover(targets: list[str], keep_going: bool = False) -> list[Flow]:
                 if entry.is_dir():
                     _import_package(entry)
                 else:
-                    _import(*_module_of(entry), entry)
+                    # Named for where it sits under the directory being synced,
+                    # so `dev/s1/study.py` imports as `s1.study` and one
+                    # `study.py` per study collides with nothing. No
+                    # `__init__.py` needed: the directories in between are
+                    # namespace packages. A file at the root keeps its bare
+                    # name, which is what it has always had.
+                    dotted = ".".join(entry.relative_to(path).with_suffix("").parts)
+                    _import(str(path), dotted, entry)
             except SyncError:
                 # A name collision is never somebody else's problem: it would
                 # put the wrong file behind a node.
@@ -439,33 +447,52 @@ def _ask_params(definition: dict[str, Any]) -> dict[str, Any]:
 def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
     """Turn `--lr 0.05` into a typed parameter, using the flow's own inputs."""
     types = _input_types(definition)
-    params: dict[str, Any] = {}
+    # Collected as written and typed afterwards, so a name this flow does not
+    # have is refused by name rather than by whatever its value failed to parse
+    # as: `--param lr=0.002` is a sweep's spelling, and said so it reads as an
+    # input called `param` holding unparseable json.
+    raw: dict[str, str | bool] = {}
     pending: str | None = None
     for token in rest:
         if token.startswith("--"):
             if pending is not None:
                 # A flag with no value is a flag: `--resume` means true.
-                params[pending] = True
+                raw[pending] = True
             name, sep, value = token[2:].partition("=")
             # Only the name is spelled with dashes; a value may hold one, and
             # `--lr=1e-4` is the case that says so.
             pending = name.replace("-", "_")
             if sep:
-                params[pending] = _coerce(value, types.get(pending, "json"))
+                raw[pending] = value
                 pending = None
             continue
         if pending is None:
             raise SyncError(f"unexpected argument '{token}'")
-        params[pending] = _coerce(token, types.get(pending, "json"))
+        raw[pending] = token
         pending = None
     if pending is not None:
-        params[pending] = True
-    unknown = sorted(set(params) - set(types))
+        raw[pending] = True
+    unknown = sorted(set(raw) - set(types))
     if unknown:
+        hint = ""
+        if unknown[0] == "param":
+            hint = (
+                " — one value is `--<name> <value>`; several is a sweep: "
+                "`fluksio sweep --param name=v1,v2`"
+            )
         raise SyncError(
             f"'{unknown[0]}' is not an input of this flow (it takes "
-            f"{', '.join(sorted(types)) or 'none'})"
+            f"{', '.join(sorted(types)) or 'none'}){hint}"
         )
+    params: dict[str, Any] = {}
+    for key, written in raw.items():
+        if isinstance(written, bool):
+            params[key] = written
+            continue
+        try:
+            params[key] = _coerce(written, types[key])
+        except ValueError as exc:
+            raise SyncError(f"'{key}' takes {types[key]}: {exc}") from exc
     return params
 
 
@@ -778,7 +805,8 @@ def _status_screen(client: Client) -> Any:
                 )
                 + Text(
                     f"{str(row.get('flow', '')):<16}"
-                    f"{(row.get('duration_ms') or 0) / 1000:7.1f}s",
+                    f"{(row.get('duration_ms') or 0) / 1000:7.1f}s"
+                    f"  {_ago(row.get('finished_at') or row.get('created_at')):>9}",
                     style="dim",
                 )
             )
@@ -790,6 +818,7 @@ def _status_screen(client: Client) -> Any:
             )
             parts.append(
                 Text(f"  {where}  ", style="red")
+                + Text(f"{_ago(event.get('ts')):>9}  ", style="dim")
                 + Text(str(event.get("detail", ""))[:100], style="dim")
             )
     return Group(*parts)
@@ -831,9 +860,9 @@ def cmd_status(args: argparse.Namespace) -> int:
 #: value is read.
 PARAMS_WIDTH = 80
 
-#: What the columns before the inputs take: the id, status, flow, duration and
-#: stamp, with their spacing.
-LISTING_WIDTH = 84
+#: What the columns before the inputs take: the id, status, flow, duration,
+#: age and stamp, with their spacing.
+LISTING_WIDTH = 95
 
 
 def _declared(client: Client, flows: Iterable[str]) -> dict[str, dict[str, Any]]:
@@ -857,6 +886,28 @@ def _declared(client: Client, flows: Iterable[str]) -> dict[str, dict[str, Any]]
     return known
 
 
+def _ago(stamp: Any) -> str:
+    """How long ago something happened, in the notation the screens use.
+
+    A duration and an age read together, so they are spelled the same way:
+    seconds, then minutes, hours, days.
+    """
+    if not stamp:
+        return ""
+    try:
+        then = datetime.fromisoformat(str(stamp))
+    except ValueError:
+        return ""
+    if then.tzinfo is None:
+        # Everything the engine records is UTC; only some spellings say so.
+        then = then.replace(tzinfo=UTC)
+    seconds = max((datetime.now(UTC) - then).total_seconds(), 0)
+    for span, unit in ((86400, "d"), (3600, "h"), (60, "min")):
+        if seconds >= span:
+            return f"{seconds / span:.0f}{unit} ago"
+    return f"{seconds:.0f}s ago"
+
+
 def _stamp(row: dict[str, Any]) -> str:
     """What code a run ran: the commit, whether it was dirty, and the digest.
 
@@ -899,11 +950,49 @@ def cmd_runs(args: argparse.Namespace) -> int:
             params = params[: room - 3] + "..."
         _say(
             f"{row['id']}  {_status(row['status'], 9)} {row['flow']:<16} "
-            f"{row['duration_ms'] / 1000:7.1f}s  {_stamp(row):<22} {params}"
+            f"{row['duration_ms'] / 1000:7.1f}s {_ago(row.get('created_at')):>9}  "
+            f"{_stamp(row):<22} {params}"
         )
     return 0
 
 
+def _artifacts(client: Client, args: argparse.Namespace) -> int:
+    """List a run's files, or write one of them here."""
+    handle = RunHandle(client, args.run_id, client.run(args.run_id))
+    rows = handle.artifacts
+    if not args.name:
+        if not rows:
+            _say("This run produced no artifacts.")
+            return 0
+        for row in rows:
+            named = row.get("filename") or ""
+            _say(
+                f"{str(row['name']):<24} {row['size']:>10} B  "
+                f"{row.get('media_type', ''):<24} {named}"
+            )
+        return 0
+    data = handle.download(args.name)
+    # The name it was written under reads better than the message's, which is
+    # chosen for the graph; `--out` beats both.
+    match = next((row for row in rows if row.get("name") == args.name), {})
+    out = Path(args.out or match.get("filename") or args.name)
+    out.write_bytes(data)
+    _say(f"{out}  {len(data)} bytes")
+    return 0
+
+
+def cmd_artifacts(args: argparse.Namespace) -> int:
+    try:
+        with _client_for(args, retries=0) as client:
+            return _artifacts(client, args)
+    except KeyError as exc:
+        return _fail(str(exc.args[0]))
+    except (SyncError, ApiError) as exc:
+        return _fail(str(exc))
+    except httpx.HTTPError as exc:
+        return _unreachable(exc)
+
+
 def cmd_flavors(args: argparse.Namespace) -> int:
     """The named sizes a node can ask for."""
     try:
@@ -1267,6 +1356,21 @@ def add_parsers(subparsers: Any) -> None:
     with_engine(parser, local=True)
     parser.set_defaults(func=cmd_runs)
 
+    parser = subparsers.add_parser(
+        "artifacts", help="the files a run produced; name one to download it"
+    )
+    parser.add_argument("run_id")
+    parser.add_argument("name", nargs="?", default="")
+    parser.add_argument(
+        "-o",
+        "--out",
+        default="",
+        metavar="PATH",
+        help="where to write it (default: the name it was saved under)",
+    )
+    with_engine(parser, local=True)
+    parser.set_defaults(func=cmd_artifacts)
+
     parser = subparsers.add_parser(
         "flavors", help="the named resource sizes a node can ask for"
     )
@@ -1375,7 +1479,7 @@ def add_parsers(subparsers: Any) -> None:
         metavar="A,B",
         help=(
             "the inputs to put in columns, dotted into a record "
-            "(default: the ones that vary)"
+            "(default: every input the runs recorded)"
         ),
     )
     sub.add_argument(

backend/fluksio/sdk/client.py

@@ -428,9 +428,9 @@ class Client:
     ) -> list[dict[str, Any]]:
         """One row per run: its inputs as columns, its final numbers, its code.
 
-        The arm-comparison table. The inputs kept are the ones that vary
-        across the selection unless ``params`` names them, which is the axis a
-        sweep is read along.
+        The arm-comparison table. Every input the selection recorded is a
+        column, so the schema does not depend on which runs were asked for;
+        ``params`` narrows it to the axis a sweep is read along.
         """
         query: dict[str, Any] = {}
         if params:

backend/fluksio/tui.py

@@ -0,0 +1,342 @@
+"""The dashboard `fluksio serve` opens at a terminal.
+
+A supervisor, not an engine: it starts `serve --plain` as a child and talks to
+it over the same HTTP API every other client uses. So the engine is a process
+that can be stopped and started under the screen watching it, an engine
+somebody else started can be adopted rather than duplicated, and quitting the
+dashboard is not the same as stopping the engine.
+
+Nothing of the engine is imported here — that lives in the child.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import signal
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+from textual import work
+from textual.app import App, ComposeResult
+from textual.containers import Horizontal, Vertical
+from textual.screen import ModalScreen
+from textual.widgets import Button, DataTable, Footer, Header, Input, Label, RichLog
+
+from fluksio.cli import (
+    DEFAULT_PORT,
+    DEFAULT_PORTAL,
+    _data_dir,
+    _token_for,
+    probe_engine,
+    read_pidfile,
+)
+from fluksio.sdk.cli import WATCH_INTERVAL_S, _status_screen
+from fluksio.sdk.client import Client, config_path
+
+#: How long to keep asking the child for the credential it writes on the way
+#: up, before giving up and letting the panels say so.
+STARTUP_TRIES = 60
+
+
+def child_argv(argv: list[str]) -> list[str]:
+    """The same command, told to serve without a dashboard of its own.
+
+    Every flag is passed through: whatever `serve` was asked for is what the
+    engine under this screen is running with.
+    """
+    passed = [flag for flag in argv if flag != "--plain"]
+    return [sys.executable, "-m", "fluksio.cli", *passed, "--plain"]
+
+
+class Enroll(ModalScreen[tuple[str, str] | None]):
+    """The claim code a portal minted, and which portal minted it."""
+
+    BINDINGS = [("escape", "dismiss(None)", "cancel")]
+
+    def compose(self) -> ComposeResult:
+        with Vertical(id="enroll"):
+            yield Label("Pair this installation with a portal")
+            yield Input(placeholder="claim code", id="code")
+            yield Input(value=DEFAULT_PORTAL, id="portal")
+            with Horizontal():
+                yield Button("Enroll", variant="primary", id="go")
+                yield Button("Cancel", id="cancel")
+
+    def on_button_pressed(self, event: Button.Pressed) -> None:
+        if event.button.id == "cancel":
+            self.dismiss(None)
+            return
+        code = self.query_one("#code", Input).value.strip()
+        portal = self.query_one("#portal", Input).value.strip()
+        self.dismiss((code, portal or DEFAULT_PORTAL) if code else None)
+
+
+class ServeApp(App[int]):
+    """One screen: how the engine is, what has run, and what it is saying."""
+
+    CSS = """
+    #status { height: auto; padding: 0 1; }
+    #runs { height: 2fr; }
+    #log { height: 1fr; border-top: solid $panel; }
+    #enroll { width: 60; height: auto; padding: 1 2; background: $surface; }
+    """
+
+    BINDINGS = [
+        ("q", "quit", "quit (engine keeps running)"),
+        ("s", "stop_start", "stop/start"),
+        ("r", "restart", "restart"),
+        ("c", "cancel_run", "cancel run"),
+        ("e", "enroll", "enroll"),
+    ]
+
+    def __init__(self, args: argparse.Namespace) -> None:
+        super().__init__()
+        self.args = args
+        self.data_dir: Path = _data_dir(args.data_dir, args.shared)
+        self.child: subprocess.Popen[str] | None = None
+        #: The pid of an engine this dashboard did not start. Only ever one
+        #: whose data directory is this one — the probe is what proves it.
+        self.adopted: int | None = None
+        self.client: Client | None = None
+        self.url = ""
+
+    # -- layout ---------------------------------------------------------------
+
+    def compose(self) -> ComposeResult:
+        yield Header()
+        yield RichLog(id="status", markup=True, wrap=True)
+        table: DataTable[str] = DataTable(id="runs", cursor_type="row")
+        yield table
+        yield RichLog(id="log", markup=False, highlight=False, max_lines=2000)
+        yield Footer()
+
+    def on_mount(self) -> None:
+        self.title = f"fluksio — {self.data_dir}"
+        table = self.query_one("#runs", DataTable)
+        table.add_columns("run", "status", "flow", "took", "params")
+        self.start_engine(first=True)
+        self.set_interval(WATCH_INTERVAL_S, self.refresh_panels)
+
+    # -- the engine under the screen ------------------------------------------
+
+    def note(self, message: str) -> None:
+        self.query_one("#log", RichLog).write(message)
+
+    def start_engine(self, first: bool = False) -> None:
+        """Adopt whatever is already serving this directory, or start one."""
+        host = self.args.host
+        reachable = "127.0.0.1" if host in ("0.0.0.0", "::", "") else host
+        wanted = self.args.port or DEFAULT_PORT
+        url = f"http://{reachable}:{wanted}"
+        who = probe_engine(url, _token_for(self.data_dir)) if first else "other"
+
+        if who == "ours":
+            running = read_pidfile(self.data_dir)
+            self.adopted = running["pid"] if running else None
+            self.url = url
+            named = f" (pid {self.adopted})" if self.adopted else ""
+            self.note(f"Adopted the engine already serving this directory{named}.")
+            if self.adopted is None:
+                self.note("It wrote no pidfile, so this screen cannot stop it.")
+            self.connect()
+            return
+        if who == "foreign":
+            self.note(f"Port {wanted} holds another installation's Fluksio.")
+
+        self.child = subprocess.Popen(  # noqa: S603
+            child_argv(sys.argv[1:]),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+            env={**os.environ, "PYTHONUNBUFFERED": "1"},
+        )
+        self.tail_child(self.child)
+        self.await_engine()
+
+    @work(thread=True, exclusive=False)
+    def tail_child(self, child: subprocess.Popen[str]) -> None:
+        """The engine's own output, which is why a second terminal was needed."""
+        if child.stdout is None:
+            return
+        for line in child.stdout:
+            self.call_from_thread(self.note, line.rstrip())
+        self.call_from_thread(self.note, f"The engine stopped ({child.wait()}).")
+
+    @work(thread=True, exclusive=True, group="startup")
+    def await_engine(self) -> None:
+        """Wait for the child to say where it landed, then talk to it.
+
+        The port is the child's to choose — it moves off a taken one — and
+        `client.json` is where it says which it took.
+        """
+        import time
+
+        for _ in range(STARTUP_TRIES):
+            if self.child is not None and self.child.poll() is not None:
+                return
+            try:
+                stored = json.loads(config_path(self.data_dir).read_text())
+            except (OSError, ValueError):
+                stored = {}
+            if stored.get("url") and stored.get("token"):
+                self.url = str(stored["url"])
+                self.call_from_thread(self.connect)
+                return
+            time.sleep(0.5)
+        self.call_from_thread(self.note, "The engine did not come up.")
+
+    def connect(self) -> None:
+        stored = config_path(self.data_dir)
+        try:
+            config = json.loads(stored.read_text())
+            self.client = Client(
+                url=str(config["url"]), token=str(config["token"]), retries=0
+            )
+        except (OSError, ValueError, KeyError) as exc:
+            self.note(f"No credential to talk to the engine with: {exc}")
+            return
+        self.url = self.client.url
+        self.refresh_panels()
+
+    def engine_pid(self) -> int | None:
+        return self.child.pid if self.child is not None else self.adopted
+
+    def stop_engine(self) -> None:
+        if self.child is not None:
+            self.note(f"Stopping the engine (pid {self.child.pid}).")
+            self.child.terminate()
+            self.child = None
+        elif self.adopted is not None:
+            self.note(f"Stopping the adopted engine (pid {self.adopted}).")
+            try:
+                os.kill(self.adopted, signal.SIGTERM)
+            except OSError as exc:
+                self.note(f"Could not stop it: {exc}")
+            self.adopted = None
+        self.client = None
+
+    # -- what the panels show -------------------------------------------------
+
+    def refresh_panels(self) -> None:
+        if self.client is not None:
+            self.read_engine()
+
+    @work(thread=True, exclusive=True, group="poll")
+    def read_engine(self) -> None:
+        client = self.client
+        if client is None:
+            return
+        try:
+            screen = _status_screen(client)
+            rows = client.runs(limit=20)
+        except Exception as exc:
+            self.call_from_thread(self.show_offline, exc)
+            return
+        self.call_from_thread(self.show, screen, rows)
+
+    def show(self, screen: Any, rows: list[dict[str, Any]]) -> None:
+        status = self.query_one("#status", RichLog)
+        status.clear()
+        status.write(screen)
+        table = self.query_one("#runs", DataTable)
+        table.clear()
+        for row in rows:
+            table.add_row(
+                str(row.get("id", ""))[-8:],
+                str(row.get("status", "")),
+                str(row.get("flow", "")),
+                f"{(row.get('duration_ms') or 0) / 1000:.1f}s",
+                json.dumps(row.get("params") or {})[:60],
+                key=str(row.get("id", "")),
+            )
+
+    def show_offline(self, exc: Exception) -> None:
+        status = self.query_one("#status", RichLog)
+        status.clear()
+        which = "starting" if self.engine_pid() is not None else "not running"
+        status.write(f"[yellow]The engine is {which}.[/] ({type(exc).__name__})")
+
+    # -- keys -----------------------------------------------------------------
+
+    def action_stop_start(self) -> None:
+        if self.engine_pid() is not None:
+            self.stop_engine()
+        else:
+            self.start_engine()
+
+    def action_restart(self) -> None:
+        self.stop_engine()
+        self.start_engine()
+
+    def action_cancel_run(self) -> None:
+        table = self.query_one("#runs", DataTable)
+        if self.client is None or not table.row_count:
+            return
+        # The cell holds the tail of the id, which is what fits; the row's key
+        # is the whole of it, which is what the engine is asked about.
+        key = table.coordinate_to_cell_key(table.cursor_coordinate).row_key
+        if key.value:
+            self.cancel_run(str(key.value))
+
+    @work(thread=True)
+    def cancel_run(self, run_id: str) -> None:
+        client = self.client
+        if client is None:
+            return
+        try:
+            client.cancel(run_id)
+        except Exception as exc:
+            self.call_from_thread(self.note, f"Could not cancel {run_id}: {exc}")
+            return
+        self.call_from_thread(self.note, f"Cancelled {run_id}.")
+        self.call_from_thread(self.refresh_panels)
+
+    def action_enroll(self) -> None:
+        self.push_screen(Enroll(), self.enrolled)
+
+    def enrolled(self, answer: tuple[str, str] | None) -> None:
+        if answer is None:
+            return
+        code, portal = answer
+        self.run_enroll(code, portal)
+
+    @work(thread=True)
+    def run_enroll(self, code: str, portal: str) -> None:
+        """`fluksio enroll`, as its own process for the same reason serve is.
+
+        A running engine picks the configuration up on its own; this screen
+        only has to report what the command said.
+        """
+        done = subprocess.run(  # noqa: S603
+            [
+                sys.executable,
+                "-m",
+                "fluksio.cli",
+                "enroll",
+                code,
+                "--portal",
+                portal,
+                "--data-dir",
+                str(self.data_dir),
+            ],
+            capture_output=True,
+            text=True,
+        )
+        for line in (done.stdout + done.stderr).splitlines():
+            self.call_from_thread(self.note, line)
+
+
+def run_tui(args: argparse.Namespace) -> int:
+    """Open the dashboard, and say what is still running when it closes."""
+    app = ServeApp(args)
+    app.run()
+    pid, url = app.engine_pid(), app.url
+    if pid is not None:
+        print(f"The engine is still running: pid {pid} at {url or 'its port'}.")
+        print(f"  fluksio serve reattaches to it; kill {pid} stops it.")
+    return 0

backend/fluksio/utils.py

@@ -4,7 +4,6 @@ from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from typing import Any
 
-import emails  # type: ignore
 import jwt
 from jinja2 import Template
 from jwt.exceptions import InvalidTokenError
@@ -37,6 +36,12 @@ def send_email(
     html_content: str = "",
 ) -> None:
     assert settings.emails_enabled, "no provided configuration for email variables"
+    try:
+        import emails  # type: ignore
+    except ImportError:
+        raise RuntimeError(
+            "sending mail needs the server extra: pip install 'fluksio[server]'"
+        ) from None
     message = emails.Message(
         subject=subject,
         html=html_content,

backend/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fluksio"
-version = "0.1.4"
+version = "0.1.4+dev"
 description = "Node-based automation engine: flows, dashboards, batch runs"
 readme = "README.md"
 license = "AGPL-3.0-or-later"
@@ -25,15 +25,12 @@ dependencies = [
     "fastapi[standard]<1.0.0,>=0.114.2",
     "python-multipart<1.0.0,>=0.0.7",
     "email-validator<3.0.0.0,>=2.1.0.post1",
-    "tenacity<9.0.0,>=8.2.3",
     "pydantic>2.0",
-    "emails<1.0,>=0.6",
     "jinja2<4.0.0,>=3.1.4",
     "alembic<2.0.0,>=1.12.1",
     "httpx<1.0.0,>=0.25.1",
     "sqlmodel<1.0.0,>=0.0.21",
     "pydantic-settings<3.0.0,>=2.2.1",
-    "sentry-sdk[fastapi]>=2.20.0",
     "pyjwt<3.0.0,>=2.8.0",
     "pwdlib[argon2,bcrypt]>=0.3.0",
     "numpy>=2.2.6",
@@ -43,8 +40,6 @@ dependencies = [
     # copied onto other people's machines and stays dependency-free.
     "orjson>=3.10",
     "cryptography>=44.0.0",
-    "aiomqtt>=2.0.0",
-    "influxdb-client[async]>=1.40.0",
     "croniter>=1.3.0",
     "mcp>=1.29,<2",
     "fluksio-worker>=0.1,<0.2",
@@ -55,6 +50,9 @@ dependencies = [
     # `fluksio status` draws with it. Already here underneath fastapi's CLI,
     # named because a command that depends on it should say so.
     "rich>=13",
+    # `fluksio serve` opens a dashboard with it at a terminal. Pure python and
+    # mostly rich underneath, which is already here.
+    "textual>=1.0",
 ]
 
 [project.optional-dependencies]
@@ -62,6 +60,17 @@ dependencies = [
 # extra rather than a dependency: csv and jsonl need nothing, and pyarrow is
 # tens of megabytes for whoever wants dtypes kept.
 parquet = ["pyarrow>=17"]
+# What a deployment has and a laptop does not: the device connectors, outbound
+# mail and error reporting. The engine, the CLI and every python node work
+# without them — `pip install fluksio` in a data-science environment is the
+# case this exists for, and lxml and the aiohttp stack are most of its wait.
+# Each import is guarded and names this extra. The image installs it.
+server = [
+    "emails<1.0,>=0.6",
+    "sentry-sdk[fastapi]>=2.20.0",
+    "aiomqtt>=2.0.0",
+    "influxdb-client[async]>=1.40.0",
+]
 
 [project.urls]
 Homepage = "https://fluksio.com"
@@ -146,6 +155,8 @@ ignore = [
 # It talks to whoever ran it; that is what a command line is.
 "fluksio/cli.py" = ["T201"]
 "fluksio/sdk/cli.py" = ["T201"]
+# What it prints is the line left in the terminal after the dashboard closes.
+"fluksio/tui.py" = ["T201"]
 # Node functions take `params` whether or not they use it — that is the
 # contract the engine calls them with.
 "fluksio/flow/nodes.py" = ["ARG001", "ARG002"]

backend/tests/api/routes/test_dashboards.py

@@ -0,0 +1,88 @@
+"""Dashboards over HTTP: saving one, which is also how the first one is made."""
+
+from fastapi.testclient import TestClient
+
+from fluksio.core.config import settings
+
+PREFIX = f"{settings.API_V1_STR}/dashboards"
+
+
+def a_dashboard(name: str) -> dict:
+    return {
+        "name": name,
+        "title": "Hall",
+        "icon": "gauge",
+        "widgets": [
+            {
+                "id": "temperature",
+                "type": "stat",
+                "title": "Temperature",
+                "layout": {"lg": {"x": 0, "y": 0, "w": 3, "h": 2}},
+                "config": {"message": "house.temperature", "dtype": "float"},
+            }
+        ],
+        "settings": {"theme": {"value": "dark", "message": "", "dtype": "str"}},
+        "version": 0,
+    }
+
+
+def test_a_save_creates_a_dashboard_that_does_not_exist_yet(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    """A first draft, not a 404: creating one *is* saving it at version 0."""
+    body = a_dashboard("put_creates")
+
+    response = client.put(
+        f"{PREFIX}/put_creates", headers=superuser_token_headers, json=body
+    )
+
+    assert response.status_code == 200, response.text
+    saved = response.json()
+    assert saved["version"] == 1 and saved["has_draft"] is True
+    # A draft alone: nothing was published, so no panel can be shown it.
+    assert (
+        client.get(f"{PREFIX}/put_creates", headers=superuser_token_headers).status_code
+        == 404
+    )
+    draft = client.get(
+        f"{PREFIX}/put_creates?draft=true", headers=superuser_token_headers
+    )
+    assert draft.json()["widgets"] == body["widgets"]
+
+
+def test_a_save_of_an_existing_dashboard_keeps_what_it_did_not_change(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    seeded = a_dashboard("put_updates")
+    first = client.put(
+        f"{PREFIX}/put_updates", headers=superuser_token_headers, json=seeded
+    ).json()
+
+    renamed = {**first, "widgets": [{**first["widgets"][0], "title": "Outside"}]}
+    second = client.put(
+        f"{PREFIX}/put_updates", headers=superuser_token_headers, json=renamed
+    )
+
+    assert second.status_code == 200, second.text
+    stored = second.json()
+    assert stored["version"] == first["version"] + 1
+    assert stored["widgets"][0]["title"] == "Outside"
+    # Everything the edit did not name is still what was first written.
+    assert stored["icon"] == seeded["icon"]
+    assert stored["settings"] == seeded["settings"]
+    assert stored["widgets"][0]["layout"] == seeded["widgets"][0]["layout"]
+    assert stored["widgets"][0]["config"] == seeded["widgets"][0]["config"]
+
+
+def test_a_save_based_on_a_version_someone_moved_past_is_refused(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    body = a_dashboard("put_conflicts")
+    client.put(f"{PREFIX}/put_conflicts", headers=superuser_token_headers, json=body)
+
+    stale = client.put(
+        f"{PREFIX}/put_conflicts", headers=superuser_token_headers, json=body
+    )
+
+    assert stale.status_code == 409
+    assert stale.json()["detail"]["current_version"] == 1

backend/tests/api/routes/test_observability.py

@@ -127,6 +127,37 @@ def test_a_flow_that_cannot_run_makes_the_summary_degraded(
     assert not any("hooky" in problem for problem in body["problems"])
 
 
+def test_a_down_node_makes_the_summary_degraded(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    """A connector that cannot reach its device is not a flow that cannot run.
+
+    It is counted on its own, so the flow keeps running and "invalid" stays
+    about validation.
+    """
+    from fluksio.flow.controller import LoadedNode
+
+    controller = client.app.state.flow_controller
+    before = controller.loaded
+    controller.loaded = {
+        "house.owm": LoadedNode(
+            id="house.owm",
+            flow="house",
+            health="down",
+            health_detail="ConnectionError: name resolution failed",
+        )
+    }
+    try:
+        body = client.get(f"{PREFIX}/summary", headers=superuser_token_headers).json()
+    finally:
+        controller.loaded = before
+
+    assert body["status"] == "degraded"
+    assert body["nodes"]["unhealthy"] == 1
+    assert any("down" in problem for problem in body["problems"])
+    assert body["flows"]["invalid"] == 0
+
+
 def test_the_history_reads_back(
     client: TestClient, superuser_token_headers: dict[str, str], db: Session
 ) -> None:

backend/tests/api/routes/test_runs.py

@@ -400,6 +400,97 @@ def test_a_repeated_submit_returns_the_run_it_already_made():
     assert again.id == "dedup-1"
 
 
+class _OneFlow:
+    """A controller that has exactly one flow and no engine behind it."""
+
+    def __init__(self, flow):
+        self.store = self
+        self._flow = flow
+
+    def read_flow(self, name, draft=False):
+        return self._flow
+
+    def head(self):
+        return ""
+
+
+class _Collect:
+    def __init__(self):
+        self.items = []
+
+    def add(self, item):
+        self.items.append(item)
+
+
+def test_a_run_records_the_inputs_it_actually_starts_from():
+    """An input left out takes its declared value, and the row says so.
+
+    `params = {}` could not tell a run that took every default from one
+    submitted with those same numbers spelled out — and an export of the
+    first had a blank cell where its `lr` should be.
+    """
+    flow = FlowDef(
+        name="study",
+        mode="batch",
+        inputs=[
+            FlowInput(spec=MessageSpec(name="lr", dtype=DType.FLOAT), initial=0.01),
+            FlowInput(spec=MessageSpec(name="epochs", dtype=DType.INT)),
+        ],
+    )
+    service = RunService(controller=_OneFlow(flow), queue=_Collect())
+    made = []
+    try:
+        defaulted = service.submit("study", {"epochs": 5})
+        made.append(defaulted.id)
+        assert defaulted.params == {"lr": 0.01, "epochs": 5}
+
+        # Spelling out the declared value is the same run, and now reads as it.
+        spelled = service.submit("study", {"lr": 0.01, "epochs": 5})
+        made.append(spelled.id)
+        assert spelled.params_digest == defaulted.params_digest
+    finally:
+        with Session(db_engine) as session:
+            for run in session.exec(select(Run).where(col(Run.id).in_(made))).all():
+                session.delete(run)
+            session.commit()
+
+
+def test_the_seed_is_recorded_the_same_way_however_it_arrived():
+    """One field an export should not have to coalesce two columns for.
+
+    `--seed 1` fills the run's own column; a flow declaring a `seed` input
+    fills the parameter. Both are the seed the run used, so both are written.
+    """
+    flow = FlowDef(
+        name="seeded",
+        mode="batch",
+        inputs=[FlowInput(spec=MessageSpec(name="seed", dtype=DType.INT), initial=42)],
+    )
+    service = RunService(controller=_OneFlow(flow), queue=_Collect())
+    made = []
+    try:
+        passed = service.submit("seeded", {}, seed=1)
+        made.append(passed.id)
+        assert (passed.seed, passed.params) == (1, {"seed": 1})
+
+        # Nothing passed: the declared value is the seed it ran with, and the
+        # run-level column says so rather than staying empty.
+        defaulted = service.submit("seeded", {})
+        made.append(defaulted.id)
+        assert (defaulted.seed, defaulted.params) == (42, {"seed": 42})
+
+        # A parameter still outranks the run's own seed, as it always has —
+        # and the column follows it rather than reporting the one that lost.
+        both = service.submit("seeded", {"seed": 7}, seed=1)
+        made.append(both.id)
+        assert (both.seed, both.params) == (7, {"seed": 7})
+    finally:
+        with Session(db_engine) as session:
+            for run in session.exec(select(Run).where(col(Run.id).in_(made))).all():
+                session.delete(run)
+            session.commit()
+
+
 def test_a_key_nobody_used_submits_normally(
     client, superuser_token_headers, monkeypatch
 ):
@@ -432,6 +523,99 @@ def test_overview_is_not_read_as_a_run_id(client, superuser_token_headers):
     assert isinstance(answer.json(), list)
 
 
+# -----------------------------------------------------------------------------
+# Deleting a run
+#
+# The route owns the four statements; what these guard is that it takes the
+# children with it and refuses a run the driver is still writing to.
+# -----------------------------------------------------------------------------
+
+
+@pytest.fixture
+def deletable_run():
+    """One finished run with a node, a number and an artifact row hanging off it."""
+    run_id = "del-1"
+    with Session(db_engine) as session:
+        session.add(
+            Run(id=run_id, flow="deleted", status="ok", created_at=datetime.now(UTC))
+        )
+        session.add(RunNode(run_id=run_id, node="deleted.a", status="ok"))
+        session.add(RunMetric(run_id=run_id, name="deleted.loss", step=0, value=1.0))
+        session.add(
+            RunArtifact(
+                run_id=run_id,
+                name="deleted.out",
+                filename="out.bin",
+                node="a",
+                digest="d" * 64,
+                size=7,
+            )
+        )
+        session.commit()
+    yield run_id
+    with Session(db_engine) as session:
+        run = session.get(Run, run_id)
+        if run is not None:
+            session.delete(run)
+            session.commit()
+
+
+def test_deleting_a_run_takes_its_children_with_it(
+    client, superuser_token_headers, deletable_run
+):
+    """No foreign key cascades here, so the route has to do it itself."""
+    answer = client.delete(
+        f"{settings.API_V1_STR}/runs/{deletable_run}", headers=superuser_token_headers
+    )
+
+    assert answer.status_code == 204
+    with Session(db_engine) as session:
+        assert session.get(Run, deletable_run) is None
+        for table in (RunNode, RunMetric, RunArtifact):
+            left = session.exec(
+                select(table).where(col(table.run_id) == deletable_run)
+            ).all()
+            assert left == [], f"{table.__name__} rows outlived the run"
+
+
+def test_deleting_a_run_that_is_not_there_is_a_404(client, superuser_token_headers):
+    answer = client.delete(
+        f"{settings.API_V1_STR}/runs/nope-1", headers=superuser_token_headers
+    )
+
+    assert answer.status_code == 404
+
+
+def test_a_running_run_is_refused_rather_than_raced(client, superuser_token_headers):
+    """The driver writes its nodes back at the end; they would have no run."""
+    run_id = "del-live"
+    with Session(db_engine) as session:
+        session.add(
+            Run(
+                id=run_id,
+                flow="deleted",
+                status="running",
+                created_at=datetime.now(UTC),
+            )
+        )
+        session.commit()
+    try:
+        answer = client.delete(
+            f"{settings.API_V1_STR}/runs/{run_id}", headers=superuser_token_headers
+        )
+
+        assert answer.status_code == 409
+        assert "Cancel it" in answer.json()["detail"]
+        with Session(db_engine) as session:
+            assert session.get(Run, run_id) is not None
+    finally:
+        with Session(db_engine) as session:
+            run = session.get(Run, run_id)
+            if run is not None:
+                session.delete(run)
+                session.commit()
+
+
 # -----------------------------------------------------------------------------
 # A cached node's curve
 #
@@ -723,10 +907,10 @@ def test_an_export_strides_each_series_and_names_its_run(
     assert all(steps == [0, 2] for steps in curves.values())
 
 
-def test_an_exported_run_row_carries_the_inputs_that_vary(
+def test_an_exported_run_row_carries_every_recorded_input(
     client, superuser_token_headers, exported
 ):
-    """The sweep axis becomes columns; what every run shares stays out of them."""
+    """Every input is a column, so the schema does not move with the selection."""
 
     def export(**extra):
         answer = client.get(
@@ -740,12 +924,15 @@ def test_an_exported_run_row_carries_the_inputs_that_vary(
     rows = _lines(export(format="jsonl"))
     assert [row["id"] for row in rows] == ["exp-1", "exp-0"]
     assert {row["param.lr"] for row in rows} == {0.1, 0.01}
-    # `epochs` is the same on both runs, so it is not what they differ by, and
-    # neither is the model inside the config — but the depth beside it is.
-    assert "param.epochs" not in rows[0]
-    assert "param.config.model" not in rows[0]
+    # `epochs` is the same on both runs and stays a column anyway: which runs
+    # were asked for is not something a downstream filter should have to know.
+    assert rows[0]["param.epochs"] == 10
+    assert rows[0]["param.config.model"] == "mlp"
     assert {row["param.config.depth"] for row in rows} == {1, 2}
-    assert "param.epochs" in _lines(export(format="jsonl", params="epochs"))[0]
+
+    narrowed = _lines(export(format="jsonl", params="epochs"))[0]
+    assert "param.epochs" in narrowed
+    assert "param.lr" not in narrowed
 
     # A number inside a record is a column of its own, however deep; a string
     # is not one of the run's numbers wherever it sits.
@@ -762,7 +949,7 @@ def test_an_exported_run_row_carries_the_inputs_that_vary(
     header = export().text.splitlines()[0]
     assert header.startswith("id,flow,status,")
     assert header.endswith(
-        "param.config.depth,param.lr,"
+        "param.config.depth,param.config.model,param.epochs,param.lr,"
         "metric.acc,metric.final_metrics.train_loss,"
         "metric.test_metrics.known.perfect"
     )

backend/tests/api/routes/test_search.py

@@ -0,0 +1,80 @@
+"""The one index the global search matches against."""
+
+from fastapi.testclient import TestClient
+
+from fluksio.core.config import settings
+
+PREFIX = f"{settings.API_V1_STR}/search"
+FLOWS = f"{settings.API_V1_STR}/flows"
+DASHBOARDS = f"{settings.API_V1_STR}/dashboards"
+SECRETS = f"{settings.API_V1_STR}/secrets"
+
+
+def test_search_requires_authentication(client: TestClient) -> None:
+    assert client.get(f"{PREFIX}/").status_code == 401
+
+
+def test_index_reaches_inside_flows_and_dashboards(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    """A node and a widget are the point: neither is on any list endpoint."""
+    client.put(
+        f"{FLOWS}/searchable",
+        headers=superuser_token_headers,
+        json={
+            "name": "searchable",
+            "title": "Searchable",
+            "nodes": [{"id": "sensor", "type": "python", "title": "Hall sensor"}],
+        },
+    )
+    client.put(
+        f"{DASHBOARDS}/hall",
+        headers=superuser_token_headers,
+        json={
+            "name": "hall",
+            "title": "Hall",
+            "widgets": [
+                {
+                    "id": "temperature",
+                    "type": "stat",
+                    "title": "Temperature",
+                    "layout": {"lg": {"x": 0, "y": 0, "w": 3, "h": 2}},
+                    "config": {"message": "hall.temperature", "dtype": "float"},
+                }
+            ],
+            "version": 0,
+        },
+    )
+
+    entries = client.get(f"{PREFIX}/", headers=superuser_token_headers).json()
+    # Keyed on the parent too: an id is only unique within the document it is
+    # in, and the other suites seed their own `sensor` and `temperature`.
+    found = {
+        (entry["category"], entry["parent"], entry["name"]): entry for entry in entries
+    }
+
+    assert found[("flow", "", "searchable")]["title"] == "Searchable"
+    assert found[("node", "searchable", "sensor")]["title"] == "Hall sensor"
+    assert found[("node", "searchable", "sensor")]["kind"] == "python"
+    assert found[("dashboard", "", "hall")]["title"] == "Hall"
+    assert found[("widget", "hall", "temperature")]["title"] == "Temperature"
+    assert found[("widget", "hall", "temperature")]["kind"] == "stat"
+
+
+def test_secrets_are_named_only_to_a_superuser(
+    client: TestClient,
+    superuser_token_headers: dict[str, str],
+    normal_user_token_headers: dict[str, str],
+) -> None:
+    client.put(
+        f"{SECRETS}/broker_password",
+        headers=superuser_token_headers,
+        json={"value": "hunter2"},
+    )
+
+    def secrets(headers: dict[str, str]) -> set[str]:
+        entries = client.get(f"{PREFIX}/", headers=headers).json()
+        return {e["name"] for e in entries if e["category"] == "secret"}
+
+    assert "broker_password" in secrets(superuser_token_headers)
+    assert secrets(normal_user_token_headers) == set()

backend/tests/flow/test_connector.py

@@ -104,6 +104,40 @@ def test_a_failing_poll_reports_down_and_keeps_going():
     assert health[-1][0] == "ok"
 
 
+def test_an_undeclared_port_keeps_failing_until_the_node_declares_it():
+    """A publication that raised is retried, not remembered as published.
+
+    The loop remembers what it published. If it remembered what it read, a
+    value the node cannot publish would be skipped on the next poll, the poll
+    would succeed, and the node would go back to reporting itself healthy with
+    its port still dark.
+    """
+
+    class Chatty(Sensor):
+        """Reads a port it never declared."""
+
+        async def poll(self) -> dict[str, Any]:
+            self.polls += 1
+            return {"reading": 21.5, "lat": 48.1}
+
+    health: list[tuple[str, str | None]] = []
+    node = Chatty(
+        provides=[MessageSpec(name="reading", dtype=DType.FLOAT)],
+        params={"poll_interval": 0.01},
+    )
+    node.assign_flow("demo", "sensor")
+    node._on_health = lambda _node, status, detail: health.append((status, detail))
+    pipeline = Pipeline(nodes=[node])
+
+    run_briefly(node)
+
+    assert pipeline.state.get("demo.reading") is None
+    assert health[-1][0] == "down"
+    assert "NodeOutputError" in (health[-1][1] or "")
+    # Still failing on the last poll, not just the first.
+    assert len([entry for entry in health if entry[0] == "down"]) > 1
+
+
 class Actuator(ConnectorNode):
     """A connector that commands something instead of reading it."""
 

backend/tests/flow/test_node_teardown.py

@@ -0,0 +1,50 @@
+"""Tearing a node down must not swallow a cancellation meant for the caller."""
+
+import asyncio
+
+import pytest
+
+from fluksio.flow.connector import ConnectorNode
+from fluksio.flow.nodes import DelayNode
+
+
+async def stubborn() -> None:
+    """A loop whose shutdown does not answer the first cancellation."""
+    try:
+        await asyncio.sleep(3600)
+    except asyncio.CancelledError:
+        await asyncio.sleep(3600)
+
+
+def test_stop_cron_lets_the_callers_cancellation_through():
+    async def scenario() -> None:
+        node = DelayNode(params={"cron": "* * * * *"})
+        node._stop_cron = asyncio.Event()
+        node._cron_task = asyncio.create_task(stubborn())
+
+        stopping = asyncio.create_task(node.stop_cron())
+        await asyncio.sleep(0.05)  # let it reach the await on the cron task
+        stopping.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await stopping
+
+        node._cron_task.cancel()
+
+    asyncio.run(scenario())
+
+
+def test_connector_stop_lets_the_callers_cancellation_through():
+    async def scenario() -> None:
+        node = ConnectorNode()
+        node._stop_event = asyncio.Event()
+        node._poll_task = asyncio.create_task(stubborn())
+
+        stopping = asyncio.create_task(node.stop())
+        await asyncio.sleep(0.05)  # let it reach the await on the poll task
+        stopping.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await stopping
+
+        node._poll_task.cancel()
+
+    asyncio.run(scenario())

backend/tests/flow/test_node_types.py

@@ -197,6 +197,47 @@ def test_a_flux_request_is_run_rather_than_written(monkeypatch):
     assert out == {"answer": {"rows": [], "range_s": 3600}}
 
 
+def test_the_configured_timeout_reaches_the_influx_client(monkeypatch):
+    """The client counts in milliseconds; the param is seconds like its peers."""
+    import influxdb_client
+
+    from fluksio.flow.nodes import InfluxDbNode
+
+    seen: dict = {}
+
+    class FakeClient:
+        def __init__(self, **kwargs):
+            seen.update(kwargs)
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *_):
+            return False
+
+        def query_api(self):
+            return self
+
+        def query(self, *_, **__):
+            return []
+
+    monkeypatch.setattr(influxdb_client, "InfluxDBClient", FakeClient)
+
+    node = InfluxDbNode(
+        provides=[MessageSpec(name="answer", dtype=DType.JSON)],
+        params={
+            "url": "http://influx",
+            "token": "t",
+            "org": "o",
+            "bucket": "b",
+            "timeout": 2.5,
+        },
+    )
+    node._run_flux({"flux": 'from(bucket: "b")'})
+
+    assert seen["timeout"] == 2500
+
+
 def test_a_falsy_return_is_a_mistake_not_silence():
     """Only None means "nothing to publish"."""
     from fluksio.flow.nodes import Node

backend/tests/flow/test_placement.py

@@ -129,7 +129,7 @@ def test_a_preferred_label_falls_back_here_and_is_still_accounted(loop):
         assert placer.local.snapshot()["cpus"]["free"] == 1
 
 
-def test_asking_for_more_than_anything_has_gets_what_there_is(loop):
+def test_asking_for_more_than_anything_has_gets_what_there_is(loop, caplog):
     """A flow written on a cluster still has to run on a laptop."""
     placer = placer_over(cpus=2)
 
@@ -138,6 +138,10 @@ def test_asking_for_more_than_anything_has_gets_what_there_is(loop):
         assert allocation.cpus == 2
         assert allocation.gpus == ()
 
+    # Cards are declared, not detected, so a machine that has one reads as
+    # having none until it is told — and the warning is where that is noticed.
+    assert "fluksio serve --gpus" in caplog.text
+
 
 def test_what_is_clamped_to_is_a_machine_that_exists(loop):
     """Each dimension taken separately can describe a machine nobody has.

backend/tests/flow/test_senders.py

@@ -89,3 +89,65 @@ def test_a_string_goes_on_the_wire_bare():
     asyncio.run(node._publish_with(client, {"plug": "ON", "level": 60}))
 
     assert client.published == [("actor/plug", "ON"), ("light/level", "60")]
+
+
+def test_the_configured_timeout_reaches_the_broker_client(monkeypatch):
+    """Without one, a dead socket makes the disconnect ack wait forever."""
+    import aiomqtt
+
+    seen: dict = {}
+
+    class FakeClient:
+        def __init__(self, **kwargs):
+            seen.update(kwargs)
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *_):
+            return False
+
+        async def publish(self, *_, **__):
+            return None
+
+    monkeypatch.setattr(aiomqtt, "Client", FakeClient)
+
+    node = MqttNode(
+        requires=[MessageSpec(name="setpoint", port="setpoint", dtype=DType.FLOAT)],
+        params={"topic": {"setpoint": "heating/setpoint"}, "timeout": 2.5},
+    )
+    asyncio.run(node._publish_once({"setpoint": 21.0}))
+
+    assert seen["timeout"] == 2.5
+
+
+def test_the_configured_backlog_reaches_the_publish_queue(monkeypatch):
+    """The depth is read when the queue is built, so it has to be per node."""
+    import aiomqtt
+
+    class FakeClient:
+        def __init__(self, **kwargs):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *_):
+            return False
+
+    monkeypatch.setattr(aiomqtt, "Client", FakeClient)
+
+    node = MqttNode(
+        requires=[MessageSpec(name="setpoint", port="setpoint", dtype=DType.FLOAT)],
+        params={"topic": {"setpoint": "heating/setpoint"}, "publish_queue_size": 8},
+    )
+    node.assign_flow("heating", "out")
+
+    async def scenario() -> int:
+        await node.start_publisher()
+        assert node._publish_queue is not None
+        size = node._publish_queue.maxsize
+        await node.stop_publisher()
+        return size
+
+    assert asyncio.run(scenario()) == 8

backend/tests/flow/test_unhealthy_node_issue.py

@@ -0,0 +1,42 @@
+"""A node that loaded but is not working shows up as an issue on its flow.
+
+Health used to go nowhere: the connector reported it, the controller stored it,
+and no screen ever asked. These cover the derivation that closes that gap.
+"""
+
+from pathlib import Path
+
+from fluksio.flow.controller import FlowController, LoadedNode
+from fluksio.flow.store import FlowStore
+
+
+def a_controller(tmp_path: Path) -> FlowController:
+    controller = FlowController(FlowStore(tmp_path / "flows"))
+    controller.loaded["house.owm"] = LoadedNode(
+        id="house.owm",
+        flow="house",
+        health="down",
+        health_detail="ConnectionError: name resolution failed",
+    )
+    return controller
+
+
+def test_a_down_node_is_an_issue_on_its_flow(tmp_path: Path) -> None:
+    controller = a_controller(tmp_path)
+
+    issues = controller.flow_issues("house")
+
+    assert [issue.code for issue in issues] == ["node_unhealthy"]
+    assert issues[0].node == "house.owm"
+    assert "name resolution failed" in issues[0].message
+    # Not advisory: the canvas has to mark the node.
+    assert not issues[0].advisory
+    assert controller.flow_issues("other") == []
+
+
+def test_the_issue_clears_when_the_node_reports_itself_well(tmp_path: Path) -> None:
+    controller = a_controller(tmp_path)
+
+    controller.loaded["house.owm"].health = "ok"
+
+    assert controller.flow_issues("house") == []

backend/tests/flow/test_workers.py

@@ -69,6 +69,24 @@ def test_a_node_returns_its_value_and_what_it_printed(pool, capsys):
     assert "seen 21" in capsys.readouterr().out
 
 
+def test_a_node_can_log_through_the_module_it_imports(pool, capsys):
+    """The SDK exports `logger`; inside a worker `fluksio` is the reporter.
+
+    Without it, `fluksio.logger.info(...)` died with AttributeError — after
+    the training it was reporting on had already succeeded.
+    """
+    result = run(
+        pool,
+        "import fluksio\n\n\n"
+        "def process(value):\n"
+        "    fluksio.logger.info('tuned %s', value)\n"
+        "    return {'out': value}\n",
+        value=7,
+    )
+    assert result == {"out": 7}
+    assert "tuned 7" in capsys.readouterr().out
+
+
 def test_a_failure_keeps_its_class_and_points_at_the_node(pool):
     with pytest.raises(Exception) as caught:
         run(pool, "def process():\n    raise ValueError('bad input')\n")
@@ -667,6 +685,19 @@ def test_retiring_workers_reaches_the_children(pool):
     assert child._generation > before
 
 
+def test_retiring_the_cards_leaves_the_other_pools_warm(pool):
+    """A library that preallocated the card only gives it back by dying."""
+    card = pool.for_env({"CUDA_VISIBLE_DEVICES": "0"})
+    threads = pool.for_env({"OMP_NUM_THREADS": "2"})
+    before = (card._generation, threads._generation)
+
+    pool.retire_gpu_children()
+
+    assert card._generation > before[0]
+    # Nothing to hand back, so nothing pays a cold start for it.
+    assert threads._generation == before[1]
+
+
 def test_cancelling_reaches_a_node_running_in_a_child(pool):
     child = pool.for_env({"OMP_NUM_THREADS": "2"})
     started = threading.Event()

backend/tests/sdk/test_build.py

@@ -6,7 +6,7 @@ from fluksio.flow.schemas import FlowDef
 from fluksio.sdk import MARKER, Flow, Port, SyncError, node, use
 
 # The functions live here rather than in each test: a node's module is what the
-# generated body imports, and `__main__` is refused for exactly that reason.
+# generated body imports, and `__main__` is refused at sync for that reason.
 
 
 @node(provides=[Port("dataset", "artifact"), Port("rows", "int")])
@@ -316,3 +316,27 @@ def test_a_negative_timeout_is_refused():
 def test_a_zero_timeout_means_no_limit():
     decorated = node(requires=["a"], timeout=0)(one_default)
     assert decorated.__fluksio__.timeout == 0
+
+
+def test_a_node_in_a_script_run_directly_is_refused_at_sync_not_at_import():
+    """A study module has to be runnable as a script for a self-check.
+
+    The refusal belongs where the body is generated: the decorator hands the
+    function back untouched, so `python study.py` declares its nodes, calls
+    them and checks itself. What cannot be done is syncing them, because
+    nothing could import `__main__`.
+    """
+
+    def probe(rows=1):
+        return {"score": float(rows)}
+
+    probe.__module__ = "__main__"
+    decorated = node(provides=[Port("score", "float")])(probe)
+    # Declared and callable, which is the whole point of running the file.
+    assert decorated(rows=2) == {"score": 2.0}
+
+    flow = Flow("mainflow", nodes=[decorated], outputs=["score"])
+    with pytest.raises(SyncError, match="run directly"):
+        flow.document()
+    with pytest.raises(SyncError, match="run directly"):
+        flow.shims()

backend/tests/test_cli.py

@@ -88,6 +88,24 @@ def test_run_arguments_are_typed_by_the_flow_they_are_for() -> None:
         _params(definition, ["--nonesuch", "1"])
 
 
+def test_a_sweeps_param_spelling_is_refused_by_name() -> None:
+    """`run --param lr=0.002` is a name this flow has not got, and says so."""
+    import pytest
+
+    from fluksio.sdk import SyncError
+    from fluksio.sdk.cli import _params
+
+    definition = {"inputs": [{"spec": {"name": "lr", "dtype": "float"}}]}
+
+    # Not a JSONDecodeError over `lr=0.002`, which is what reading the value
+    # before the name used to give.
+    with pytest.raises(SyncError, match="sweep --param"):
+        _params(definition, ["--param", "lr=0.002"])
+
+    with pytest.raises(SyncError, match="'lr' takes float"):
+        _params(definition, ["--lr", "fast"])
+
+
 def test_serve_uses_the_installation_the_directory_belongs_to(
     tmp_path: Path, monkeypatch
 ) -> None:
@@ -372,6 +390,43 @@ def test_the_metric_names_are_asked_for_rather_than_guessed() -> None:
     assert _list_names(Engine(), args) == 0
 
 
+def test_a_runs_artifact_is_listed_and_downloaded(tmp_path, monkeypatch) -> None:
+    """`save_artifact` had no counterpart: the bytes were API-only."""
+    from fluksio.cli import _parser
+    from fluksio.sdk.cli import _artifacts
+
+    row = {
+        "name": "weights",
+        "node": "fit",
+        "digest": "sha256:abc",
+        "size": 3,
+        "media_type": "application/octet-stream",
+        "filename": "weights.npz",
+    }
+
+    class Engine:
+        def run(self, run_id):
+            assert run_id == "r-1"
+            return {"id": run_id, "status": "ok", "artifacts": [row]}
+
+        def download(self, digest):
+            assert digest == "sha256:abc"
+            return b"abc"
+
+    parser = _parser()
+    monkeypatch.chdir(tmp_path)
+
+    assert _artifacts(Engine(), parser.parse_args(["artifacts", "r-1"])) == 0
+
+    # Written under the name the node saved it as, not the message's.
+    assert _artifacts(Engine(), parser.parse_args(["artifacts", "r-1", "weights"])) == 0
+    assert (tmp_path / "weights.npz").read_bytes() == b"abc"
+
+    args = parser.parse_args(["artifacts", "r-1", "weights", "-o", "here.bin"])
+    assert _artifacts(Engine(), args) == 0
+    assert (tmp_path / "here.bin").read_bytes() == b"abc"
+
+
 def test_an_engine_without_the_route_is_named_rather_than_404() -> None:
     """A client ships ahead of the engine; a flat 404 does not say so."""
     from fluksio.sdk.cli import _too_old
@@ -417,8 +472,38 @@ def test_a_study_in_a_subfolder_is_found(tmp_path) -> None:
     }
 
 
+def test_a_study_per_directory_imports_under_its_own_name(tmp_path) -> None:
+    """One `study.py` per folder is a layout people have, and it works.
+
+    Named for where each sits under the directory being synced, so nothing
+    collides and no `__init__.py` has to be added — which would break the
+    bare `from study import ...` a test beside it does.
+    """
+    import sys
+
+    from fluksio.sdk.cli import discover
+
+    for study in ("s1", "s2"):
+        (tmp_path / "dev" / study).mkdir(parents=True)
+        (tmp_path / "dev" / study / "study.py").write_text(f"VALUE = {study!r}\n")
+
+    try:
+        discover([str(tmp_path / "dev")])
+        assert sys.modules["s1.study"].VALUE == "s1"
+        assert sys.modules["s2.study"].VALUE == "s2"
+    finally:
+        for name in ("s1.study", "s2.study", "s1", "s2"):
+            sys.modules.pop(name, None)
+        sys.path[:] = [entry for entry in sys.path if entry != str(tmp_path / "dev")]
+
+
 def test_two_files_of_one_name_are_refused(tmp_path) -> None:
-    """Python keeps one module per name, and a node's body imports by it."""
+    """Python keeps one module per name, and a node's body imports by it.
+
+    Unreachable from one sync of a directory now that a file is named for
+    where it sits; this is the spelling that still gets there — two files
+    named on the command line, each rooted at its own directory.
+    """
     import pytest
 
     from fluksio.sdk import SyncError
@@ -435,6 +520,77 @@ def test_two_files_of_one_name_are_refused(tmp_path) -> None:
         _import(*_module_of(second), second)
 
 
+def test_how_long_ago_reads_like_a_duration() -> None:
+    """A failure with no time on it says nothing about whether it is current."""
+    from datetime import UTC, datetime, timedelta
+
+    from fluksio.sdk.cli import _ago
+
+    def then(**delta):
+        return (datetime.now(UTC) - timedelta(**delta)).isoformat()
+
+    assert _ago(then(seconds=5)) == "5s ago"
+    assert _ago(then(minutes=3)) == "3min ago"
+    assert _ago(then(hours=2)) == "2h ago"
+    assert _ago(then(days=3)) == "3d ago"
+    # What the engine stores is UTC whether or not the spelling says so.
+    assert _ago(datetime.now(UTC).replace(tzinfo=None).isoformat()) == "0s ago"
+    assert _ago(None) == ""
+
+
+def test_serve_says_when_a_flow_wants_a_card_nobody_declared(
+    tmp_path, monkeypatch, capsys
+) -> None:
+    """The clamp warning goes to the log; this is said while someone is reading.
+
+    Cards are declared rather than detected, so a fresh install that forgets
+    `--gpus` clamps a GPU node to zero and runs them all at once.
+    """
+    from fluksio import cli
+    from fluksio.core.config import settings
+    from fluksio.flow.schemas import FlowDef, NodeDef, Resources
+    from fluksio.flow.store import FlowStore
+
+    store = FlowStore(tmp_path / "flows")
+    store.write_flow(
+        FlowDef(
+            name="finetune",
+            mode="batch",
+            nodes=[NodeDef(id="fit", type="python", resources=Resources(gpus=1))],
+        )
+    )
+    monkeypatch.setattr(settings, "FLOWS_DIR", tmp_path / "flows")
+
+    monkeypatch.setattr(settings, "FLOW_GPUS", 0)
+    cli._mention_undeclared_cards()
+    said = capsys.readouterr().out
+    assert "finetune asks for one" in said
+    assert "--gpus" in said
+
+    # Told how many there are, it has nothing to say.
+    monkeypatch.setattr(settings, "FLOW_GPUS", 1)
+    cli._mention_undeclared_cards()
+    assert capsys.readouterr().out == ""
+
+
+def test_a_serve_limit_is_refused_as_a_flag_not_as_a_traceback(capsys) -> None:
+    """These are written into the environment before the settings are built."""
+    import pytest
+
+    from fluksio.cli import _parser
+
+    parser = _parser()
+    assert parser.parse_args(["serve", "--max-workers", "2"]).max_workers == 2
+    # A machine may genuinely have no card, so zero is a number here.
+    assert parser.parse_args(["serve", "--gpus", "0"]).gpus == 0
+    assert parser.parse_args(["serve"]).gpus is None
+
+    for flag, value in (("--max-workers", "0"), ("--gpus", "-1")):
+        with pytest.raises(SystemExit):
+            parser.parse_args(["serve", flag, value])
+    assert "at least" in capsys.readouterr().err
+
+
 def test_run_and_sweep_take_what_to_sync() -> None:
     from fluksio.cli import _parser
 
@@ -446,6 +602,98 @@ def test_run_and_sweep_take_what_to_sync() -> None:
     assert parser.parse_args(["sweep", "train"]).sync == []
 
 
+def test_the_dashboard_runs_the_engine_as_a_child_of_itself(monkeypatch) -> None:
+    """At a terminal `serve` is a dashboard; the engine is a plain serve.
+
+    Every flag is passed through, so what the child runs with is what serve
+    was asked for — and `--plain` is what stops it opening a second one.
+    """
+    import sys
+
+    from fluksio import cli
+    from fluksio.tui import child_argv
+
+    argv = child_argv(["serve", "--port", "8123", "--gpus", "1"])
+    assert argv[:3] == [sys.executable, "-m", "fluksio.cli"]
+    assert argv[3:] == ["serve", "--port", "8123", "--gpus", "1", "--plain"]
+    # Already plain: told once, not twice.
+    assert child_argv(["serve", "--plain"])[3:] == ["serve", "--plain"]
+
+    opened: list[str] = []
+    monkeypatch.setattr(
+        "fluksio.tui.run_tui", lambda args: opened.append("tui") or 0, raising=False
+    )
+    monkeypatch.setattr(sys.stdout, "isatty", lambda: True, raising=False)
+    monkeypatch.setattr(sys.stdin, "isatty", lambda: True, raising=False)
+
+    parser = cli._parser()
+    assert cli.cmd_serve(parser.parse_args(["serve"])) == 0
+    assert opened == ["tui"]
+
+    # `--plain` goes past it, which is what the child and every container does.
+    # Nothing else of serve runs here, so it fails on the data directory it is
+    # given rather than opening a dashboard.
+    opened.clear()
+    monkeypatch.setattr(
+        cli, "_data_dir", lambda *a, **k: (_ for _ in ()).throw(SystemExit(3))
+    )
+    with __import__("pytest").raises(SystemExit):
+        cli.cmd_serve(parser.parse_args(["serve", "--plain"]))
+    assert opened == []
+
+
+def test_a_serving_engine_records_itself_until_it_stops(tmp_path) -> None:
+    """A pid nobody is running is the same as no pidfile at all."""
+    import os
+
+    from fluksio.cli import read_pidfile, write_pidfile
+
+    assert read_pidfile(tmp_path) is None
+
+    written = write_pidfile(tmp_path, 8000)
+    assert read_pidfile(tmp_path) == {"pid": os.getpid(), "port": 8000}
+
+    # Killed outright: the file outlives the process it names.
+    written.write_text('{"pid": 2147483646, "port": 8000}')
+    assert read_pidfile(tmp_path) is None
+
+    written.write_text("not json")
+    assert read_pidfile(tmp_path) is None
+
+
+def test_who_holds_the_port_is_told_apart_by_the_token() -> None:
+    """Only this directory's own engine may be reported as already up.
+
+    The token is signed with this directory's secret key, so an engine that
+    accepts it is one reading this directory's database. Another
+    installation's Fluksio answers the health check and refuses it.
+    """
+    import httpx
+
+    from fluksio.cli import probe_engine
+
+    def engine(health: int, summary: int):
+        def handle(request: httpx.Request) -> httpx.Response:
+            if request.url.path.endswith("/health-check/"):
+                return httpx.Response(health)
+            return httpx.Response(summary)
+
+        return httpx.Client(transport=httpx.MockTransport(handle))
+
+    with engine(200, 200) as client:
+        assert probe_engine("http://x", "t", client) == "ours"
+    with engine(200, 401) as client:
+        assert probe_engine("http://x", "t", client) == "foreign"
+    # A directory with no credential yet cannot prove anything is its own, and
+    # `Bearer ` is not a legal header value — so it asks without one.
+    with engine(200, 401) as client:
+        assert probe_engine("http://x", "", client) == "foreign"
+    # Somebody else's dev server, or nothing listening at all.
+    with engine(404, 404) as client:
+        assert probe_engine("http://x", "t", client) == "other"
+    assert probe_engine("http://127.0.0.1:1", "t") == "other"
+
+
 def test_serve_moves_off_a_port_that_is_taken() -> None:
     """A first start should not die on somebody else's dev server."""
     import socket