Close eight open SDK tasks: the pidfile, the log, cards, names and a live curve

Each was a loose end recorded under `### SDK` in the notepad.

`serve` takes its own pidfile down on SIGTERM. uvicorn restores the handler it
found and re-raises the signal it stopped on, so the default handler ended the
process without unwinding and the `finally` never ran — which is what a stop
sends, and what left `serve.pid` behind.

`serve.log` is cut back past 5 MB by the engine rather than by the screen that
started it, so an adopted engine is bounded too. Gated on its own stdout being
an appended regular file, which is what makes the cut safe: the kernel then
puts the next write at the new end.

Cards are counted from `/dev/nvidia[0-9]*`, so `FLOW_GPUS`/`--gpus` of 0 means
"work it out" the way `FLOW_CPUS` always has. The engine counts, not the
accountant — a remote worker builds one of those from its own inventory, and
detecting there would hand it the engine host's cards. The worker counts last:
what a batch job says it was granted still wins.

`GET /runs/metrics/names` is the distinct over a selection that `--list` and
the terminal's metric picker were approximating by reading the newest run that
had measured anything, which missed a name only an older run ever wrote.

`MetricSink` announces each batch it has written (`run_metric`, carrying the
names). Not a per-point event: one covers up to 500 points or two seconds of
them, and the rows stay the record. The terminal comparison fills in as the
first readings land instead of staying blank until reopened, and the browser
refetches the run and any comparison rather than the list behind them.

`retry --group` pages the list route by `before` instead of stopping at 500.

The terminal dashboard takes the terminal's colours (`ansi-dark`), and the web
UI can re-pair from Settings without disconnecting first.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PRQ9bmTvCbqCwXo9mxZzzV

backend/fluksio/cli.py

@@ -22,6 +22,7 @@ import os
 import secrets
 import socket
 import sys
+import time
 from collections.abc import Callable
 from pathlib import Path
 from typing import Any
@@ -83,17 +84,19 @@ def _mention_other_instance(data_dir: Path) -> None:
 
 
 def _mention_undeclared_cards() -> None:
-    """Say when a stored flow asks for a card this engine does not have.
+    """Say when a stored flow asks for a card this engine cannot find.
 
-    Cards are declared rather than detected, so an engine told nothing has
-    none — and a node asking for one is clamped to zero and runs beside every
-    other, which on a GPU is the deadlock the declaration exists to prevent.
-    The placer says so once it happens, into the log; this says it while
-    somebody is still reading the terminal.
+    NVIDIA's device nodes are counted; anything else has to be declared, so an
+    engine that finds none and is told none has none — and a node asking for
+    one is clamped to zero and runs beside every other, which on a GPU is the
+    deadlock the declaration exists to prevent. The placer says so once it
+    happens, into the log; this says it while somebody is still reading the
+    terminal.
     """
     from fluksio.core.config import settings
+    from fluksio.flow.resources import machine_gpus
 
-    if settings.FLOW_GPUS:
+    if settings.FLOW_GPUS or machine_gpus():
         return
     from fluksio.flow.runs import required_resources
     from fluksio.flow.store import FlowStore
@@ -111,8 +114,8 @@ def _mention_undeclared_cards() -> None:
     if not asking:
         return
     named = ", ".join(sorted(asking)[:3]) + (" …" if len(asking) > 3 else "")
-    _say(f"  Cards    0 declared, but {named} asks for one.")
-    _say("           Nothing detects them: `--gpus N` says how many are here.")
+    _say(f"  Cards    none found, but {named} asks for one.")
+    _say("           `--gpus N` says how many when /dev/nvidia* does not.")
 
 
 def _warn_if_networked(path: Path) -> None:
@@ -318,6 +321,84 @@ def read_pidfile(data_dir: Path) -> dict[str, int] | None:
     return {"pid": pid, "port": port}
 
 
+#: How large `serve.log` is allowed to get before it is cut back to nothing.
+#: The dashboard starts the engine with that file as its stdout and reads it
+#: back as the Logs tab, so an engine left running would otherwise fill a disk
+#: with nobody watching.
+LOG_KEEP_BYTES = 5 * 1024 * 1024
+
+#: How often the size is looked at. Cheap — one fstat — and nowhere near a
+#: path anything else takes.
+LOG_CHECK_S = 30.0
+
+
+def _appended_log(fd: int = 1) -> bool:
+    """Whether this descriptor is a file the engine may cut.
+
+    Only a regular file opened for appending: the kernel then puts every write
+    at the new end, where a plain `>` redirect would keep the offset it had and
+    leave a hole the size of what was dropped.
+    """
+    try:
+        import fcntl
+        import stat
+
+        if not stat.S_ISREG(os.fstat(fd).st_mode):
+            return False
+        return bool(fcntl.fcntl(fd, fcntl.F_GETFL) & os.O_APPEND)
+    except (ImportError, OSError, AttributeError):
+        return False
+
+
+def _trim_log(fd: int = 1, limit: int = LOG_KEEP_BYTES) -> bool:
+    """Empty the file behind stdout once it passes `limit`.
+
+    ponytail: a cut, not a rotation — copy the tail aside here when somebody
+    wants yesterday's log. The dashboard's reader already survives it: it
+    notices the file has shrunk under it and reads from the top again.
+    """
+    try:
+        if os.fstat(fd).st_size <= limit:
+            return False
+        os.ftruncate(fd, 0)
+    except OSError:
+        return False
+    return True
+
+
+def _watch_log(fd: int = 1, limit: int = LOG_KEEP_BYTES) -> None:
+    """Keep the engine's own output bounded, whoever started it.
+
+    The dashboard used to cut the file when *it* spawned an engine, which left
+    an adopted one — or one whose screen was closed — writing without a bound.
+    """
+    import threading
+
+    def loop() -> None:
+        while True:
+            time.sleep(LOG_CHECK_S)
+            _trim_log(fd, limit)
+
+    threading.Thread(target=loop, daemon=True).start()
+
+
+def _hold_pidfile(pidfile: Path, serve: Callable[[], None]) -> None:
+    """Run the engine, and take the pidfile down however it ends.
+
+    uvicorn restores the SIGTERM handler it found and re-raises the signal it
+    stopped on, so the default handler would end the process without unwinding
+    — and the file would outlive it. This leaves through the `finally` instead.
+    Ctrl-C already did: the restored handler there raises `KeyboardInterrupt`.
+    """
+    import signal
+
+    signal.signal(signal.SIGTERM, lambda signum, _frame: sys.exit(128 + signum))
+    try:
+        serve()
+    finally:
+        pidfile.unlink(missing_ok=True)
+
+
 def _token_for(data_dir: Path) -> str:
     """The credential this directory's last engine wrote, if it wrote one."""
     from fluksio.sdk.client import config_path
@@ -516,16 +597,19 @@ def cmd_serve(args: argparse.Namespace) -> int:
     # One process: it holds the flow engine, and a second worker would be a
     # second engine — duplicated subscriptions, cron ticks and webhooks.
     pidfile = write_pidfile(data_dir, port)
-    try:
-        uvicorn.run(
+    if _appended_log():
+        # Started by the dashboard, with `serve.log` as this process's stdout.
+        _watch_log()
+    _hold_pidfile(
+        pidfile,
+        lambda: uvicorn.run(
             app,
             host=args.host,
             port=port,
             log_level=args.log_level,
             log_config=_log_config(args.log_level),
-        )
-    finally:
-        pidfile.unlink(missing_ok=True)
+        ),
+    )
     return 0
 
 
@@ -641,7 +725,7 @@ def _parser() -> argparse.ArgumentParser:
         type=_at_least(0),
         default=None,
         metavar="N",
-        help="GPUs on this machine a node may be given (default 0, FLOW_GPUS)",
+        help="GPUs a node may be given (default: /dev/nvidia* counted, FLOW_GPUS)",
     )
     serve.set_defaults(func=cmd_serve)