Close eight open SDK tasks: the pidfile, the log, cards, names and a live curve

Each was a loose end recorded under `### SDK` in the notepad.

`serve` takes its own pidfile down on SIGTERM. uvicorn restores the handler it
found and re-raises the signal it stopped on, so the default handler ended the
process without unwinding and the `finally` never ran — which is what a stop
sends, and what left `serve.pid` behind.

`serve.log` is cut back past 5 MB by the engine rather than by the screen that
started it, so an adopted engine is bounded too. Gated on its own stdout being
an appended regular file, which is what makes the cut safe: the kernel then
puts the next write at the new end.

Cards are counted from `/dev/nvidia[0-9]*`, so `FLOW_GPUS`/`--gpus` of 0 means
"work it out" the way `FLOW_CPUS` always has. The engine counts, not the
accountant — a remote worker builds one of those from its own inventory, and
detecting there would hand it the engine host's cards. The worker counts last:
what a batch job says it was granted still wins.

`GET /runs/metrics/names` is the distinct over a selection that `--list` and
the terminal's metric picker were approximating by reading the newest run that
had measured anything, which missed a name only an older run ever wrote.

`MetricSink` announces each batch it has written (`run_metric`, carrying the
names). Not a per-point event: one covers up to 500 points or two seconds of
them, and the rows stay the record. The terminal comparison fills in as the
first readings land instead of staying blank until reopened, and the browser
refetches the run and any comparison rather than the list behind them.

`retry --group` pages the list route by `before` instead of stopping at 500.

The terminal dashboard takes the terminal's colours (`ansi-dark`), and the web
UI can re-pair from Settings without disconnecting first.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PRQ9bmTvCbqCwXo9mxZzzV

backend/tests/api/routes/test_runs.py

@@ -886,6 +886,64 @@ def plotted():
         session.commit()
 
 
+def test_the_metric_names_of_a_selection_are_exact(
+    client, superuser_token_headers, plotted
+):
+    """Names are flow-qualified, so this is what says which spellings exist.
+
+    Read over the whole selection rather than off the newest run that measured
+    anything, which is what missed a name only an older run ever wrote — here,
+    the older run's `study.grad` against the newer one's two.
+    """
+    older = new_run_id()
+    with Session(db_engine) as session:
+        session.add(
+            Run(
+                id=older,
+                flow="study",
+                status="ok",
+                created_at=datetime.now(UTC) - timedelta(hours=1),
+            )
+        )
+        session.add(_metric(older, "study.grad", 0, 0.1, 50.0))
+        session.commit()
+
+    try:
+        both = client.get(
+            f"{settings.API_V1_STR}/runs/metrics/names",
+            params={"ids": f"{plotted},{older}"},
+            headers=superuser_token_headers,
+        )
+        assert both.status_code == 200
+        assert both.json() == ["study.epoch", "study.grad", "study.loss"]
+
+        # The same question by filter rather than by name.
+        assert client.get(
+            f"{settings.API_V1_STR}/runs/metrics/names",
+            params={"flow": "study"},
+            headers=superuser_token_headers,
+        ).json() == ["study.epoch", "study.grad", "study.loss"]
+
+        # A selection that recorded nothing has nothing to offer, which is not
+        # an error: a run opened before its first reading is the ordinary case.
+        assert (
+            client.get(
+                f"{settings.API_V1_STR}/runs/metrics/names",
+                params={"ids": "no-such-run"},
+                headers=superuser_token_headers,
+            ).json()
+            == []
+        )
+    finally:
+        with Session(db_engine) as session:
+            for row in session.exec(
+                select(RunMetric).where(col(RunMetric.run_id) == older)
+            ).all():
+                session.delete(row)
+            session.delete(session.get(Run, older))
+            session.commit()
+
+
 def test_a_comparison_is_plotted_against_the_step_by_default(
     client, superuser_token_headers, plotted
 ):

backend/tests/flow/test_resources.py

@@ -18,10 +18,32 @@ from fluksio.flow.resources import (
     ResourceAccountant,
     derive_env,
     fair_share_env,
+    machine_gpus,
 )
 from fluksio.flow.schemas import NodeDef, Resources
 
 
+def test_the_cards_are_counted_from_the_devices_the_driver_makes(monkeypatch):
+    """A box with cards and an engine told nothing used to have none of them.
+
+    NVIDIA's own device nodes, counted — no vendor tool, so the one dependency
+    does not become two. The control files beside them are not cards.
+    """
+    from fluksio.flow import resources
+
+    monkeypatch.setattr(
+        resources.glob,
+        "glob",
+        lambda pattern: (
+            ["/dev/nvidia0", "/dev/nvidia1"] if pattern == "/dev/nvidia[0-9]*" else []
+        ),
+    )
+    assert machine_gpus() == 2
+
+    monkeypatch.setattr(resources.glob, "glob", lambda pattern: [])
+    assert machine_gpus() == 0
+
+
 def test_what_is_free_is_what_was_handed_out():
     accountant = ResourceAccountant(cpus=4, gpus=0)
     held = accountant.try_take(cpus=3, gpus=0)

backend/tests/flow/test_runs.py

@@ -296,6 +296,30 @@ def test_emissions_reach_the_run_as_a_series_with_a_step_each():
     assert sink._steps == {"study.loss": 1}
 
 
+def test_a_written_batch_says_it_is_there():
+    """What a screen watching a live run waits on.
+
+    The rows are still the record — this is a nudge carrying names, one per
+    batch rather than one per point, so nothing lands on the hot path.
+    """
+    said = []
+    sink = MetricSink("run-1", batch=1, on_flush=said.append)
+
+    sink.handle("study.train", {"study.loss": 1.0, "study.tag": "ignored"})
+    assert said == [["study.loss"]]
+
+    # Held back until the batch is due, then announced once for all of it.
+    quiet = MetricSink("run-2", batch=10, interval=3600, on_flush=said.append)
+    quiet.handle("study.train", {"study.loss": 1.0, "study.acc": 0.5})
+    assert said == [["study.loss"]]
+    quiet.flush()
+    assert said[-1] == ["study.acc", "study.loss"]
+
+    # Nothing to write is nothing to say.
+    quiet.flush()
+    assert len(said) == 2
+
+
 # -----------------------------------------------------------------------------
 # What a caller may ask for
 # -----------------------------------------------------------------------------

backend/tests/test_cli.py

@@ -414,8 +414,13 @@ def test_an_export_is_parsed_with_its_selection() -> None:
     assert runs.format == "jsonl"
 
 
-def test_the_metric_names_are_asked_for_rather_than_guessed() -> None:
-    """A name is flow-qualified, so `--list` is what says what would match."""
+def test_the_metric_names_are_asked_for_rather_than_guessed(capsys) -> None:
+    """A name is flow-qualified, so `--list` is what says what would match.
+
+    The engine answers it over the whole selection, so a name only an older run
+    ever recorded is listed — which reading the newest run that measured
+    anything could not do.
+    """
     from fluksio.cli import _parser
     from fluksio.sdk.cli import _list_names
 
@@ -423,20 +428,19 @@ def test_the_metric_names_are_asked_for_rather_than_guessed() -> None:
     assert parser.parse_args(["export", "metrics"]).list_names is False
     assert parser.parse_args(["export", "metrics", "--list"]).list_names is True
 
-    class Engine:
-        def runs(self, flow="", limit=0, **filters):
-            assert filters == {"status": "ok"}
-            return [{"id": "r-empty"}, {"id": "r-1"}]
+    asked = {}
 
-        def metrics(self, run_id, name="", stride=1):
-            # The newest run failed before it measured anything; the next one
-            # carries the vocabulary.
-            return [] if run_id == "r-empty" else [{"name": "train.train_loss"}]
+    class Engine:
+        def metric_names(self, ids=(), flow="", **filters):
+            asked.update({"ids": list(ids), "flow": flow, **filters})
+            return ["train.train_loss", "train.val_loss"]
 
     args = parser.parse_args(
         ["export", "metrics", "--flow", "train", "--status", "ok", "--list"]
     )
     assert _list_names(Engine(), args) == 0
+    assert asked == {"ids": [], "flow": "train", "status": "ok"}
+    assert capsys.readouterr().out.split() == ["train.train_loss", "train.val_loss"]
 
 
 def test_a_runs_artifact_is_listed_and_downloaded(tmp_path, monkeypatch) -> None:
@@ -592,14 +596,19 @@ def test_serve_says_when_a_flow_wants_a_card_nobody_declared(
 ) -> None:
     """The clamp warning goes to the log; this is said while someone is reading.
 
-    Cards are declared rather than detected, so a fresh install that forgets
-    `--gpus` clamps a GPU node to zero and runs them all at once.
+    A card behind a driver whose device nodes are not NVIDIA's is not counted,
+    so an install that forgets `--gpus` there clamps a GPU node to zero and
+    runs them all at once. Detection is stubbed either way: whether this test
+    says anything must not depend on the machine running it.
     """
     from fluksio import cli
     from fluksio.core.config import settings
+    from fluksio.flow import resources
     from fluksio.flow.schemas import FlowDef, NodeDef, Resources
     from fluksio.flow.store import FlowStore
 
+    monkeypatch.setattr(resources, "machine_gpus", lambda: 0)
+
     store = FlowStore(tmp_path / "flows")
     store.write_flow(
         FlowDef(
@@ -621,6 +630,13 @@ def test_serve_says_when_a_flow_wants_a_card_nobody_declared(
     cli._mention_undeclared_cards()
     assert capsys.readouterr().out == ""
 
+    # Nor when it found them itself, which is the ordinary case on a box with
+    # NVIDIA's driver: nobody has to say what the device nodes already do.
+    monkeypatch.setattr(settings, "FLOW_GPUS", 0)
+    monkeypatch.setattr(resources, "machine_gpus", lambda: 1)
+    cli._mention_undeclared_cards()
+    assert capsys.readouterr().out == ""
+
 
 def test_a_serve_limit_is_refused_as_a_flag_not_as_a_traceback(capsys) -> None:
     """These are written into the environment before the settings are built."""
@@ -745,6 +761,92 @@ def test_a_serving_engine_records_itself_until_it_stops(tmp_path) -> None:
     assert read_pidfile(tmp_path) is None
 
 
+def test_the_pidfile_goes_with_the_process_however_it_ends(tmp_path) -> None:
+    """A SIGTERM used to leave `serve.pid` behind, which is what a stop sends.
+
+    uvicorn puts back the handler it found and re-raises the signal it stopped
+    on; the default handler ends the process without unwinding, so the `finally`
+    that removes the file never ran. A stale file costs the dashboard a probe
+    and, if a pid is ever recycled, points at a stranger.
+    """
+    import signal
+
+    from fluksio.cli import _hold_pidfile, write_pidfile
+
+    was = signal.getsignal(signal.SIGTERM)
+    try:
+        written = write_pidfile(tmp_path, 8000)
+        assert written.exists()
+        with pytest.raises(SystemExit):
+            _hold_pidfile(written, lambda: signal.raise_signal(signal.SIGTERM))
+        assert not written.exists()
+
+        # And the ordinary way out, which always worked.
+        written = write_pidfile(tmp_path, 8000)
+        _hold_pidfile(written, lambda: None)
+        assert not written.exists()
+    finally:
+        signal.signal(signal.SIGTERM, was)
+
+
+def test_the_engine_cuts_its_own_log_when_it_grows(tmp_path) -> None:
+    """The dashboard cut it only when *it* started the engine, so an adopted
+    one — or one whose screen was closed — wrote without a bound.
+
+    Only an appended file is cut: the kernel then puts the next write at the
+    new end, where a `>` redirect would keep its offset and leave a hole.
+    """
+    from fluksio.cli import _appended_log, _trim_log
+
+    path = tmp_path / "serve.log"
+    with path.open("ab", buffering=0) as handle:
+        fd = handle.fileno()
+        handle.write(b"0123456789")
+        assert _appended_log(fd) is True
+        assert _trim_log(fd, limit=100) is False
+        assert path.stat().st_size == 10
+        assert _trim_log(fd, limit=5) is True
+        assert path.stat().st_size == 0
+        # It keeps writing into the same descriptor afterwards.
+        handle.write(b"after")
+        assert path.read_bytes() == b"after"
+
+    with path.open("wb", buffering=0) as handle:
+        # Not appended, so not this engine's to cut.
+        handle.write(b"0123456789")
+        assert _appended_log(handle.fileno()) is False
+
+
+def test_a_sweep_larger_than_a_page_is_retried_whole() -> None:
+    """`retry --group` read one page of the list route and stopped there.
+
+    Pages come newest first and `before` is the cursor, so the next page is
+    taken from the last row's own timestamp rather than from an offset that
+    shifts under a run submitted meanwhile.
+    """
+    from fluksio.sdk.cli import PAGE, _group_runs
+
+    asked = []
+
+    class Engine:
+        def runs(self, flow="", limit=0, **filters):
+            asked.append(filters)
+            page = 0 if "before" not in filters else 1
+            if page:
+                return [{"id": "last", "created_at": "2026-09-02T00:00:00Z"}]
+            return [
+                {"id": f"r{index}", "created_at": f"2026-09-02T00:00:{index:02d}Z"}
+                for index in range(PAGE)
+            ]
+
+    rows = list(_group_runs(Engine(), "sweep-1"))
+
+    assert len(rows) == PAGE + 1
+    assert asked[0] == {"group": "sweep-1"}
+    # The second page starts where the first ended.
+    assert asked[1]["before"] == rows[PAGE - 1]["created_at"]
+
+
 def test_who_holds_the_port_is_told_apart_by_the_token() -> None:
     """Only this directory's own engine may be reported as already up.