Merge branch 'main' of git.stroblme.de:Fluksio/app

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01C5H4uLCCpsbipL1R7WKCee

backend/tests/api/routes/test_dashboards.py

@@ -0,0 +1,88 @@
+"""Dashboards over HTTP: saving one, which is also how the first one is made."""
+
+from fastapi.testclient import TestClient
+
+from fluksio.core.config import settings
+
+PREFIX = f"{settings.API_V1_STR}/dashboards"
+
+
+def a_dashboard(name: str) -> dict:
+    return {
+        "name": name,
+        "title": "Hall",
+        "icon": "gauge",
+        "widgets": [
+            {
+                "id": "temperature",
+                "type": "stat",
+                "title": "Temperature",
+                "layout": {"lg": {"x": 0, "y": 0, "w": 3, "h": 2}},
+                "config": {"message": "house.temperature", "dtype": "float"},
+            }
+        ],
+        "settings": {"theme": {"value": "dark", "message": "", "dtype": "str"}},
+        "version": 0,
+    }
+
+
+def test_a_save_creates_a_dashboard_that_does_not_exist_yet(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    """A first draft, not a 404: creating one *is* saving it at version 0."""
+    body = a_dashboard("put_creates")
+
+    response = client.put(
+        f"{PREFIX}/put_creates", headers=superuser_token_headers, json=body
+    )
+
+    assert response.status_code == 200, response.text
+    saved = response.json()
+    assert saved["version"] == 1 and saved["has_draft"] is True
+    # A draft alone: nothing was published, so no panel can be shown it.
+    assert (
+        client.get(f"{PREFIX}/put_creates", headers=superuser_token_headers).status_code
+        == 404
+    )
+    draft = client.get(
+        f"{PREFIX}/put_creates?draft=true", headers=superuser_token_headers
+    )
+    assert draft.json()["widgets"] == body["widgets"]
+
+
+def test_a_save_of_an_existing_dashboard_keeps_what_it_did_not_change(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    seeded = a_dashboard("put_updates")
+    first = client.put(
+        f"{PREFIX}/put_updates", headers=superuser_token_headers, json=seeded
+    ).json()
+
+    renamed = {**first, "widgets": [{**first["widgets"][0], "title": "Outside"}]}
+    second = client.put(
+        f"{PREFIX}/put_updates", headers=superuser_token_headers, json=renamed
+    )
+
+    assert second.status_code == 200, second.text
+    stored = second.json()
+    assert stored["version"] == first["version"] + 1
+    assert stored["widgets"][0]["title"] == "Outside"
+    # Everything the edit did not name is still what was first written.
+    assert stored["icon"] == seeded["icon"]
+    assert stored["settings"] == seeded["settings"]
+    assert stored["widgets"][0]["layout"] == seeded["widgets"][0]["layout"]
+    assert stored["widgets"][0]["config"] == seeded["widgets"][0]["config"]
+
+
+def test_a_save_based_on_a_version_someone_moved_past_is_refused(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    body = a_dashboard("put_conflicts")
+    client.put(f"{PREFIX}/put_conflicts", headers=superuser_token_headers, json=body)
+
+    stale = client.put(
+        f"{PREFIX}/put_conflicts", headers=superuser_token_headers, json=body
+    )
+
+    assert stale.status_code == 409
+    assert stale.json()["detail"]["current_version"] == 1

backend/tests/api/routes/test_observability.py

@@ -127,6 +127,37 @@ def test_a_flow_that_cannot_run_makes_the_summary_degraded(
     assert not any("hooky" in problem for problem in body["problems"])
 
 
+def test_a_down_node_makes_the_summary_degraded(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    """A connector that cannot reach its device is not a flow that cannot run.
+
+    It is counted on its own, so the flow keeps running and "invalid" stays
+    about validation.
+    """
+    from fluksio.flow.controller import LoadedNode
+
+    controller = client.app.state.flow_controller
+    before = controller.loaded
+    controller.loaded = {
+        "house.owm": LoadedNode(
+            id="house.owm",
+            flow="house",
+            health="down",
+            health_detail="ConnectionError: name resolution failed",
+        )
+    }
+    try:
+        body = client.get(f"{PREFIX}/summary", headers=superuser_token_headers).json()
+    finally:
+        controller.loaded = before
+
+    assert body["status"] == "degraded"
+    assert body["nodes"]["unhealthy"] == 1
+    assert any("down" in problem for problem in body["problems"])
+    assert body["flows"]["invalid"] == 0
+
+
 def test_the_history_reads_back(
     client: TestClient, superuser_token_headers: dict[str, str], db: Session
 ) -> None:

backend/tests/api/routes/test_runs.py

@@ -400,6 +400,97 @@ def test_a_repeated_submit_returns_the_run_it_already_made():
     assert again.id == "dedup-1"
 
 
+class _OneFlow:
+    """A controller that has exactly one flow and no engine behind it."""
+
+    def __init__(self, flow):
+        self.store = self
+        self._flow = flow
+
+    def read_flow(self, name, draft=False):
+        return self._flow
+
+    def head(self):
+        return ""
+
+
+class _Collect:
+    def __init__(self):
+        self.items = []
+
+    def add(self, item):
+        self.items.append(item)
+
+
+def test_a_run_records_the_inputs_it_actually_starts_from():
+    """An input left out takes its declared value, and the row says so.
+
+    `params = {}` could not tell a run that took every default from one
+    submitted with those same numbers spelled out — and an export of the
+    first had a blank cell where its `lr` should be.
+    """
+    flow = FlowDef(
+        name="study",
+        mode="batch",
+        inputs=[
+            FlowInput(spec=MessageSpec(name="lr", dtype=DType.FLOAT), initial=0.01),
+            FlowInput(spec=MessageSpec(name="epochs", dtype=DType.INT)),
+        ],
+    )
+    service = RunService(controller=_OneFlow(flow), queue=_Collect())
+    made = []
+    try:
+        defaulted = service.submit("study", {"epochs": 5})
+        made.append(defaulted.id)
+        assert defaulted.params == {"lr": 0.01, "epochs": 5}
+
+        # Spelling out the declared value is the same run, and now reads as it.
+        spelled = service.submit("study", {"lr": 0.01, "epochs": 5})
+        made.append(spelled.id)
+        assert spelled.params_digest == defaulted.params_digest
+    finally:
+        with Session(db_engine) as session:
+            for run in session.exec(select(Run).where(col(Run.id).in_(made))).all():
+                session.delete(run)
+            session.commit()
+
+
+def test_the_seed_is_recorded_the_same_way_however_it_arrived():
+    """One field an export should not have to coalesce two columns for.
+
+    `--seed 1` fills the run's own column; a flow declaring a `seed` input
+    fills the parameter. Both are the seed the run used, so both are written.
+    """
+    flow = FlowDef(
+        name="seeded",
+        mode="batch",
+        inputs=[FlowInput(spec=MessageSpec(name="seed", dtype=DType.INT), initial=42)],
+    )
+    service = RunService(controller=_OneFlow(flow), queue=_Collect())
+    made = []
+    try:
+        passed = service.submit("seeded", {}, seed=1)
+        made.append(passed.id)
+        assert (passed.seed, passed.params) == (1, {"seed": 1})
+
+        # Nothing passed: the declared value is the seed it ran with, and the
+        # run-level column says so rather than staying empty.
+        defaulted = service.submit("seeded", {})
+        made.append(defaulted.id)
+        assert (defaulted.seed, defaulted.params) == (42, {"seed": 42})
+
+        # A parameter still outranks the run's own seed, as it always has —
+        # and the column follows it rather than reporting the one that lost.
+        both = service.submit("seeded", {"seed": 7}, seed=1)
+        made.append(both.id)
+        assert (both.seed, both.params) == (7, {"seed": 7})
+    finally:
+        with Session(db_engine) as session:
+            for run in session.exec(select(Run).where(col(Run.id).in_(made))).all():
+                session.delete(run)
+            session.commit()
+
+
 def test_a_key_nobody_used_submits_normally(
     client, superuser_token_headers, monkeypatch
 ):
@@ -432,6 +523,99 @@ def test_overview_is_not_read_as_a_run_id(client, superuser_token_headers):
     assert isinstance(answer.json(), list)
 
 
+# -----------------------------------------------------------------------------
+# Deleting a run
+#
+# The route owns the four statements; what these guard is that it takes the
+# children with it and refuses a run the driver is still writing to.
+# -----------------------------------------------------------------------------
+
+
+@pytest.fixture
+def deletable_run():
+    """One finished run with a node, a number and an artifact row hanging off it."""
+    run_id = "del-1"
+    with Session(db_engine) as session:
+        session.add(
+            Run(id=run_id, flow="deleted", status="ok", created_at=datetime.now(UTC))
+        )
+        session.add(RunNode(run_id=run_id, node="deleted.a", status="ok"))
+        session.add(RunMetric(run_id=run_id, name="deleted.loss", step=0, value=1.0))
+        session.add(
+            RunArtifact(
+                run_id=run_id,
+                name="deleted.out",
+                filename="out.bin",
+                node="a",
+                digest="d" * 64,
+                size=7,
+            )
+        )
+        session.commit()
+    yield run_id
+    with Session(db_engine) as session:
+        run = session.get(Run, run_id)
+        if run is not None:
+            session.delete(run)
+            session.commit()
+
+
+def test_deleting_a_run_takes_its_children_with_it(
+    client, superuser_token_headers, deletable_run
+):
+    """No foreign key cascades here, so the route has to do it itself."""
+    answer = client.delete(
+        f"{settings.API_V1_STR}/runs/{deletable_run}", headers=superuser_token_headers
+    )
+
+    assert answer.status_code == 204
+    with Session(db_engine) as session:
+        assert session.get(Run, deletable_run) is None
+        for table in (RunNode, RunMetric, RunArtifact):
+            left = session.exec(
+                select(table).where(col(table.run_id) == deletable_run)
+            ).all()
+            assert left == [], f"{table.__name__} rows outlived the run"
+
+
+def test_deleting_a_run_that_is_not_there_is_a_404(client, superuser_token_headers):
+    answer = client.delete(
+        f"{settings.API_V1_STR}/runs/nope-1", headers=superuser_token_headers
+    )
+
+    assert answer.status_code == 404
+
+
+def test_a_running_run_is_refused_rather_than_raced(client, superuser_token_headers):
+    """The driver writes its nodes back at the end; they would have no run."""
+    run_id = "del-live"
+    with Session(db_engine) as session:
+        session.add(
+            Run(
+                id=run_id,
+                flow="deleted",
+                status="running",
+                created_at=datetime.now(UTC),
+            )
+        )
+        session.commit()
+    try:
+        answer = client.delete(
+            f"{settings.API_V1_STR}/runs/{run_id}", headers=superuser_token_headers
+        )
+
+        assert answer.status_code == 409
+        assert "Cancel it" in answer.json()["detail"]
+        with Session(db_engine) as session:
+            assert session.get(Run, run_id) is not None
+    finally:
+        with Session(db_engine) as session:
+            run = session.get(Run, run_id)
+            if run is not None:
+                session.delete(run)
+                session.commit()
+
+
 # -----------------------------------------------------------------------------
 # A cached node's curve
 #
@@ -723,10 +907,10 @@ def test_an_export_strides_each_series_and_names_its_run(
     assert all(steps == [0, 2] for steps in curves.values())
 
 
-def test_an_exported_run_row_carries_the_inputs_that_vary(
+def test_an_exported_run_row_carries_every_recorded_input(
     client, superuser_token_headers, exported
 ):
-    """The sweep axis becomes columns; what every run shares stays out of them."""
+    """Every input is a column, so the schema does not move with the selection."""
 
     def export(**extra):
         answer = client.get(
@@ -740,12 +924,15 @@ def test_an_exported_run_row_carries_the_inputs_that_vary(
     rows = _lines(export(format="jsonl"))
     assert [row["id"] for row in rows] == ["exp-1", "exp-0"]
     assert {row["param.lr"] for row in rows} == {0.1, 0.01}
-    # `epochs` is the same on both runs, so it is not what they differ by, and
-    # neither is the model inside the config — but the depth beside it is.
-    assert "param.epochs" not in rows[0]
-    assert "param.config.model" not in rows[0]
+    # `epochs` is the same on both runs and stays a column anyway: which runs
+    # were asked for is not something a downstream filter should have to know.
+    assert rows[0]["param.epochs"] == 10
+    assert rows[0]["param.config.model"] == "mlp"
     assert {row["param.config.depth"] for row in rows} == {1, 2}
-    assert "param.epochs" in _lines(export(format="jsonl", params="epochs"))[0]
+
+    narrowed = _lines(export(format="jsonl", params="epochs"))[0]
+    assert "param.epochs" in narrowed
+    assert "param.lr" not in narrowed
 
     # A number inside a record is a column of its own, however deep; a string
     # is not one of the run's numbers wherever it sits.
@@ -762,7 +949,7 @@ def test_an_exported_run_row_carries_the_inputs_that_vary(
     header = export().text.splitlines()[0]
     assert header.startswith("id,flow,status,")
     assert header.endswith(
-        "param.config.depth,param.lr,"
+        "param.config.depth,param.config.model,param.epochs,param.lr,"
         "metric.acc,metric.final_metrics.train_loss,"
         "metric.test_metrics.known.perfect"
     )

backend/tests/api/routes/test_search.py

@@ -0,0 +1,80 @@
+"""The one index the global search matches against."""
+
+from fastapi.testclient import TestClient
+
+from fluksio.core.config import settings
+
+PREFIX = f"{settings.API_V1_STR}/search"
+FLOWS = f"{settings.API_V1_STR}/flows"
+DASHBOARDS = f"{settings.API_V1_STR}/dashboards"
+SECRETS = f"{settings.API_V1_STR}/secrets"
+
+
+def test_search_requires_authentication(client: TestClient) -> None:
+    assert client.get(f"{PREFIX}/").status_code == 401
+
+
+def test_index_reaches_inside_flows_and_dashboards(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    """A node and a widget are the point: neither is on any list endpoint."""
+    client.put(
+        f"{FLOWS}/searchable",
+        headers=superuser_token_headers,
+        json={
+            "name": "searchable",
+            "title": "Searchable",
+            "nodes": [{"id": "sensor", "type": "python", "title": "Hall sensor"}],
+        },
+    )
+    client.put(
+        f"{DASHBOARDS}/hall",
+        headers=superuser_token_headers,
+        json={
+            "name": "hall",
+            "title": "Hall",
+            "widgets": [
+                {
+                    "id": "temperature",
+                    "type": "stat",
+                    "title": "Temperature",
+                    "layout": {"lg": {"x": 0, "y": 0, "w": 3, "h": 2}},
+                    "config": {"message": "hall.temperature", "dtype": "float"},
+                }
+            ],
+            "version": 0,
+        },
+    )
+
+    entries = client.get(f"{PREFIX}/", headers=superuser_token_headers).json()
+    # Keyed on the parent too: an id is only unique within the document it is
+    # in, and the other suites seed their own `sensor` and `temperature`.
+    found = {
+        (entry["category"], entry["parent"], entry["name"]): entry for entry in entries
+    }
+
+    assert found[("flow", "", "searchable")]["title"] == "Searchable"
+    assert found[("node", "searchable", "sensor")]["title"] == "Hall sensor"
+    assert found[("node", "searchable", "sensor")]["kind"] == "python"
+    assert found[("dashboard", "", "hall")]["title"] == "Hall"
+    assert found[("widget", "hall", "temperature")]["title"] == "Temperature"
+    assert found[("widget", "hall", "temperature")]["kind"] == "stat"
+
+
+def test_secrets_are_named_only_to_a_superuser(
+    client: TestClient,
+    superuser_token_headers: dict[str, str],
+    normal_user_token_headers: dict[str, str],
+) -> None:
+    client.put(
+        f"{SECRETS}/broker_password",
+        headers=superuser_token_headers,
+        json={"value": "hunter2"},
+    )
+
+    def secrets(headers: dict[str, str]) -> set[str]:
+        entries = client.get(f"{PREFIX}/", headers=headers).json()
+        return {e["name"] for e in entries if e["category"] == "secret"}
+
+    assert "broker_password" in secrets(superuser_token_headers)
+    assert secrets(normal_user_token_headers) == set()

backend/tests/flow/test_connector.py

@@ -104,6 +104,40 @@ def test_a_failing_poll_reports_down_and_keeps_going():
     assert health[-1][0] == "ok"
 
 
+def test_an_undeclared_port_keeps_failing_until_the_node_declares_it():
+    """A publication that raised is retried, not remembered as published.
+
+    The loop remembers what it published. If it remembered what it read, a
+    value the node cannot publish would be skipped on the next poll, the poll
+    would succeed, and the node would go back to reporting itself healthy with
+    its port still dark.
+    """
+
+    class Chatty(Sensor):
+        """Reads a port it never declared."""
+
+        async def poll(self) -> dict[str, Any]:
+            self.polls += 1
+            return {"reading": 21.5, "lat": 48.1}
+
+    health: list[tuple[str, str | None]] = []
+    node = Chatty(
+        provides=[MessageSpec(name="reading", dtype=DType.FLOAT)],
+        params={"poll_interval": 0.01},
+    )
+    node.assign_flow("demo", "sensor")
+    node._on_health = lambda _node, status, detail: health.append((status, detail))
+    pipeline = Pipeline(nodes=[node])
+
+    run_briefly(node)
+
+    assert pipeline.state.get("demo.reading") is None
+    assert health[-1][0] == "down"
+    assert "NodeOutputError" in (health[-1][1] or "")
+    # Still failing on the last poll, not just the first.
+    assert len([entry for entry in health if entry[0] == "down"]) > 1
+
+
 class Actuator(ConnectorNode):
     """A connector that commands something instead of reading it."""
 

backend/tests/flow/test_node_teardown.py

@@ -0,0 +1,50 @@
+"""Tearing a node down must not swallow a cancellation meant for the caller."""
+
+import asyncio
+
+import pytest
+
+from fluksio.flow.connector import ConnectorNode
+from fluksio.flow.nodes import DelayNode
+
+
+async def stubborn() -> None:
+    """A loop whose shutdown does not answer the first cancellation."""
+    try:
+        await asyncio.sleep(3600)
+    except asyncio.CancelledError:
+        await asyncio.sleep(3600)
+
+
+def test_stop_cron_lets_the_callers_cancellation_through():
+    async def scenario() -> None:
+        node = DelayNode(params={"cron": "* * * * *"})
+        node._stop_cron = asyncio.Event()
+        node._cron_task = asyncio.create_task(stubborn())
+
+        stopping = asyncio.create_task(node.stop_cron())
+        await asyncio.sleep(0.05)  # let it reach the await on the cron task
+        stopping.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await stopping
+
+        node._cron_task.cancel()
+
+    asyncio.run(scenario())
+
+
+def test_connector_stop_lets_the_callers_cancellation_through():
+    async def scenario() -> None:
+        node = ConnectorNode()
+        node._stop_event = asyncio.Event()
+        node._poll_task = asyncio.create_task(stubborn())
+
+        stopping = asyncio.create_task(node.stop())
+        await asyncio.sleep(0.05)  # let it reach the await on the poll task
+        stopping.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await stopping
+
+        node._poll_task.cancel()
+
+    asyncio.run(scenario())

backend/tests/flow/test_node_types.py

@@ -197,6 +197,47 @@ def test_a_flux_request_is_run_rather_than_written(monkeypatch):
     assert out == {"answer": {"rows": [], "range_s": 3600}}
 
 
+def test_the_configured_timeout_reaches_the_influx_client(monkeypatch):
+    """The client counts in milliseconds; the param is seconds like its peers."""
+    import influxdb_client
+
+    from fluksio.flow.nodes import InfluxDbNode
+
+    seen: dict = {}
+
+    class FakeClient:
+        def __init__(self, **kwargs):
+            seen.update(kwargs)
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *_):
+            return False
+
+        def query_api(self):
+            return self
+
+        def query(self, *_, **__):
+            return []
+
+    monkeypatch.setattr(influxdb_client, "InfluxDBClient", FakeClient)
+
+    node = InfluxDbNode(
+        provides=[MessageSpec(name="answer", dtype=DType.JSON)],
+        params={
+            "url": "http://influx",
+            "token": "t",
+            "org": "o",
+            "bucket": "b",
+            "timeout": 2.5,
+        },
+    )
+    node._run_flux({"flux": 'from(bucket: "b")'})
+
+    assert seen["timeout"] == 2500
+
+
 def test_a_falsy_return_is_a_mistake_not_silence():
     """Only None means "nothing to publish"."""
     from fluksio.flow.nodes import Node

backend/tests/flow/test_placement.py

@@ -129,7 +129,7 @@ def test_a_preferred_label_falls_back_here_and_is_still_accounted(loop):
         assert placer.local.snapshot()["cpus"]["free"] == 1
 
 
-def test_asking_for_more_than_anything_has_gets_what_there_is(loop):
+def test_asking_for_more_than_anything_has_gets_what_there_is(loop, caplog):
     """A flow written on a cluster still has to run on a laptop."""
     placer = placer_over(cpus=2)
 
@@ -138,6 +138,10 @@ def test_asking_for_more_than_anything_has_gets_what_there_is(loop):
         assert allocation.cpus == 2
         assert allocation.gpus == ()
 
+    # Cards are declared, not detected, so a machine that has one reads as
+    # having none until it is told — and the warning is where that is noticed.
+    assert "fluksio serve --gpus" in caplog.text
+
 
 def test_what_is_clamped_to_is_a_machine_that_exists(loop):
     """Each dimension taken separately can describe a machine nobody has.

backend/tests/flow/test_senders.py

@@ -89,3 +89,65 @@ def test_a_string_goes_on_the_wire_bare():
     asyncio.run(node._publish_with(client, {"plug": "ON", "level": 60}))
 
     assert client.published == [("actor/plug", "ON"), ("light/level", "60")]
+
+
+def test_the_configured_timeout_reaches_the_broker_client(monkeypatch):
+    """Without one, a dead socket makes the disconnect ack wait forever."""
+    import aiomqtt
+
+    seen: dict = {}
+
+    class FakeClient:
+        def __init__(self, **kwargs):
+            seen.update(kwargs)
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *_):
+            return False
+
+        async def publish(self, *_, **__):
+            return None
+
+    monkeypatch.setattr(aiomqtt, "Client", FakeClient)
+
+    node = MqttNode(
+        requires=[MessageSpec(name="setpoint", port="setpoint", dtype=DType.FLOAT)],
+        params={"topic": {"setpoint": "heating/setpoint"}, "timeout": 2.5},
+    )
+    asyncio.run(node._publish_once({"setpoint": 21.0}))
+
+    assert seen["timeout"] == 2.5
+
+
+def test_the_configured_backlog_reaches_the_publish_queue(monkeypatch):
+    """The depth is read when the queue is built, so it has to be per node."""
+    import aiomqtt
+
+    class FakeClient:
+        def __init__(self, **kwargs):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *_):
+            return False
+
+    monkeypatch.setattr(aiomqtt, "Client", FakeClient)
+
+    node = MqttNode(
+        requires=[MessageSpec(name="setpoint", port="setpoint", dtype=DType.FLOAT)],
+        params={"topic": {"setpoint": "heating/setpoint"}, "publish_queue_size": 8},
+    )
+    node.assign_flow("heating", "out")
+
+    async def scenario() -> int:
+        await node.start_publisher()
+        assert node._publish_queue is not None
+        size = node._publish_queue.maxsize
+        await node.stop_publisher()
+        return size
+
+    assert asyncio.run(scenario()) == 8

backend/tests/flow/test_unhealthy_node_issue.py

@@ -0,0 +1,42 @@
+"""A node that loaded but is not working shows up as an issue on its flow.
+
+Health used to go nowhere: the connector reported it, the controller stored it,
+and no screen ever asked. These cover the derivation that closes that gap.
+"""
+
+from pathlib import Path
+
+from fluksio.flow.controller import FlowController, LoadedNode
+from fluksio.flow.store import FlowStore
+
+
+def a_controller(tmp_path: Path) -> FlowController:
+    controller = FlowController(FlowStore(tmp_path / "flows"))
+    controller.loaded["house.owm"] = LoadedNode(
+        id="house.owm",
+        flow="house",
+        health="down",
+        health_detail="ConnectionError: name resolution failed",
+    )
+    return controller
+
+
+def test_a_down_node_is_an_issue_on_its_flow(tmp_path: Path) -> None:
+    controller = a_controller(tmp_path)
+
+    issues = controller.flow_issues("house")
+
+    assert [issue.code for issue in issues] == ["node_unhealthy"]
+    assert issues[0].node == "house.owm"
+    assert "name resolution failed" in issues[0].message
+    # Not advisory: the canvas has to mark the node.
+    assert not issues[0].advisory
+    assert controller.flow_issues("other") == []
+
+
+def test_the_issue_clears_when_the_node_reports_itself_well(tmp_path: Path) -> None:
+    controller = a_controller(tmp_path)
+
+    controller.loaded["house.owm"].health = "ok"
+
+    assert controller.flow_issues("house") == []

backend/tests/flow/test_workers.py

@@ -69,6 +69,24 @@ def test_a_node_returns_its_value_and_what_it_printed(pool, capsys):
     assert "seen 21" in capsys.readouterr().out
 
 
+def test_a_node_can_log_through_the_module_it_imports(pool, capsys):
+    """The SDK exports `logger`; inside a worker `fluksio` is the reporter.
+
+    Without it, `fluksio.logger.info(...)` died with AttributeError — after
+    the training it was reporting on had already succeeded.
+    """
+    result = run(
+        pool,
+        "import fluksio\n\n\n"
+        "def process(value):\n"
+        "    fluksio.logger.info('tuned %s', value)\n"
+        "    return {'out': value}\n",
+        value=7,
+    )
+    assert result == {"out": 7}
+    assert "tuned 7" in capsys.readouterr().out
+
+
 def test_a_failure_keeps_its_class_and_points_at_the_node(pool):
     with pytest.raises(Exception) as caught:
         run(pool, "def process():\n    raise ValueError('bad input')\n")
@@ -667,6 +685,19 @@ def test_retiring_workers_reaches_the_children(pool):
     assert child._generation > before
 
 
+def test_retiring_the_cards_leaves_the_other_pools_warm(pool):
+    """A library that preallocated the card only gives it back by dying."""
+    card = pool.for_env({"CUDA_VISIBLE_DEVICES": "0"})
+    threads = pool.for_env({"OMP_NUM_THREADS": "2"})
+    before = (card._generation, threads._generation)
+
+    pool.retire_gpu_children()
+
+    assert card._generation > before[0]
+    # Nothing to hand back, so nothing pays a cold start for it.
+    assert threads._generation == before[1]
+
+
 def test_cancelling_reaches_a_node_running_in_a_child(pool):
     child = pool.for_env({"OMP_NUM_THREADS": "2"})
     started = threading.Event()

backend/tests/sdk/test_build.py

@@ -6,7 +6,7 @@ from fluksio.flow.schemas import FlowDef
 from fluksio.sdk import MARKER, Flow, Port, SyncError, node, use
 
 # The functions live here rather than in each test: a node's module is what the
-# generated body imports, and `__main__` is refused for exactly that reason.
+# generated body imports, and `__main__` is refused at sync for that reason.
 
 
 @node(provides=[Port("dataset", "artifact"), Port("rows", "int")])
@@ -316,3 +316,27 @@ def test_a_negative_timeout_is_refused():
 def test_a_zero_timeout_means_no_limit():
     decorated = node(requires=["a"], timeout=0)(one_default)
     assert decorated.__fluksio__.timeout == 0
+
+
+def test_a_node_in_a_script_run_directly_is_refused_at_sync_not_at_import():
+    """A study module has to be runnable as a script for a self-check.
+
+    The refusal belongs where the body is generated: the decorator hands the
+    function back untouched, so `python study.py` declares its nodes, calls
+    them and checks itself. What cannot be done is syncing them, because
+    nothing could import `__main__`.
+    """
+
+    def probe(rows=1):
+        return {"score": float(rows)}
+
+    probe.__module__ = "__main__"
+    decorated = node(provides=[Port("score", "float")])(probe)
+    # Declared and callable, which is the whole point of running the file.
+    assert decorated(rows=2) == {"score": 2.0}
+
+    flow = Flow("mainflow", nodes=[decorated], outputs=["score"])
+    with pytest.raises(SyncError, match="run directly"):
+        flow.document()
+    with pytest.raises(SyncError, match="run directly"):
+        flow.shims()

backend/tests/test_cli.py

@@ -88,6 +88,24 @@ def test_run_arguments_are_typed_by_the_flow_they_are_for() -> None:
         _params(definition, ["--nonesuch", "1"])
 
 
+def test_a_sweeps_param_spelling_is_refused_by_name() -> None:
+    """`run --param lr=0.002` is a name this flow has not got, and says so."""
+    import pytest
+
+    from fluksio.sdk import SyncError
+    from fluksio.sdk.cli import _params
+
+    definition = {"inputs": [{"spec": {"name": "lr", "dtype": "float"}}]}
+
+    # Not a JSONDecodeError over `lr=0.002`, which is what reading the value
+    # before the name used to give.
+    with pytest.raises(SyncError, match="sweep --param"):
+        _params(definition, ["--param", "lr=0.002"])
+
+    with pytest.raises(SyncError, match="'lr' takes float"):
+        _params(definition, ["--lr", "fast"])
+
+
 def test_serve_uses_the_installation_the_directory_belongs_to(
     tmp_path: Path, monkeypatch
 ) -> None:
@@ -372,6 +390,43 @@ def test_the_metric_names_are_asked_for_rather_than_guessed() -> None:
     assert _list_names(Engine(), args) == 0
 
 
+def test_a_runs_artifact_is_listed_and_downloaded(tmp_path, monkeypatch) -> None:
+    """`save_artifact` had no counterpart: the bytes were API-only."""
+    from fluksio.cli import _parser
+    from fluksio.sdk.cli import _artifacts
+
+    row = {
+        "name": "weights",
+        "node": "fit",
+        "digest": "sha256:abc",
+        "size": 3,
+        "media_type": "application/octet-stream",
+        "filename": "weights.npz",
+    }
+
+    class Engine:
+        def run(self, run_id):
+            assert run_id == "r-1"
+            return {"id": run_id, "status": "ok", "artifacts": [row]}
+
+        def download(self, digest):
+            assert digest == "sha256:abc"
+            return b"abc"
+
+    parser = _parser()
+    monkeypatch.chdir(tmp_path)
+
+    assert _artifacts(Engine(), parser.parse_args(["artifacts", "r-1"])) == 0
+
+    # Written under the name the node saved it as, not the message's.
+    assert _artifacts(Engine(), parser.parse_args(["artifacts", "r-1", "weights"])) == 0
+    assert (tmp_path / "weights.npz").read_bytes() == b"abc"
+
+    args = parser.parse_args(["artifacts", "r-1", "weights", "-o", "here.bin"])
+    assert _artifacts(Engine(), args) == 0
+    assert (tmp_path / "here.bin").read_bytes() == b"abc"
+
+
 def test_an_engine_without_the_route_is_named_rather_than_404() -> None:
     """A client ships ahead of the engine; a flat 404 does not say so."""
     from fluksio.sdk.cli import _too_old
@@ -417,8 +472,38 @@ def test_a_study_in_a_subfolder_is_found(tmp_path) -> None:
     }
 
 
+def test_a_study_per_directory_imports_under_its_own_name(tmp_path) -> None:
+    """One `study.py` per folder is a layout people have, and it works.
+
+    Named for where each sits under the directory being synced, so nothing
+    collides and no `__init__.py` has to be added — which would break the
+    bare `from study import ...` a test beside it does.
+    """
+    import sys
+
+    from fluksio.sdk.cli import discover
+
+    for study in ("s1", "s2"):
+        (tmp_path / "dev" / study).mkdir(parents=True)
+        (tmp_path / "dev" / study / "study.py").write_text(f"VALUE = {study!r}\n")
+
+    try:
+        discover([str(tmp_path / "dev")])
+        assert sys.modules["s1.study"].VALUE == "s1"
+        assert sys.modules["s2.study"].VALUE == "s2"
+    finally:
+        for name in ("s1.study", "s2.study", "s1", "s2"):
+            sys.modules.pop(name, None)
+        sys.path[:] = [entry for entry in sys.path if entry != str(tmp_path / "dev")]
+
+
 def test_two_files_of_one_name_are_refused(tmp_path) -> None:
-    """Python keeps one module per name, and a node's body imports by it."""
+    """Python keeps one module per name, and a node's body imports by it.
+
+    Unreachable from one sync of a directory now that a file is named for
+    where it sits; this is the spelling that still gets there — two files
+    named on the command line, each rooted at its own directory.
+    """
     import pytest
 
     from fluksio.sdk import SyncError
@@ -435,6 +520,77 @@ def test_two_files_of_one_name_are_refused(tmp_path) -> None:
         _import(*_module_of(second), second)
 
 
+def test_how_long_ago_reads_like_a_duration() -> None:
+    """A failure with no time on it says nothing about whether it is current."""
+    from datetime import UTC, datetime, timedelta
+
+    from fluksio.sdk.cli import _ago
+
+    def then(**delta):
+        return (datetime.now(UTC) - timedelta(**delta)).isoformat()
+
+    assert _ago(then(seconds=5)) == "5s ago"
+    assert _ago(then(minutes=3)) == "3min ago"
+    assert _ago(then(hours=2)) == "2h ago"
+    assert _ago(then(days=3)) == "3d ago"
+    # What the engine stores is UTC whether or not the spelling says so.
+    assert _ago(datetime.now(UTC).replace(tzinfo=None).isoformat()) == "0s ago"
+    assert _ago(None) == ""
+
+
+def test_serve_says_when_a_flow_wants_a_card_nobody_declared(
+    tmp_path, monkeypatch, capsys
+) -> None:
+    """The clamp warning goes to the log; this is said while someone is reading.
+
+    Cards are declared rather than detected, so a fresh install that forgets
+    `--gpus` clamps a GPU node to zero and runs them all at once.
+    """
+    from fluksio import cli
+    from fluksio.core.config import settings
+    from fluksio.flow.schemas import FlowDef, NodeDef, Resources
+    from fluksio.flow.store import FlowStore
+
+    store = FlowStore(tmp_path / "flows")
+    store.write_flow(
+        FlowDef(
+            name="finetune",
+            mode="batch",
+            nodes=[NodeDef(id="fit", type="python", resources=Resources(gpus=1))],
+        )
+    )
+    monkeypatch.setattr(settings, "FLOWS_DIR", tmp_path / "flows")
+
+    monkeypatch.setattr(settings, "FLOW_GPUS", 0)
+    cli._mention_undeclared_cards()
+    said = capsys.readouterr().out
+    assert "finetune asks for one" in said
+    assert "--gpus" in said
+
+    # Told how many there are, it has nothing to say.
+    monkeypatch.setattr(settings, "FLOW_GPUS", 1)
+    cli._mention_undeclared_cards()
+    assert capsys.readouterr().out == ""
+
+
+def test_a_serve_limit_is_refused_as_a_flag_not_as_a_traceback(capsys) -> None:
+    """These are written into the environment before the settings are built."""
+    import pytest
+
+    from fluksio.cli import _parser
+
+    parser = _parser()
+    assert parser.parse_args(["serve", "--max-workers", "2"]).max_workers == 2
+    # A machine may genuinely have no card, so zero is a number here.
+    assert parser.parse_args(["serve", "--gpus", "0"]).gpus == 0
+    assert parser.parse_args(["serve"]).gpus is None
+
+    for flag, value in (("--max-workers", "0"), ("--gpus", "-1")):
+        with pytest.raises(SystemExit):
+            parser.parse_args(["serve", flag, value])
+    assert "at least" in capsys.readouterr().err
+
+
 def test_run_and_sweep_take_what_to_sync() -> None:
     from fluksio.cli import _parser
 
@@ -446,6 +602,98 @@ def test_run_and_sweep_take_what_to_sync() -> None:
     assert parser.parse_args(["sweep", "train"]).sync == []
 
 
+def test_the_dashboard_runs_the_engine_as_a_child_of_itself(monkeypatch) -> None:
+    """At a terminal `serve` is a dashboard; the engine is a plain serve.
+
+    Every flag is passed through, so what the child runs with is what serve
+    was asked for — and `--plain` is what stops it opening a second one.
+    """
+    import sys
+
+    from fluksio import cli
+    from fluksio.tui import child_argv
+
+    argv = child_argv(["serve", "--port", "8123", "--gpus", "1"])
+    assert argv[:3] == [sys.executable, "-m", "fluksio.cli"]
+    assert argv[3:] == ["serve", "--port", "8123", "--gpus", "1", "--plain"]
+    # Already plain: told once, not twice.
+    assert child_argv(["serve", "--plain"])[3:] == ["serve", "--plain"]
+
+    opened: list[str] = []
+    monkeypatch.setattr(
+        "fluksio.tui.run_tui", lambda args: opened.append("tui") or 0, raising=False
+    )
+    monkeypatch.setattr(sys.stdout, "isatty", lambda: True, raising=False)
+    monkeypatch.setattr(sys.stdin, "isatty", lambda: True, raising=False)
+
+    parser = cli._parser()
+    assert cli.cmd_serve(parser.parse_args(["serve"])) == 0
+    assert opened == ["tui"]
+
+    # `--plain` goes past it, which is what the child and every container does.
+    # Nothing else of serve runs here, so it fails on the data directory it is
+    # given rather than opening a dashboard.
+    opened.clear()
+    monkeypatch.setattr(
+        cli, "_data_dir", lambda *a, **k: (_ for _ in ()).throw(SystemExit(3))
+    )
+    with __import__("pytest").raises(SystemExit):
+        cli.cmd_serve(parser.parse_args(["serve", "--plain"]))
+    assert opened == []
+
+
+def test_a_serving_engine_records_itself_until_it_stops(tmp_path) -> None:
+    """A pid nobody is running is the same as no pidfile at all."""
+    import os
+
+    from fluksio.cli import read_pidfile, write_pidfile
+
+    assert read_pidfile(tmp_path) is None
+
+    written = write_pidfile(tmp_path, 8000)
+    assert read_pidfile(tmp_path) == {"pid": os.getpid(), "port": 8000}
+
+    # Killed outright: the file outlives the process it names.
+    written.write_text('{"pid": 2147483646, "port": 8000}')
+    assert read_pidfile(tmp_path) is None
+
+    written.write_text("not json")
+    assert read_pidfile(tmp_path) is None
+
+
+def test_who_holds_the_port_is_told_apart_by_the_token() -> None:
+    """Only this directory's own engine may be reported as already up.
+
+    The token is signed with this directory's secret key, so an engine that
+    accepts it is one reading this directory's database. Another
+    installation's Fluksio answers the health check and refuses it.
+    """
+    import httpx
+
+    from fluksio.cli import probe_engine
+
+    def engine(health: int, summary: int):
+        def handle(request: httpx.Request) -> httpx.Response:
+            if request.url.path.endswith("/health-check/"):
+                return httpx.Response(health)
+            return httpx.Response(summary)
+
+        return httpx.Client(transport=httpx.MockTransport(handle))
+
+    with engine(200, 200) as client:
+        assert probe_engine("http://x", "t", client) == "ours"
+    with engine(200, 401) as client:
+        assert probe_engine("http://x", "t", client) == "foreign"
+    # A directory with no credential yet cannot prove anything is its own, and
+    # `Bearer ` is not a legal header value — so it asks without one.
+    with engine(200, 401) as client:
+        assert probe_engine("http://x", "", client) == "foreign"
+    # Somebody else's dev server, or nothing listening at all.
+    with engine(404, 404) as client:
+        assert probe_engine("http://x", "t", client) == "other"
+    assert probe_engine("http://127.0.0.1:1", "t") == "other"
+
+
 def test_serve_moves_off_a_port_that_is_taken() -> None:
     """A first start should not die on somebody else's dev server."""
     import socket