SDK: artifacts work outside a node, run --follow prints status and queue waits, fluksio cancel

save_artifact/load_artifact outside a node use the worker's own store code in
$FLUKSIO_ARTIFACT_DIR or <tmp>/fluksio-artifacts, so a node can be called by
hand. --follow prints status changes and each node's resource wait; the
placer's waiting snapshot now carries the run. fluksio cancel <run>...

backend/tests/test_cli.py

@@ -309,6 +309,103 @@ def test_ctrl_c_while_waiting_cancels_the_run(monkeypatch) -> None:
     assert cancelled == ["run-1"]
 
 
+def test_follow_says_what_a_run_that_reports_nothing_is_doing(capsys) -> None:
+    """Queued for a worker, then waiting for a card: both are said once."""
+    from fluksio.sdk import cli
+    from fluksio.sdk.client import ApiError
+
+    held = "Waiting for a worker with 1 gpu(s)"
+    rows = iter(
+        [
+            {"status": "queued", "status_reason": held},
+            {"status": "queued", "status_reason": held},
+            # The engine leaves the reason on the row once the run starts.
+            {"status": "running", "status_reason": held},
+            {"status": "running", "status_reason": held},
+            {"status": "running", "status_reason": held},
+            {"status": "ok", "status_reason": ""},
+        ]
+    )
+
+    class FakeHandle:
+        id = "r1"
+        status = "queued"
+        status_reason = ""
+
+        @property
+        def done(self) -> bool:
+            return self.status == "ok"
+
+        def refresh(self) -> "FakeHandle":
+            row = next(rows)
+            self.status, self.status_reason = row["status"], row["status_reason"]
+            return self
+
+    asked: list[int] = []
+
+    class FakeClient:
+        def metrics(self, run_id: str) -> list:
+            return []
+
+        def resources(self) -> dict:
+            asked.append(1)
+            if len(asked) == 3:
+                raise ApiError(503, "Resources are not accounted here")
+            return {
+                "waiting": [
+                    {
+                        "node": "train.fit",
+                        "run": "r1",
+                        "reason": "waiting for 1 gpu(s)",
+                    },
+                    {
+                        "node": "train.fit",
+                        "run": "r2",
+                        "reason": "waiting for 9 gpu(s)",
+                    },
+                ]
+            }
+
+    cli._follow(FakeClient(), FakeHandle(), poll=0)
+
+    assert capsys.readouterr().out.splitlines() == [
+        f"  queued  {held}",
+        "  running",
+        "  train.fit  waiting for 1 gpu(s)",
+    ]
+
+
+def test_cancel_stops_the_runs_it_is_given(monkeypatch, capsys) -> None:
+    from contextlib import contextmanager
+
+    from fluksio.cli import _parser
+    from fluksio.sdk import cli
+    from fluksio.sdk.client import ApiError
+
+    cancelled: list[str] = []
+
+    class FakeClient:
+        def cancel(self, run_id: str) -> dict[str, object]:
+            if run_id == "gone":
+                raise ApiError(404, "No such run")
+            cancelled.append(run_id)
+            return {"status": "ok" if run_id == "done" else "cancelled"}
+
+    @contextmanager
+    def fake_client(_args):
+        yield FakeClient()
+
+    monkeypatch.setattr(cli, "_client_for", fake_client)
+
+    args = _parser().parse_args(["cancel", "r1", "done"])
+    assert cli.cmd_cancel(args) == 0
+    assert cancelled == ["r1", "done"]
+    # A run already past its last node says how it ended, not that it stopped.
+    assert capsys.readouterr().out.splitlines() == ["r1  cancelled", "done  ok"]
+
+    assert cli.cmd_cancel(_parser().parse_args(["cancel", "gone"])) == 1
+
+
 def test_a_failed_run_prints_what_each_node_said(monkeypatch, capsys) -> None:
     """Otherwise the traceback is stored and nothing at a terminal shows it."""
     from contextlib import contextmanager