SDK: artifacts work outside a node, run --follow prints status and queue waits, fluksio cancel

save_artifact/load_artifact outside a node use the worker's own store code in
$FLUKSIO_ARTIFACT_DIR or <tmp>/fluksio-artifacts, so a node can be called by
hand. --follow prints status changes and each node's resource wait; the
placer's waiting snapshot now carries the run. fluksio cancel <run>...
This commit is contained in:
2026-10-28 08:07:35 +01:00
parent 4c2df681d3
commit 34243e6902
14 changed files with 274 additions and 27 deletions
+3 -1
View File
@@ -176,7 +176,7 @@ def test_a_node_waits_for_a_busy_machine_and_says_so(loop):
started = threading.Event()
def second() -> None:
with placer.claim(Resources(cpus=2), node="study.b"):
with placer.claim(Resources(cpus=2), node="study.b", run="r2"):
started.set()
with placer.claim(Resources(cpus=2), node="study.a"):
@@ -185,6 +185,8 @@ def test_a_node_waits_for_a_busy_machine_and_says_so(loop):
# Long enough to have run if nothing was holding it back.
assert not started.wait(0.3)
assert placer.snapshot()["waiting"][0]["node"] == "study.b"
# Whose wait it is, so a client following one run reads only its own.
assert placer.snapshot()["waiting"][0]["run"] == "r2"
assert events[0]["type"] == "node_queued"
assert "2 cpu(s)" in events[0]["detail"]
+37
View File
@@ -1,5 +1,7 @@
"""What the decorators declare, and what the engine would be given."""
from pathlib import Path
import pytest
from fluksio.flow.schemas import FlowDef
@@ -396,3 +398,38 @@ def test_a_flow_records_the_module_that_declared_it():
# A repository the file is not in has no path to give it.
elsewhere = flow.document({"kind": "python", "repo": "/elsewhere"})["origin"]
assert (elsewhere["declared_in"], elsewhere["module"]) == ("", "")
def test_artifacts_round_trip_outside_a_node(tmp_path, monkeypatch):
"""A node that returns an artifact can be smoke-tested by calling it."""
import fluksio
monkeypatch.setenv("FLUKSIO_ARTIFACT_DIR", str(tmp_path))
ref = fluksio.save_artifact(b"weights", "w.bin", "application/x-test")
assert ref["name"] == "w.bin"
assert ref["size"] == 7
assert ref["media_type"] == "application/x-test"
assert ref["digest"].startswith("sha256:")
assert Path(fluksio.load_artifact(ref)).read_bytes() == b"weights"
assert str(tmp_path) in fluksio.load_artifact(ref)
source = tmp_path / "plot.png"
source.write_bytes(b"png")
assert fluksio.save_artifact(source)["name"] == "plot.png"
def test_the_local_artifact_store_defaults_to_the_temporary_directory(monkeypatch):
import tempfile
import fluksio
# Set and removed again, so the variable the default writes is undone too.
monkeypatch.setenv("FLUKSIO_ARTIFACT_DIR", "")
monkeypatch.delenv("FLUKSIO_ARTIFACT_DIR")
ref = fluksio.save_artifact(b"x" * 3)
assert fluksio.load_artifact(ref).startswith(
str(Path(tempfile.gettempdir()) / "fluksio-artifacts")
)
+97
View File
@@ -309,6 +309,103 @@ def test_ctrl_c_while_waiting_cancels_the_run(monkeypatch) -> None:
assert cancelled == ["run-1"]
def test_follow_says_what_a_run_that_reports_nothing_is_doing(capsys) -> None:
"""Queued for a worker, then waiting for a card: both are said once."""
from fluksio.sdk import cli
from fluksio.sdk.client import ApiError
held = "Waiting for a worker with 1 gpu(s)"
rows = iter(
[
{"status": "queued", "status_reason": held},
{"status": "queued", "status_reason": held},
# The engine leaves the reason on the row once the run starts.
{"status": "running", "status_reason": held},
{"status": "running", "status_reason": held},
{"status": "running", "status_reason": held},
{"status": "ok", "status_reason": ""},
]
)
class FakeHandle:
id = "r1"
status = "queued"
status_reason = ""
@property
def done(self) -> bool:
return self.status == "ok"
def refresh(self) -> "FakeHandle":
row = next(rows)
self.status, self.status_reason = row["status"], row["status_reason"]
return self
asked: list[int] = []
class FakeClient:
def metrics(self, run_id: str) -> list:
return []
def resources(self) -> dict:
asked.append(1)
if len(asked) == 3:
raise ApiError(503, "Resources are not accounted here")
return {
"waiting": [
{
"node": "train.fit",
"run": "r1",
"reason": "waiting for 1 gpu(s)",
},
{
"node": "train.fit",
"run": "r2",
"reason": "waiting for 9 gpu(s)",
},
]
}
cli._follow(FakeClient(), FakeHandle(), poll=0)
assert capsys.readouterr().out.splitlines() == [
f" queued {held}",
" running",
" train.fit waiting for 1 gpu(s)",
]
def test_cancel_stops_the_runs_it_is_given(monkeypatch, capsys) -> None:
from contextlib import contextmanager
from fluksio.cli import _parser
from fluksio.sdk import cli
from fluksio.sdk.client import ApiError
cancelled: list[str] = []
class FakeClient:
def cancel(self, run_id: str) -> dict[str, object]:
if run_id == "gone":
raise ApiError(404, "No such run")
cancelled.append(run_id)
return {"status": "ok" if run_id == "done" else "cancelled"}
@contextmanager
def fake_client(_args):
yield FakeClient()
monkeypatch.setattr(cli, "_client_for", fake_client)
args = _parser().parse_args(["cancel", "r1", "done"])
assert cli.cmd_cancel(args) == 0
assert cancelled == ["r1", "done"]
# A run already past its last node says how it ended, not that it stopped.
assert capsys.readouterr().out.splitlines() == ["r1 cancelled", "done ok"]
assert cli.cmd_cancel(_parser().parse_args(["cancel", "gone"])) == 1
def test_a_failed_run_prints_what_each_node_said(monkeypatch, capsys) -> None:
"""Otherwise the traceback is stored and nothing at a terminal shows it."""
from contextlib import contextmanager