SDK: artifacts work outside a node, run --follow prints status and queue waits, fluksio cancel
save_artifact/load_artifact outside a node use the worker's own store code in $FLUKSIO_ARTIFACT_DIR or <tmp>/fluksio-artifacts, so a node can be called by hand. --follow prints status changes and each node's resource wait; the placer's waiting snapshot now carries the run. fluksio cancel <run>...
This commit is contained in:
@@ -176,7 +176,7 @@ def test_a_node_waits_for_a_busy_machine_and_says_so(loop):
|
||||
started = threading.Event()
|
||||
|
||||
def second() -> None:
|
||||
with placer.claim(Resources(cpus=2), node="study.b"):
|
||||
with placer.claim(Resources(cpus=2), node="study.b", run="r2"):
|
||||
started.set()
|
||||
|
||||
with placer.claim(Resources(cpus=2), node="study.a"):
|
||||
@@ -185,6 +185,8 @@ def test_a_node_waits_for_a_busy_machine_and_says_so(loop):
|
||||
# Long enough to have run if nothing was holding it back.
|
||||
assert not started.wait(0.3)
|
||||
assert placer.snapshot()["waiting"][0]["node"] == "study.b"
|
||||
# Whose wait it is, so a client following one run reads only its own.
|
||||
assert placer.snapshot()["waiting"][0]["run"] == "r2"
|
||||
assert events[0]["type"] == "node_queued"
|
||||
assert "2 cpu(s)" in events[0]["detail"]
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""What the decorators declare, and what the engine would be given."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from fluksio.flow.schemas import FlowDef
|
||||
@@ -396,3 +398,38 @@ def test_a_flow_records_the_module_that_declared_it():
|
||||
# A repository the file is not in has no path to give it.
|
||||
elsewhere = flow.document({"kind": "python", "repo": "/elsewhere"})["origin"]
|
||||
assert (elsewhere["declared_in"], elsewhere["module"]) == ("", "")
|
||||
|
||||
|
||||
def test_artifacts_round_trip_outside_a_node(tmp_path, monkeypatch):
|
||||
"""A node that returns an artifact can be smoke-tested by calling it."""
|
||||
import fluksio
|
||||
|
||||
monkeypatch.setenv("FLUKSIO_ARTIFACT_DIR", str(tmp_path))
|
||||
|
||||
ref = fluksio.save_artifact(b"weights", "w.bin", "application/x-test")
|
||||
|
||||
assert ref["name"] == "w.bin"
|
||||
assert ref["size"] == 7
|
||||
assert ref["media_type"] == "application/x-test"
|
||||
assert ref["digest"].startswith("sha256:")
|
||||
assert Path(fluksio.load_artifact(ref)).read_bytes() == b"weights"
|
||||
assert str(tmp_path) in fluksio.load_artifact(ref)
|
||||
|
||||
source = tmp_path / "plot.png"
|
||||
source.write_bytes(b"png")
|
||||
assert fluksio.save_artifact(source)["name"] == "plot.png"
|
||||
|
||||
|
||||
def test_the_local_artifact_store_defaults_to_the_temporary_directory(monkeypatch):
|
||||
import tempfile
|
||||
|
||||
import fluksio
|
||||
|
||||
# Set and removed again, so the variable the default writes is undone too.
|
||||
monkeypatch.setenv("FLUKSIO_ARTIFACT_DIR", "")
|
||||
monkeypatch.delenv("FLUKSIO_ARTIFACT_DIR")
|
||||
ref = fluksio.save_artifact(b"x" * 3)
|
||||
|
||||
assert fluksio.load_artifact(ref).startswith(
|
||||
str(Path(tempfile.gettempdir()) / "fluksio-artifacts")
|
||||
)
|
||||
|
||||
@@ -309,6 +309,103 @@ def test_ctrl_c_while_waiting_cancels_the_run(monkeypatch) -> None:
|
||||
assert cancelled == ["run-1"]
|
||||
|
||||
|
||||
def test_follow_says_what_a_run_that_reports_nothing_is_doing(capsys) -> None:
|
||||
"""Queued for a worker, then waiting for a card: both are said once."""
|
||||
from fluksio.sdk import cli
|
||||
from fluksio.sdk.client import ApiError
|
||||
|
||||
held = "Waiting for a worker with 1 gpu(s)"
|
||||
rows = iter(
|
||||
[
|
||||
{"status": "queued", "status_reason": held},
|
||||
{"status": "queued", "status_reason": held},
|
||||
# The engine leaves the reason on the row once the run starts.
|
||||
{"status": "running", "status_reason": held},
|
||||
{"status": "running", "status_reason": held},
|
||||
{"status": "running", "status_reason": held},
|
||||
{"status": "ok", "status_reason": ""},
|
||||
]
|
||||
)
|
||||
|
||||
class FakeHandle:
|
||||
id = "r1"
|
||||
status = "queued"
|
||||
status_reason = ""
|
||||
|
||||
@property
|
||||
def done(self) -> bool:
|
||||
return self.status == "ok"
|
||||
|
||||
def refresh(self) -> "FakeHandle":
|
||||
row = next(rows)
|
||||
self.status, self.status_reason = row["status"], row["status_reason"]
|
||||
return self
|
||||
|
||||
asked: list[int] = []
|
||||
|
||||
class FakeClient:
|
||||
def metrics(self, run_id: str) -> list:
|
||||
return []
|
||||
|
||||
def resources(self) -> dict:
|
||||
asked.append(1)
|
||||
if len(asked) == 3:
|
||||
raise ApiError(503, "Resources are not accounted here")
|
||||
return {
|
||||
"waiting": [
|
||||
{
|
||||
"node": "train.fit",
|
||||
"run": "r1",
|
||||
"reason": "waiting for 1 gpu(s)",
|
||||
},
|
||||
{
|
||||
"node": "train.fit",
|
||||
"run": "r2",
|
||||
"reason": "waiting for 9 gpu(s)",
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
cli._follow(FakeClient(), FakeHandle(), poll=0)
|
||||
|
||||
assert capsys.readouterr().out.splitlines() == [
|
||||
f" queued {held}",
|
||||
" running",
|
||||
" train.fit waiting for 1 gpu(s)",
|
||||
]
|
||||
|
||||
|
||||
def test_cancel_stops_the_runs_it_is_given(monkeypatch, capsys) -> None:
|
||||
from contextlib import contextmanager
|
||||
|
||||
from fluksio.cli import _parser
|
||||
from fluksio.sdk import cli
|
||||
from fluksio.sdk.client import ApiError
|
||||
|
||||
cancelled: list[str] = []
|
||||
|
||||
class FakeClient:
|
||||
def cancel(self, run_id: str) -> dict[str, object]:
|
||||
if run_id == "gone":
|
||||
raise ApiError(404, "No such run")
|
||||
cancelled.append(run_id)
|
||||
return {"status": "ok" if run_id == "done" else "cancelled"}
|
||||
|
||||
@contextmanager
|
||||
def fake_client(_args):
|
||||
yield FakeClient()
|
||||
|
||||
monkeypatch.setattr(cli, "_client_for", fake_client)
|
||||
|
||||
args = _parser().parse_args(["cancel", "r1", "done"])
|
||||
assert cli.cmd_cancel(args) == 0
|
||||
assert cancelled == ["r1", "done"]
|
||||
# A run already past its last node says how it ended, not that it stopped.
|
||||
assert capsys.readouterr().out.splitlines() == ["r1 cancelled", "done ok"]
|
||||
|
||||
assert cli.cmd_cancel(_parser().parse_args(["cancel", "gone"])) == 1
|
||||
|
||||
|
||||
def test_a_failed_run_prints_what_each_node_said(monkeypatch, capsys) -> None:
|
||||
"""Otherwise the traceback is stored and nothing at a terminal shows it."""
|
||||
from contextlib import contextmanager
|
||||
|
||||
Reference in New Issue
Block a user