SDK: artifacts work outside a node, run --follow prints status and queue waits, fluksio cancel
save_artifact/load_artifact outside a node use the worker's own store code in $FLUKSIO_ARTIFACT_DIR or <tmp>/fluksio-artifacts, so a node can be called by hand. --follow prints status changes and each node's resource wait; the placer's waiting snapshot now carries the run. fluksio cancel <run>...
This commit is contained in:
+33
-11
@@ -11,9 +11,10 @@ Inside a node, ``import fluksio`` is not this package at all: the worker
|
||||
installs a reporter of its own under that name before any node code runs, so
|
||||
what a node gets is :mod:`fluksio_worker.worker_main`'s ``emit`` and
|
||||
``save_artifact`` — and inert copies of the decorators, since a module the
|
||||
node imports may well declare them at its top. The stubs below stand in the
|
||||
same place everywhere else, and say so rather than failing as a missing
|
||||
attribute.
|
||||
node imports may well declare them at its top. Everywhere else the functions
|
||||
below stand in, so a node called directly — in a test, in a notebook — behaves
|
||||
like the real one: ``emit`` does nothing and the artifact helpers use a local
|
||||
directory.
|
||||
|
||||
Nothing is imported from the rest of the package here. Every ``from fluksio.x
|
||||
import y`` in the engine passes through this module, and an import cycle or a
|
||||
@@ -21,6 +22,7 @@ second of start-up cost would both begin here.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from importlib.metadata import PackageNotFoundError
|
||||
from importlib.metadata import version as _version
|
||||
from typing import Any
|
||||
@@ -46,11 +48,6 @@ __all__ = [
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_OUTSIDE = (
|
||||
"fluksio.{name}() only works inside a node: the worker running it installs "
|
||||
"the real one. There is nothing to {verb} out here."
|
||||
)
|
||||
|
||||
|
||||
def emit(**ports: Any) -> None:
|
||||
"""Publish values on a node's declared output ports, mid-run.
|
||||
@@ -63,16 +60,41 @@ def emit(**ports: Any) -> None:
|
||||
logger.debug("fluksio.emit(%s) outside a node", ", ".join(ports))
|
||||
|
||||
|
||||
def _local_store() -> Any:
|
||||
"""The worker's own artifact code, pointed at a directory of this machine.
|
||||
|
||||
``$FLUKSIO_ARTIFACT_DIR`` when it is set, which is how a worker finds the
|
||||
engine's store, and a directory in the temporary location otherwise. The
|
||||
references that come out are the ones a node gets inside the engine, so
|
||||
what a node returns can be handed to the next one by hand.
|
||||
"""
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from fluksio_worker.worker_main import ARTIFACT_DIR_ENV, _Reporter
|
||||
|
||||
os.environ.setdefault(
|
||||
ARTIFACT_DIR_ENV, str(Path(tempfile.gettempdir()) / "fluksio-artifacts")
|
||||
)
|
||||
return _Reporter("fluksio")
|
||||
|
||||
|
||||
def save_artifact(
|
||||
source: Any,
|
||||
name: str = "",
|
||||
media_type: str = "application/octet-stream",
|
||||
volatile: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Put bytes in the artifact store and return a reference to them."""
|
||||
raise RuntimeError(_OUTSIDE.format(name="save_artifact", verb="save to"))
|
||||
"""Put bytes or a file in the artifact store and return a reference to them.
|
||||
|
||||
Outside a node the store is a local directory, see :func:`_local_store`.
|
||||
"""
|
||||
reference: dict[str, Any] = _local_store().save_artifact(
|
||||
source, name, media_type, volatile
|
||||
)
|
||||
return reference
|
||||
|
||||
|
||||
def load_artifact(ref: dict[str, Any]) -> str:
|
||||
"""Fetch what an artifact reference points at, and return a path to it."""
|
||||
raise RuntimeError(_OUTSIDE.format(name="load_artifact", verb="load from"))
|
||||
return str(_local_store().load_artifact(ref))
|
||||
|
||||
@@ -134,6 +134,8 @@ class ResourceLevel(BaseModel):
|
||||
|
||||
class WaitingNode(BaseModel):
|
||||
node: str
|
||||
#: The run it belongs to, so a client can tell its own wait from another's.
|
||||
run: str = ""
|
||||
reason: str
|
||||
seconds: float
|
||||
|
||||
|
||||
@@ -79,6 +79,7 @@ class _Waiting:
|
||||
node: str
|
||||
since: float
|
||||
reason: str
|
||||
run: str = ""
|
||||
|
||||
|
||||
class Placer:
|
||||
@@ -333,7 +334,7 @@ class Placer:
|
||||
"""
|
||||
self._ticket += 1
|
||||
ticket = self._ticket
|
||||
self._waiting[ticket] = _Waiting(node=node, since=since, reason=reason)
|
||||
self._waiting[ticket] = _Waiting(node=node, since=since, reason=reason, run=run)
|
||||
logger.info("%s is %s", node or "a node", reason)
|
||||
if self.events is not None:
|
||||
self.events.publish(
|
||||
@@ -407,6 +408,7 @@ class Placer:
|
||||
"waiting": [
|
||||
{
|
||||
"node": entry.node,
|
||||
"run": entry.run,
|
||||
"reason": entry.reason,
|
||||
"seconds": round(time.monotonic() - entry.since, 1),
|
||||
}
|
||||
|
||||
@@ -585,8 +585,38 @@ def _sync_first(
|
||||
_say(f"synced {', '.join(r.flow for r in changed)}")
|
||||
|
||||
|
||||
def _waits(client: Client, run_id: str) -> list[tuple[str, str]]:
|
||||
"""What this run's nodes are queued for on a machine, as (node, reason).
|
||||
|
||||
A run that says nothing for an hour is either hung or waiting its turn, and
|
||||
this is the half of that answer a metric cannot give. Best effort: an
|
||||
engine that does not account for resources, or is too old to say whose a
|
||||
wait is, simply has nothing to tell.
|
||||
"""
|
||||
try:
|
||||
waiting = client.resources().get("waiting") or []
|
||||
except (httpx.HTTPError, ApiError):
|
||||
return []
|
||||
return [
|
||||
(str(entry.get("node", "")), str(entry.get("reason", "")))
|
||||
for entry in waiting
|
||||
if entry.get("run") == run_id
|
||||
]
|
||||
|
||||
|
||||
def _state(handle: RunHandle) -> tuple[str, str]:
|
||||
"""Its status, and why — the reason only while it is true.
|
||||
|
||||
The engine leaves "waiting for a worker" on the row after the run starts.
|
||||
"""
|
||||
return handle.status, handle.status_reason if handle.status == "queued" else ""
|
||||
|
||||
|
||||
def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
|
||||
"""Print a run's numbers as they arrive, until it is over.
|
||||
"""Print what a run does as it does it, until it is over.
|
||||
|
||||
Its numbers, a change of status, and what a queued node is waiting for, so
|
||||
a run that reports nothing is not a silent one.
|
||||
|
||||
Polled rather than pushed: the engine writes a metric down when it is
|
||||
reported, so asking once a second draws the same curve a socket would
|
||||
@@ -594,6 +624,8 @@ def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
|
||||
before the numbers, so the last batch is never the one that gets missed.
|
||||
"""
|
||||
seen: set[tuple[str, int]] = set()
|
||||
told: set[tuple[str, str]] = set()
|
||||
state = _state(handle)
|
||||
failures = 0
|
||||
while True:
|
||||
try:
|
||||
@@ -610,6 +642,16 @@ def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
|
||||
time.sleep(poll)
|
||||
continue
|
||||
failures = 0
|
||||
if not done:
|
||||
now = _state(handle)
|
||||
if now != state:
|
||||
state = now
|
||||
_say(f" {_status(now[0])}" + (f" {now[1]}" if now[1] else ""))
|
||||
if handle.status == "running":
|
||||
for wait in _waits(client, handle.id):
|
||||
if wait not in told:
|
||||
told.add(wait)
|
||||
_say(f" {wait[0]} {wait[1]}")
|
||||
for point in points:
|
||||
mark = (str(point.get("name", "")), int(point.get("step", -1)))
|
||||
if mark in seen:
|
||||
@@ -1114,6 +1156,20 @@ def cmd_retry(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_cancel(args: argparse.Namespace) -> int:
|
||||
"""Stop runs by id, which `run` without `--wait` leaves no other way to do."""
|
||||
try:
|
||||
with _client_for(args) as client:
|
||||
for run_id in args.run_id:
|
||||
made = client.cancel(run_id)
|
||||
_say(f"{run_id} {_status(str(made.get('status') or 'cancelled'))}")
|
||||
except (SyncError, ApiError) as exc:
|
||||
return _fail(str(exc))
|
||||
except httpx.HTTPError as exc:
|
||||
return _unreachable(exc)
|
||||
return 0
|
||||
|
||||
|
||||
def _artifacts(client: Client, args: argparse.Namespace) -> int:
|
||||
"""List a run's files, or write one of them here."""
|
||||
handle = RunHandle(client, args.run_id, client.run(args.run_id))
|
||||
@@ -1524,6 +1580,11 @@ def add_parsers(subparsers: Any) -> None:
|
||||
with_engine(parser)
|
||||
parser.set_defaults(func=cmd_retry)
|
||||
|
||||
parser = subparsers.add_parser("cancel", help="stop runs that are still going")
|
||||
parser.add_argument("run_id", nargs="+", help="the runs to stop")
|
||||
with_engine(parser)
|
||||
parser.set_defaults(func=cmd_cancel)
|
||||
|
||||
parser = subparsers.add_parser(
|
||||
"artifacts", help="the files a run produced; name one to download it"
|
||||
)
|
||||
|
||||
@@ -577,6 +577,11 @@ class RunHandle:
|
||||
def status(self) -> str:
|
||||
return str(self._row.get("status", ""))
|
||||
|
||||
@property
|
||||
def status_reason(self) -> str:
|
||||
"""Why it is in that status, such as what a queued run is waiting for."""
|
||||
return str(self._row.get("status_reason") or "")
|
||||
|
||||
@property
|
||||
def done(self) -> bool:
|
||||
return self.status in DONE
|
||||
|
||||
@@ -176,7 +176,7 @@ def test_a_node_waits_for_a_busy_machine_and_says_so(loop):
|
||||
started = threading.Event()
|
||||
|
||||
def second() -> None:
|
||||
with placer.claim(Resources(cpus=2), node="study.b"):
|
||||
with placer.claim(Resources(cpus=2), node="study.b", run="r2"):
|
||||
started.set()
|
||||
|
||||
with placer.claim(Resources(cpus=2), node="study.a"):
|
||||
@@ -185,6 +185,8 @@ def test_a_node_waits_for_a_busy_machine_and_says_so(loop):
|
||||
# Long enough to have run if nothing was holding it back.
|
||||
assert not started.wait(0.3)
|
||||
assert placer.snapshot()["waiting"][0]["node"] == "study.b"
|
||||
# Whose wait it is, so a client following one run reads only its own.
|
||||
assert placer.snapshot()["waiting"][0]["run"] == "r2"
|
||||
assert events[0]["type"] == "node_queued"
|
||||
assert "2 cpu(s)" in events[0]["detail"]
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""What the decorators declare, and what the engine would be given."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from fluksio.flow.schemas import FlowDef
|
||||
@@ -396,3 +398,38 @@ def test_a_flow_records_the_module_that_declared_it():
|
||||
# A repository the file is not in has no path to give it.
|
||||
elsewhere = flow.document({"kind": "python", "repo": "/elsewhere"})["origin"]
|
||||
assert (elsewhere["declared_in"], elsewhere["module"]) == ("", "")
|
||||
|
||||
|
||||
def test_artifacts_round_trip_outside_a_node(tmp_path, monkeypatch):
|
||||
"""A node that returns an artifact can be smoke-tested by calling it."""
|
||||
import fluksio
|
||||
|
||||
monkeypatch.setenv("FLUKSIO_ARTIFACT_DIR", str(tmp_path))
|
||||
|
||||
ref = fluksio.save_artifact(b"weights", "w.bin", "application/x-test")
|
||||
|
||||
assert ref["name"] == "w.bin"
|
||||
assert ref["size"] == 7
|
||||
assert ref["media_type"] == "application/x-test"
|
||||
assert ref["digest"].startswith("sha256:")
|
||||
assert Path(fluksio.load_artifact(ref)).read_bytes() == b"weights"
|
||||
assert str(tmp_path) in fluksio.load_artifact(ref)
|
||||
|
||||
source = tmp_path / "plot.png"
|
||||
source.write_bytes(b"png")
|
||||
assert fluksio.save_artifact(source)["name"] == "plot.png"
|
||||
|
||||
|
||||
def test_the_local_artifact_store_defaults_to_the_temporary_directory(monkeypatch):
|
||||
import tempfile
|
||||
|
||||
import fluksio
|
||||
|
||||
# Set and removed again, so the variable the default writes is undone too.
|
||||
monkeypatch.setenv("FLUKSIO_ARTIFACT_DIR", "")
|
||||
monkeypatch.delenv("FLUKSIO_ARTIFACT_DIR")
|
||||
ref = fluksio.save_artifact(b"x" * 3)
|
||||
|
||||
assert fluksio.load_artifact(ref).startswith(
|
||||
str(Path(tempfile.gettempdir()) / "fluksio-artifacts")
|
||||
)
|
||||
|
||||
@@ -309,6 +309,103 @@ def test_ctrl_c_while_waiting_cancels_the_run(monkeypatch) -> None:
|
||||
assert cancelled == ["run-1"]
|
||||
|
||||
|
||||
def test_follow_says_what_a_run_that_reports_nothing_is_doing(capsys) -> None:
|
||||
"""Queued for a worker, then waiting for a card: both are said once."""
|
||||
from fluksio.sdk import cli
|
||||
from fluksio.sdk.client import ApiError
|
||||
|
||||
held = "Waiting for a worker with 1 gpu(s)"
|
||||
rows = iter(
|
||||
[
|
||||
{"status": "queued", "status_reason": held},
|
||||
{"status": "queued", "status_reason": held},
|
||||
# The engine leaves the reason on the row once the run starts.
|
||||
{"status": "running", "status_reason": held},
|
||||
{"status": "running", "status_reason": held},
|
||||
{"status": "running", "status_reason": held},
|
||||
{"status": "ok", "status_reason": ""},
|
||||
]
|
||||
)
|
||||
|
||||
class FakeHandle:
|
||||
id = "r1"
|
||||
status = "queued"
|
||||
status_reason = ""
|
||||
|
||||
@property
|
||||
def done(self) -> bool:
|
||||
return self.status == "ok"
|
||||
|
||||
def refresh(self) -> "FakeHandle":
|
||||
row = next(rows)
|
||||
self.status, self.status_reason = row["status"], row["status_reason"]
|
||||
return self
|
||||
|
||||
asked: list[int] = []
|
||||
|
||||
class FakeClient:
|
||||
def metrics(self, run_id: str) -> list:
|
||||
return []
|
||||
|
||||
def resources(self) -> dict:
|
||||
asked.append(1)
|
||||
if len(asked) == 3:
|
||||
raise ApiError(503, "Resources are not accounted here")
|
||||
return {
|
||||
"waiting": [
|
||||
{
|
||||
"node": "train.fit",
|
||||
"run": "r1",
|
||||
"reason": "waiting for 1 gpu(s)",
|
||||
},
|
||||
{
|
||||
"node": "train.fit",
|
||||
"run": "r2",
|
||||
"reason": "waiting for 9 gpu(s)",
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
cli._follow(FakeClient(), FakeHandle(), poll=0)
|
||||
|
||||
assert capsys.readouterr().out.splitlines() == [
|
||||
f" queued {held}",
|
||||
" running",
|
||||
" train.fit waiting for 1 gpu(s)",
|
||||
]
|
||||
|
||||
|
||||
def test_cancel_stops_the_runs_it_is_given(monkeypatch, capsys) -> None:
|
||||
from contextlib import contextmanager
|
||||
|
||||
from fluksio.cli import _parser
|
||||
from fluksio.sdk import cli
|
||||
from fluksio.sdk.client import ApiError
|
||||
|
||||
cancelled: list[str] = []
|
||||
|
||||
class FakeClient:
|
||||
def cancel(self, run_id: str) -> dict[str, object]:
|
||||
if run_id == "gone":
|
||||
raise ApiError(404, "No such run")
|
||||
cancelled.append(run_id)
|
||||
return {"status": "ok" if run_id == "done" else "cancelled"}
|
||||
|
||||
@contextmanager
|
||||
def fake_client(_args):
|
||||
yield FakeClient()
|
||||
|
||||
monkeypatch.setattr(cli, "_client_for", fake_client)
|
||||
|
||||
args = _parser().parse_args(["cancel", "r1", "done"])
|
||||
assert cli.cmd_cancel(args) == 0
|
||||
assert cancelled == ["r1", "done"]
|
||||
# A run already past its last node says how it ended, not that it stopped.
|
||||
assert capsys.readouterr().out.splitlines() == ["r1 cancelled", "done ok"]
|
||||
|
||||
assert cli.cmd_cancel(_parser().parse_args(["cancel", "gone"])) == 1
|
||||
|
||||
|
||||
def test_a_failed_run_prints_what_each_node_said(monkeypatch, capsys) -> None:
|
||||
"""Otherwise the traceback is stored and nothing at a terminal shows it."""
|
||||
from contextlib import contextmanager
|
||||
|
||||
Reference in New Issue
Block a user