SDK: artifacts work outside a node, run --follow prints status and queue waits, fluksio cancel

save_artifact/load_artifact outside a node use the worker's own store code in
$FLUKSIO_ARTIFACT_DIR or <tmp>/fluksio-artifacts, so a node can be called by
hand. --follow prints status changes and each node's resource wait; the
placer's waiting snapshot now carries the run. fluksio cancel <run>...
This commit is contained in:
2026-10-28 08:07:35 +01:00
parent 4c2df681d3
commit 34243e6902
14 changed files with 274 additions and 27 deletions
+62 -1
View File
@@ -585,8 +585,38 @@ def _sync_first(
_say(f"synced {', '.join(r.flow for r in changed)}")
def _waits(client: Client, run_id: str) -> list[tuple[str, str]]:
"""What this run's nodes are queued for on a machine, as (node, reason).
A run that says nothing for an hour is either hung or waiting its turn, and
this is the half of that answer a metric cannot give. Best effort: an
engine that does not account for resources, or is too old to say whose a
wait is, simply has nothing to tell.
"""
try:
waiting = client.resources().get("waiting") or []
except (httpx.HTTPError, ApiError):
return []
return [
(str(entry.get("node", "")), str(entry.get("reason", "")))
for entry in waiting
if entry.get("run") == run_id
]
def _state(handle: RunHandle) -> tuple[str, str]:
"""Its status, and why — the reason only while it is true.
The engine leaves "waiting for a worker" on the row after the run starts.
"""
return handle.status, handle.status_reason if handle.status == "queued" else ""
def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
"""Print a run's numbers as they arrive, until it is over.
"""Print what a run does as it does it, until it is over.
Its numbers, a change of status, and what a queued node is waiting for, so
a run that reports nothing is not a silent one.
Polled rather than pushed: the engine writes a metric down when it is
reported, so asking once a second draws the same curve a socket would
@@ -594,6 +624,8 @@ def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
before the numbers, so the last batch is never the one that gets missed.
"""
seen: set[tuple[str, int]] = set()
told: set[tuple[str, str]] = set()
state = _state(handle)
failures = 0
while True:
try:
@@ -610,6 +642,16 @@ def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
time.sleep(poll)
continue
failures = 0
if not done:
now = _state(handle)
if now != state:
state = now
_say(f" {_status(now[0])}" + (f" {now[1]}" if now[1] else ""))
if handle.status == "running":
for wait in _waits(client, handle.id):
if wait not in told:
told.add(wait)
_say(f" {wait[0]} {wait[1]}")
for point in points:
mark = (str(point.get("name", "")), int(point.get("step", -1)))
if mark in seen:
@@ -1114,6 +1156,20 @@ def cmd_retry(args: argparse.Namespace) -> int:
return 0
def cmd_cancel(args: argparse.Namespace) -> int:
"""Stop runs by id, which `run` without `--wait` leaves no other way to do."""
try:
with _client_for(args) as client:
for run_id in args.run_id:
made = client.cancel(run_id)
_say(f"{run_id} {_status(str(made.get('status') or 'cancelled'))}")
except (SyncError, ApiError) as exc:
return _fail(str(exc))
except httpx.HTTPError as exc:
return _unreachable(exc)
return 0
def _artifacts(client: Client, args: argparse.Namespace) -> int:
"""List a run's files, or write one of them here."""
handle = RunHandle(client, args.run_id, client.run(args.run_id))
@@ -1524,6 +1580,11 @@ def add_parsers(subparsers: Any) -> None:
with_engine(parser)
parser.set_defaults(func=cmd_retry)
parser = subparsers.add_parser("cancel", help="stop runs that are still going")
parser.add_argument("run_id", nargs="+", help="the runs to stop")
with_engine(parser)
parser.set_defaults(func=cmd_cancel)
parser = subparsers.add_parser(
"artifacts", help="the files a run produced; name one to download it"
)