SDK: artifacts work outside a node, run --follow prints status and queue waits, fluksio cancel

save_artifact/load_artifact outside a node use the worker's own store code in
$FLUKSIO_ARTIFACT_DIR or <tmp>/fluksio-artifacts, so a node can be called by
hand. --follow prints status changes and each node's resource wait; the
placer's waiting snapshot now carries the run. fluksio cancel <run>...
This commit is contained in:
2026-10-28 08:07:35 +01:00
parent 4c2df681d3
commit 34243e6902
14 changed files with 274 additions and 27 deletions
+33 -11
View File
@@ -11,9 +11,10 @@ Inside a node, ``import fluksio`` is not this package at all: the worker
installs a reporter of its own under that name before any node code runs, so installs a reporter of its own under that name before any node code runs, so
what a node gets is :mod:`fluksio_worker.worker_main`'s ``emit`` and what a node gets is :mod:`fluksio_worker.worker_main`'s ``emit`` and
``save_artifact`` — and inert copies of the decorators, since a module the ``save_artifact`` — and inert copies of the decorators, since a module the
node imports may well declare them at its top. The stubs below stand in the node imports may well declare them at its top. Everywhere else the functions
same place everywhere else, and say so rather than failing as a missing below stand in, so a node called directly — in a test, in a notebook — behaves
attribute. like the real one: ``emit`` does nothing and the artifact helpers use a local
directory.
Nothing is imported from the rest of the package here. Every ``from fluksio.x Nothing is imported from the rest of the package here. Every ``from fluksio.x
import y`` in the engine passes through this module, and an import cycle or a import y`` in the engine passes through this module, and an import cycle or a
@@ -21,6 +22,7 @@ second of start-up cost would both begin here.
""" """
import logging import logging
import os
from importlib.metadata import PackageNotFoundError from importlib.metadata import PackageNotFoundError
from importlib.metadata import version as _version from importlib.metadata import version as _version
from typing import Any from typing import Any
@@ -46,11 +48,6 @@ __all__ = [
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_OUTSIDE = (
"fluksio.{name}() only works inside a node: the worker running it installs "
"the real one. There is nothing to {verb} out here."
)
def emit(**ports: Any) -> None: def emit(**ports: Any) -> None:
"""Publish values on a node's declared output ports, mid-run. """Publish values on a node's declared output ports, mid-run.
@@ -63,16 +60,41 @@ def emit(**ports: Any) -> None:
logger.debug("fluksio.emit(%s) outside a node", ", ".join(ports)) logger.debug("fluksio.emit(%s) outside a node", ", ".join(ports))
def _local_store() -> Any:
"""The worker's own artifact code, pointed at a directory of this machine.
``$FLUKSIO_ARTIFACT_DIR`` when it is set, which is how a worker finds the
engine's store, and a directory in the temporary location otherwise. The
references that come out are the ones a node gets inside the engine, so
what a node returns can be handed to the next one by hand.
"""
import tempfile
from pathlib import Path
from fluksio_worker.worker_main import ARTIFACT_DIR_ENV, _Reporter
os.environ.setdefault(
ARTIFACT_DIR_ENV, str(Path(tempfile.gettempdir()) / "fluksio-artifacts")
)
return _Reporter("fluksio")
def save_artifact( def save_artifact(
source: Any, source: Any,
name: str = "", name: str = "",
media_type: str = "application/octet-stream", media_type: str = "application/octet-stream",
volatile: bool = False, volatile: bool = False,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Put bytes in the artifact store and return a reference to them.""" """Put bytes or a file in the artifact store and return a reference to them.
raise RuntimeError(_OUTSIDE.format(name="save_artifact", verb="save to"))
Outside a node the store is a local directory, see :func:`_local_store`.
"""
reference: dict[str, Any] = _local_store().save_artifact(
source, name, media_type, volatile
)
return reference
def load_artifact(ref: dict[str, Any]) -> str: def load_artifact(ref: dict[str, Any]) -> str:
"""Fetch what an artifact reference points at, and return a path to it.""" """Fetch what an artifact reference points at, and return a path to it."""
raise RuntimeError(_OUTSIDE.format(name="load_artifact", verb="load from")) return str(_local_store().load_artifact(ref))
+2
View File
@@ -134,6 +134,8 @@ class ResourceLevel(BaseModel):
class WaitingNode(BaseModel): class WaitingNode(BaseModel):
node: str node: str
#: The run it belongs to, so a client can tell its own wait from another's.
run: str = ""
reason: str reason: str
seconds: float seconds: float
+3 -1
View File
@@ -79,6 +79,7 @@ class _Waiting:
node: str node: str
since: float since: float
reason: str reason: str
run: str = ""
class Placer: class Placer:
@@ -333,7 +334,7 @@ class Placer:
""" """
self._ticket += 1 self._ticket += 1
ticket = self._ticket ticket = self._ticket
self._waiting[ticket] = _Waiting(node=node, since=since, reason=reason) self._waiting[ticket] = _Waiting(node=node, since=since, reason=reason, run=run)
logger.info("%s is %s", node or "a node", reason) logger.info("%s is %s", node or "a node", reason)
if self.events is not None: if self.events is not None:
self.events.publish( self.events.publish(
@@ -407,6 +408,7 @@ class Placer:
"waiting": [ "waiting": [
{ {
"node": entry.node, "node": entry.node,
"run": entry.run,
"reason": entry.reason, "reason": entry.reason,
"seconds": round(time.monotonic() - entry.since, 1), "seconds": round(time.monotonic() - entry.since, 1),
} }
+62 -1
View File
@@ -585,8 +585,38 @@ def _sync_first(
_say(f"synced {', '.join(r.flow for r in changed)}") _say(f"synced {', '.join(r.flow for r in changed)}")
def _waits(client: Client, run_id: str) -> list[tuple[str, str]]:
"""What this run's nodes are queued for on a machine, as (node, reason).
A run that says nothing for an hour is either hung or waiting its turn, and
this is the half of that answer a metric cannot give. Best effort: an
engine that does not account for resources, or is too old to say whose a
wait is, simply has nothing to tell.
"""
try:
waiting = client.resources().get("waiting") or []
except (httpx.HTTPError, ApiError):
return []
return [
(str(entry.get("node", "")), str(entry.get("reason", "")))
for entry in waiting
if entry.get("run") == run_id
]
def _state(handle: RunHandle) -> tuple[str, str]:
"""Its status, and why — the reason only while it is true.
The engine leaves "waiting for a worker" on the row after the run starts.
"""
return handle.status, handle.status_reason if handle.status == "queued" else ""
def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None: def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
"""Print a run's numbers as they arrive, until it is over. """Print what a run does as it does it, until it is over.
Its numbers, a change of status, and what a queued node is waiting for, so
a run that reports nothing is not a silent one.
Polled rather than pushed: the engine writes a metric down when it is Polled rather than pushed: the engine writes a metric down when it is
reported, so asking once a second draws the same curve a socket would reported, so asking once a second draws the same curve a socket would
@@ -594,6 +624,8 @@ def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
before the numbers, so the last batch is never the one that gets missed. before the numbers, so the last batch is never the one that gets missed.
""" """
seen: set[tuple[str, int]] = set() seen: set[tuple[str, int]] = set()
told: set[tuple[str, str]] = set()
state = _state(handle)
failures = 0 failures = 0
while True: while True:
try: try:
@@ -610,6 +642,16 @@ def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
time.sleep(poll) time.sleep(poll)
continue continue
failures = 0 failures = 0
if not done:
now = _state(handle)
if now != state:
state = now
_say(f" {_status(now[0])}" + (f" {now[1]}" if now[1] else ""))
if handle.status == "running":
for wait in _waits(client, handle.id):
if wait not in told:
told.add(wait)
_say(f" {wait[0]} {wait[1]}")
for point in points: for point in points:
mark = (str(point.get("name", "")), int(point.get("step", -1))) mark = (str(point.get("name", "")), int(point.get("step", -1)))
if mark in seen: if mark in seen:
@@ -1114,6 +1156,20 @@ def cmd_retry(args: argparse.Namespace) -> int:
return 0 return 0
def cmd_cancel(args: argparse.Namespace) -> int:
"""Stop runs by id, which `run` without `--wait` leaves no other way to do."""
try:
with _client_for(args) as client:
for run_id in args.run_id:
made = client.cancel(run_id)
_say(f"{run_id} {_status(str(made.get('status') or 'cancelled'))}")
except (SyncError, ApiError) as exc:
return _fail(str(exc))
except httpx.HTTPError as exc:
return _unreachable(exc)
return 0
def _artifacts(client: Client, args: argparse.Namespace) -> int: def _artifacts(client: Client, args: argparse.Namespace) -> int:
"""List a run's files, or write one of them here.""" """List a run's files, or write one of them here."""
handle = RunHandle(client, args.run_id, client.run(args.run_id)) handle = RunHandle(client, args.run_id, client.run(args.run_id))
@@ -1524,6 +1580,11 @@ def add_parsers(subparsers: Any) -> None:
with_engine(parser) with_engine(parser)
parser.set_defaults(func=cmd_retry) parser.set_defaults(func=cmd_retry)
parser = subparsers.add_parser("cancel", help="stop runs that are still going")
parser.add_argument("run_id", nargs="+", help="the runs to stop")
with_engine(parser)
parser.set_defaults(func=cmd_cancel)
parser = subparsers.add_parser( parser = subparsers.add_parser(
"artifacts", help="the files a run produced; name one to download it" "artifacts", help="the files a run produced; name one to download it"
) )
+5
View File
@@ -577,6 +577,11 @@ class RunHandle:
def status(self) -> str: def status(self) -> str:
return str(self._row.get("status", "")) return str(self._row.get("status", ""))
@property
def status_reason(self) -> str:
"""Why it is in that status, such as what a queued run is waiting for."""
return str(self._row.get("status_reason") or "")
@property @property
def done(self) -> bool: def done(self) -> bool:
return self.status in DONE return self.status in DONE
+3 -1
View File
@@ -176,7 +176,7 @@ def test_a_node_waits_for_a_busy_machine_and_says_so(loop):
started = threading.Event() started = threading.Event()
def second() -> None: def second() -> None:
with placer.claim(Resources(cpus=2), node="study.b"): with placer.claim(Resources(cpus=2), node="study.b", run="r2"):
started.set() started.set()
with placer.claim(Resources(cpus=2), node="study.a"): with placer.claim(Resources(cpus=2), node="study.a"):
@@ -185,6 +185,8 @@ def test_a_node_waits_for_a_busy_machine_and_says_so(loop):
# Long enough to have run if nothing was holding it back. # Long enough to have run if nothing was holding it back.
assert not started.wait(0.3) assert not started.wait(0.3)
assert placer.snapshot()["waiting"][0]["node"] == "study.b" assert placer.snapshot()["waiting"][0]["node"] == "study.b"
# Whose wait it is, so a client following one run reads only its own.
assert placer.snapshot()["waiting"][0]["run"] == "r2"
assert events[0]["type"] == "node_queued" assert events[0]["type"] == "node_queued"
assert "2 cpu(s)" in events[0]["detail"] assert "2 cpu(s)" in events[0]["detail"]
+37
View File
@@ -1,5 +1,7 @@
"""What the decorators declare, and what the engine would be given.""" """What the decorators declare, and what the engine would be given."""
from pathlib import Path
import pytest import pytest
from fluksio.flow.schemas import FlowDef from fluksio.flow.schemas import FlowDef
@@ -396,3 +398,38 @@ def test_a_flow_records_the_module_that_declared_it():
# A repository the file is not in has no path to give it. # A repository the file is not in has no path to give it.
elsewhere = flow.document({"kind": "python", "repo": "/elsewhere"})["origin"] elsewhere = flow.document({"kind": "python", "repo": "/elsewhere"})["origin"]
assert (elsewhere["declared_in"], elsewhere["module"]) == ("", "") assert (elsewhere["declared_in"], elsewhere["module"]) == ("", "")
def test_artifacts_round_trip_outside_a_node(tmp_path, monkeypatch):
"""A node that returns an artifact can be smoke-tested by calling it."""
import fluksio
monkeypatch.setenv("FLUKSIO_ARTIFACT_DIR", str(tmp_path))
ref = fluksio.save_artifact(b"weights", "w.bin", "application/x-test")
assert ref["name"] == "w.bin"
assert ref["size"] == 7
assert ref["media_type"] == "application/x-test"
assert ref["digest"].startswith("sha256:")
assert Path(fluksio.load_artifact(ref)).read_bytes() == b"weights"
assert str(tmp_path) in fluksio.load_artifact(ref)
source = tmp_path / "plot.png"
source.write_bytes(b"png")
assert fluksio.save_artifact(source)["name"] == "plot.png"
def test_the_local_artifact_store_defaults_to_the_temporary_directory(monkeypatch):
import tempfile
import fluksio
# Set and removed again, so the variable the default writes is undone too.
monkeypatch.setenv("FLUKSIO_ARTIFACT_DIR", "")
monkeypatch.delenv("FLUKSIO_ARTIFACT_DIR")
ref = fluksio.save_artifact(b"x" * 3)
assert fluksio.load_artifact(ref).startswith(
str(Path(tempfile.gettempdir()) / "fluksio-artifacts")
)
+97
View File
@@ -309,6 +309,103 @@ def test_ctrl_c_while_waiting_cancels_the_run(monkeypatch) -> None:
assert cancelled == ["run-1"] assert cancelled == ["run-1"]
def test_follow_says_what_a_run_that_reports_nothing_is_doing(capsys) -> None:
"""Queued for a worker, then waiting for a card: both are said once."""
from fluksio.sdk import cli
from fluksio.sdk.client import ApiError
held = "Waiting for a worker with 1 gpu(s)"
rows = iter(
[
{"status": "queued", "status_reason": held},
{"status": "queued", "status_reason": held},
# The engine leaves the reason on the row once the run starts.
{"status": "running", "status_reason": held},
{"status": "running", "status_reason": held},
{"status": "running", "status_reason": held},
{"status": "ok", "status_reason": ""},
]
)
class FakeHandle:
id = "r1"
status = "queued"
status_reason = ""
@property
def done(self) -> bool:
return self.status == "ok"
def refresh(self) -> "FakeHandle":
row = next(rows)
self.status, self.status_reason = row["status"], row["status_reason"]
return self
asked: list[int] = []
class FakeClient:
def metrics(self, run_id: str) -> list:
return []
def resources(self) -> dict:
asked.append(1)
if len(asked) == 3:
raise ApiError(503, "Resources are not accounted here")
return {
"waiting": [
{
"node": "train.fit",
"run": "r1",
"reason": "waiting for 1 gpu(s)",
},
{
"node": "train.fit",
"run": "r2",
"reason": "waiting for 9 gpu(s)",
},
]
}
cli._follow(FakeClient(), FakeHandle(), poll=0)
assert capsys.readouterr().out.splitlines() == [
f" queued {held}",
" running",
" train.fit waiting for 1 gpu(s)",
]
def test_cancel_stops_the_runs_it_is_given(monkeypatch, capsys) -> None:
from contextlib import contextmanager
from fluksio.cli import _parser
from fluksio.sdk import cli
from fluksio.sdk.client import ApiError
cancelled: list[str] = []
class FakeClient:
def cancel(self, run_id: str) -> dict[str, object]:
if run_id == "gone":
raise ApiError(404, "No such run")
cancelled.append(run_id)
return {"status": "ok" if run_id == "done" else "cancelled"}
@contextmanager
def fake_client(_args):
yield FakeClient()
monkeypatch.setattr(cli, "_client_for", fake_client)
args = _parser().parse_args(["cancel", "r1", "done"])
assert cli.cmd_cancel(args) == 0
assert cancelled == ["r1", "done"]
# A run already past its last node says how it ended, not that it stopped.
assert capsys.readouterr().out.splitlines() == ["r1 cancelled", "done ok"]
assert cli.cmd_cancel(_parser().parse_args(["cancel", "gone"])) == 1
def test_a_failed_run_prints_what_each_node_said(monkeypatch, capsys) -> None: def test_a_failed_run_prints_what_each_node_said(monkeypatch, capsys) -> None:
"""Otherwise the traceback is stored and nothing at a terminal shows it.""" """Otherwise the traceback is stored and nothing at a terminal shows it."""
from contextlib import contextmanager from contextlib import contextmanager
+12 -2
View File
@@ -206,7 +206,7 @@ warnings. Use `--sync PATH` to choose a target explicitly.
| Flag | Purpose | | Flag | Purpose |
|---|---| |---|---|
| `--wait` | Wait for completion; exit non-zero if the run fails | | `--wait` | Wait for completion; exit non-zero if the run fails |
| `--follow` | Wait and print emitted metrics | | `--follow` | Wait and print emitted metrics, status changes and what a queued node waits for |
| `--timeout SECONDS` | Stop waiting after this long; leave the run active | | `--timeout SECONDS` | Stop waiting after this long; leave the run active |
| `--seed N` | Record the seed and fill a declared input named `seed` | | `--seed N` | Record the seed and fill a declared input named `seed` |
| `--defaults` | Use declared input defaults without prompting | | `--defaults` | Use declared input defaults without prompting |
@@ -217,7 +217,8 @@ warnings. Use `--sync PATH` to choose a target explicitly.
With no parameters at a terminal, the command prompts for each input. With no parameters at a terminal, the command prompts for each input.
Press Enter to keep its default. Scripted commands do not prompt. Press Enter to keep its default. Scripted commands do not prompt.
Ctrl-C while waiting cancels the run and exits with status 130. Ctrl-C while waiting cancels the run and exits with status 130. A run started
without `--wait` is stopped with [`fluksio cancel`](#fluksio-cancel).
A seed does not initialize your libraries' random number generators. Declare A seed does not initialize your libraries' random number generators. Declare
a `seed` input and pass it to your code if it controls randomness. a `seed` input and pass it to your code if it controls randomness.
@@ -256,6 +257,15 @@ fluksio runs --flow train --limit 10
Lists recent runs, newest first. Omit `--flow` for all flows. Lists recent runs, newest first. Omit `--flow` for all flows.
`--local` reads the local instance. `--local` reads the local instance.
### `fluksio cancel`
```sh
fluksio cancel <run-id> [<run-id> ...]
```
Stops the runs and their active node processes. A run already past its last
node is left as it finished, and its status is printed.
### `fluksio retry` ### `fluksio retry`
```sh ```sh
+4 -2
View File
@@ -110,8 +110,10 @@ def process():
``` ```
Another node can read it with `fluksio.load_artifact(report)`, which returns Another node can read it with `fluksio.load_artifact(report)`, which returns
a local file path. These two helpers require a running node; for tests, a local file path. Called directly, outside a node, the two helpers store the
keep the underlying calculation in ordinary Python functions. bytes in `$FLUKSIO_ARTIFACT_DIR` or, when that is unset, in `fluksio-artifacts`
under the temporary directory. The reference has the same shape as inside the
engine, so a test can pass what one node returns to the next.
Artifacts are stored by their SHA-256 digest, so identical bytes share storage. Artifacts are stored by their SHA-256 digest, so identical bytes share storage.
Use [`fluksio artifacts`](cli.md#fluksio-artifacts) to list and download a Use [`fluksio artifacts`](cli.md#fluksio-artifacts) to list and download a
+6 -4
View File
@@ -113,7 +113,8 @@ fluksio run train --lr 0.1 --epochs 20 --follow
`sync` uploads the flow and generates node code that imports your functions. `sync` uploads the flow and generates node code that imports your functions.
Keep the project files available to the engine. Later `run` commands sync Keep the project files available to the engine. Later `run` commands sync
edits before submitting; `--follow` waits and prints metrics as they arrive. edits before submitting. `--follow` waits and prints metrics as they arrive,
along with status changes and what a queued node is waiting for.
This run should finish with `final_loss` around `0.1353` and `score` around This run should finish with `final_loss` around `0.1353` and `score` around
`0.8647`. `0.8647`.
@@ -155,9 +156,10 @@ Pass either list to `pandas.DataFrame` if you use pandas.
## Bring your own experiment ## Bring your own experiment
Replace the toy calculation in `fit` with your training code. Keep ordinary Replace the toy calculation in `fit` with your training code. You can call a
Python helpers separate from artifact storage so you can still test them node directly to test it without an engine: `save_artifact` and
without an engine: `save_artifact` and `load_artifact` work only inside nodes. `load_artifact` then use a folder under the temporary directory, and `emit`
does nothing.
The public repository's `examples/myresearch/` package shows this pattern The public repository's `examples/myresearch/` package shows this pattern
across several files. across several files.
+4 -5
View File
@@ -2,9 +2,8 @@
Each node is a thin wrapper over a plain function: `prepare` stores what Each node is a thin wrapper over a plain function: `prepare` stores what
`make_rows` made, `fit` yields what `train_curve` computed, `evaluate` reports `make_rows` made, `fit` yields what `train_curve` computed, `evaluate` reports
what `score` worked out. That split is the point — `fluksio.save_artifact` and what `score` worked out. That split is the point — the arithmetic stays plain
`load_artifact` only work *inside* a node, so a body calling them cannot be run Python, so `python -m myresearch.pipeline` runs it with no engine anywhere,
by hand, while `python -m myresearch.pipeline` runs the arithmetic beside them while the wrappers own the artifact storage. `fluksio sync examples/myresearch`
with no engine anywhere. `fluksio sync examples/myresearch` is what turns the is what turns the wrappers into a flow.
wrappers into a flow.
""" """
+5
View File
@@ -5002,6 +5002,11 @@ export const WaitingNodeSchema = {
type: 'string', type: 'string',
title: 'Node' title: 'Node'
}, },
run: {
type: 'string',
title: 'Run',
default: ''
},
reason: { reason: {
type: 'string', type: 'string',
title: 'Reason' title: 'Reason'
+1
View File
@@ -1580,6 +1580,7 @@ export type ValidationResult = {
export type WaitingNode = { export type WaitingNode = {
node: string; node: string;
run?: string;
reason: string; reason: string;
seconds: number; seconds: number;
}; };