stroblmeandClaude Opus 5
8a94bf10d7
Docs / docs (push) Successful in 27s
Playwright Tests / test-playwright (1, 2) (push) Failing after 17s
Playwright Tests / test-playwright (2, 2) (push) Failing after 12s
pre-commit / pre-commit (push) Failing after 1m59s
Test Backend / test-backend (push) Failing after 2m30s
Compose Smoke Test / test-compose (push) Failing after 13s
Playwright Tests / merge-reports (push) Failing after 2m19s
serve: refuse a second engine for one data directory whatever port it was asked for, using the pidfile and a token this directory signed. The check runs before the database is touched and before the credential is written, which is what left every later CLI call pointing at a dead port. The terminal dashboard is three tabs (Overview, Runs, Logs) with the toolbar following the focused pane, the engine's output goes to serve.log rather than down a pipe, and closing the screen stops both reader threads so the prompt comes back. It adopts a running engine on every start, so stop/start and restart work on one it did not start, and a stop waits for the process to be gone before the next start. Enrolment reports itself in the modal. enroll: a new claim code replaces the pairing instead of being refused. The code is redeemed before anything is written, mappings to a portal being left are cleared, and a running engine redials when the stored enrolment changes. runs: an engine re-queues the runs left `queued` by the one before it, and `fluksio retry <id>` / `retry --group <sweep>` submits an interrupted run again with the same inputs and group, recorded through Run.parent_id. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01U9BoNGq6V9MdRWAte7JBuC
1595 lines
58 KiB
Python
1595 lines
58 KiB
Python
"""The `fluksio sync`, `run`, `runs`, `sweep`, `status` and `login` commands.
|
|
|
|
Kept beside the SDK rather than in `fluksio.cli`: these are the client half of
|
|
the tool, and none of them needs the engine to be importable — `--local`, which
|
|
does, imports it inside the branch that asked for it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import getpass
|
|
import importlib
|
|
import itertools
|
|
import json
|
|
import pkgutil
|
|
import shutil
|
|
import sys
|
|
import time
|
|
from collections.abc import Callable, Iterable, Iterator
|
|
from contextlib import contextmanager, nullcontext
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from fluksio import __version__
|
|
from fluksio.sdk import FLOWS, Flow, SyncError
|
|
from fluksio.sdk.client import (
|
|
GLOBAL_DATA_DIR,
|
|
RETRIES,
|
|
WAIT_TOLERANCE,
|
|
ApiError,
|
|
Client,
|
|
RunHandle,
|
|
data_dir,
|
|
login,
|
|
origin_of,
|
|
repo_root,
|
|
sync,
|
|
)
|
|
|
|
__all__ = ["add_parsers", "discover"]
|
|
|
|
#: Statuses worth a colour, and the SGR code each gets.
|
|
_COLORS = {
|
|
"ok": "32",
|
|
"error": "31",
|
|
"cached": "36",
|
|
"cancelled": "33",
|
|
"abandoned": "31",
|
|
"running": "36",
|
|
"queued": "33",
|
|
}
|
|
|
|
|
|
def _say(message: str = "") -> None:
|
|
print(message)
|
|
|
|
|
|
def _fail(message: str) -> int:
|
|
print(f"fluksio: {message}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
#: What to try when nothing answered at all. By far the commonest reason is
|
|
#: that no engine is running, and the message said only that it was not.
|
|
NO_ENGINE = "Is one running? `fluksio serve` starts one."
|
|
|
|
|
|
def _unreachable(exc: Exception, note: str = NO_ENGINE) -> int:
|
|
"""The engine did not answer. Say so as a sentence, not a traceback."""
|
|
return _fail(
|
|
f"engine not answering ({type(exc).__name__}: {exc})"
|
|
+ (f". {note}" if note else "")
|
|
)
|
|
|
|
|
|
def _status(text: str, width: int = 0) -> str:
|
|
"""A status, coloured when a terminal is reading it.
|
|
|
|
Padded before it is coloured: the escape sequences are characters as far
|
|
as `str.format` is concerned, and a column that lines up in a pipe would
|
|
not line up on screen.
|
|
"""
|
|
body = f"{text:<{width}}" if width else text
|
|
code = _COLORS.get(text)
|
|
if not code or not sys.stdout.isatty():
|
|
return body
|
|
return f"\033[{code}m{body}\033[0m"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Discovery
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _package_of(directory: Path) -> tuple[str, str]:
|
|
"""The path root and dotted name of a package directory."""
|
|
parts = [directory.name]
|
|
parent = directory.parent
|
|
while (parent / "__init__.py").exists():
|
|
parts.append(parent.name)
|
|
parent = parent.parent
|
|
return str(parent), ".".join(reversed(parts))
|
|
|
|
|
|
def _module_of(path: Path) -> tuple[str, str]:
|
|
"""The path root and dotted name of a module file."""
|
|
parts = [path.stem]
|
|
directory = path.parent
|
|
while (directory / "__init__.py").exists():
|
|
parts.append(directory.name)
|
|
directory = directory.parent
|
|
return str(directory), ".".join(reversed(parts))
|
|
|
|
|
|
#: Directories a walk never goes into: none of them is a study, and some of
|
|
#: them are enormous.
|
|
_SKIP_DIRS = frozenset({"__pycache__", "node_modules", "site-packages"})
|
|
|
|
|
|
def _import(root: str, dotted: str, expect: Path | None = None) -> None:
|
|
if root not in sys.path:
|
|
sys.path.insert(0, root)
|
|
module = importlib.import_module(dotted)
|
|
if expect is None:
|
|
return
|
|
# Python keeps one module per name, so a second file importing under a
|
|
# name already taken is silently the first one — and the generated body
|
|
# imports by that name too, so a worker would run the wrong study's code.
|
|
actual = getattr(module, "__file__", "") or ""
|
|
if actual and Path(actual).resolve() != expect.resolve():
|
|
raise SyncError(
|
|
f"two files would both import as '{dotted}':\n"
|
|
f" {actual}\n {expect}\n"
|
|
"Python keeps one module per name, and a node's generated body "
|
|
"imports by that name, so the second would run the first's code. "
|
|
"Sync the directory they are both under — each then imports as "
|
|
"'<dir>.<file>' — or rename one of the files."
|
|
)
|
|
|
|
|
|
def _walkable(entry: Path) -> bool:
|
|
"""Whether a walk should look inside this directory at all."""
|
|
return (
|
|
entry.is_dir()
|
|
and not entry.name.startswith(".")
|
|
and entry.name not in _SKIP_DIRS
|
|
and not (entry / "pyvenv.cfg").exists()
|
|
)
|
|
|
|
|
|
def _below(root: Path) -> Iterator[Path]:
|
|
"""Every module and package under a plain directory, however deep.
|
|
|
|
One directory per study — `dev/s1_baseline/study.py` — is a layout people
|
|
have, and naming each of them on the command line is bookkeeping the tool
|
|
can do. A package is yielded whole and not descended into: its own walk
|
|
imports its submodules under the right names.
|
|
"""
|
|
for entry in sorted(root.iterdir()):
|
|
if entry.is_file() and entry.suffix == ".py":
|
|
yield entry
|
|
elif _walkable(entry):
|
|
if (entry / "__init__.py").exists():
|
|
yield entry
|
|
else:
|
|
yield from _below(entry)
|
|
|
|
|
|
def _import_package(directory: Path) -> None:
|
|
"""A package and every module in it, by their dotted names."""
|
|
root, dotted = _package_of(directory)
|
|
_import(root, dotted, directory / "__init__.py")
|
|
for info in pkgutil.walk_packages(sys.modules[dotted].__path__, f"{dotted}."):
|
|
importlib.import_module(info.name)
|
|
|
|
|
|
def discover(targets: list[str], keep_going: bool = False) -> list[Flow]:
|
|
"""Import what was named and hand back the flows it declared.
|
|
|
|
Imported by dotted name with its root on the path, never from a file
|
|
location: the generated node bodies import the same way, and a module
|
|
loaded under a different name would generate an import that does not
|
|
resolve.
|
|
|
|
``keep_going`` warns about a module that will not import instead of
|
|
stopping, which is what a *run* wants: a study half-way through an edit
|
|
two directories away is not a reason to refuse to run this one. A
|
|
collision between two module names is never skipped — it would produce a
|
|
node body that imports the wrong file.
|
|
"""
|
|
for target in targets:
|
|
path = Path(target)
|
|
if not path.exists():
|
|
_import(str(Path.cwd()), target)
|
|
continue
|
|
path = path.resolve()
|
|
if path.is_file():
|
|
_import(*_module_of(path), path)
|
|
continue
|
|
if (path / "__init__.py").exists():
|
|
_import_package(path)
|
|
continue
|
|
# A plain directory — a repository root, or a directory of studies.
|
|
# Its own modules, the packages inside it, and the same again all the
|
|
# way down, so one folder per study needs no naming.
|
|
for entry in _below(path):
|
|
try:
|
|
if entry.is_dir():
|
|
_import_package(entry)
|
|
else:
|
|
# Named for where it sits under the directory being synced,
|
|
# so `dev/s1/study.py` imports as `s1.study` and one
|
|
# `study.py` per study collides with nothing. No
|
|
# `__init__.py` needed: the directories in between are
|
|
# namespace packages. A file at the root keeps its bare
|
|
# name, which is what it has always had.
|
|
dotted = ".".join(entry.relative_to(path).with_suffix("").parts)
|
|
_import(str(path), dotted, entry)
|
|
except SyncError:
|
|
# A name collision is never somebody else's problem: it would
|
|
# put the wrong file behind a node.
|
|
raise
|
|
except Exception as exc:
|
|
# Importing a module runs it, so this is whatever the study
|
|
# does at its top level, and a walk meets every study.
|
|
if keep_going:
|
|
_say(f"warning: skipped {entry} — {type(exc).__name__}: {exc}")
|
|
continue
|
|
raise SyncError(
|
|
f"{entry} failed to import — {type(exc).__name__}: {exc}"
|
|
) from exc
|
|
return list(FLOWS.values())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Which engine
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@contextmanager
|
|
def _engine_client() -> Iterator[Client]:
|
|
"""The engine itself, in this process, behind the ordinary client.
|
|
|
|
Everything `fluksio serve` does apart from listening on a socket: the same
|
|
data directory, the same database, the same admin, the same lifespan. The
|
|
app is driven through its ASGI interface, so a run costs what it costs on
|
|
a served engine and lands in the same history — which is what makes the
|
|
stage cache carry across the two.
|
|
|
|
The engine only exists for the length of the command, so nothing here is
|
|
written back as a login: a stored token belongs to whichever engine
|
|
`fluksio login` was pointed at.
|
|
"""
|
|
# Imported here, not at module scope: everything else in this file is the
|
|
# client half and must keep working with no engine installed.
|
|
import logging
|
|
|
|
from fluksio.cli import _data_dir, _prepare, _print_new_admin
|
|
|
|
directory = _data_dir(None)
|
|
# Settings are read when the app is imported, so this comes first.
|
|
_prepare(directory)
|
|
# Nothing here goes over a network, so httpx logging each call as a
|
|
# request to "testserver" is noise that also happens to be untrue.
|
|
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
|
|
from datetime import timedelta
|
|
|
|
from fastapi.testclient import TestClient
|
|
from sqlmodel import Session
|
|
|
|
from fluksio.core import security
|
|
from fluksio.core.bootstrap import ensure_superuser
|
|
from fluksio.core.db import engine
|
|
|
|
with Session(engine) as session:
|
|
admin, generated = ensure_superuser(session)
|
|
admin_id = admin.id
|
|
if generated:
|
|
_print_new_admin(admin.email, generated)
|
|
token = security.create_access_token(admin_id, expires_delta=timedelta(hours=12))
|
|
|
|
from fluksio.main import app
|
|
|
|
# Entering the client is what runs the lifespan: the worker pool, the
|
|
# controller and the run service all start here and stop on the way out.
|
|
with TestClient(app) as http:
|
|
yield Client(http=http, token=token)
|
|
|
|
|
|
@contextmanager
|
|
def _client_for(args: argparse.Namespace, retries: int = RETRIES) -> Iterator[Client]:
|
|
"""The engine this command talks to: one running somewhere, or this one.
|
|
|
|
``retries=0`` is what a read passes: an engine that is not there otherwise
|
|
takes the backoff — seconds — to say so, and nothing in a read is worth
|
|
waiting out a restart for. A command that submits keeps them.
|
|
"""
|
|
if getattr(args, "local", False):
|
|
with _engine_client() as client:
|
|
yield client
|
|
else:
|
|
yield Client(url=args.url, token=args.token, retries=retries)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Commands
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def cmd_login(args: argparse.Namespace) -> int:
|
|
"""Only needed for an engine somewhere else — `serve` signs you in here."""
|
|
email = args.email or input("Email: ")
|
|
password = args.password or getpass.getpass("Password: ")
|
|
directory = GLOBAL_DATA_DIR.expanduser() if args.shared else data_dir()
|
|
try:
|
|
path = login(args.url, email, password, directory=directory)
|
|
except ApiError as exc:
|
|
return _fail(f"could not log in: {exc.detail}")
|
|
_say(f"Logged in to {args.url}; the token is in {path}.")
|
|
return 0
|
|
|
|
|
|
def cmd_sync(args: argparse.Namespace) -> int:
|
|
targets = args.targets or ["."]
|
|
try:
|
|
flows = discover(targets)
|
|
except (ImportError, SyncError) as exc:
|
|
return _fail(str(exc))
|
|
if not flows:
|
|
return _fail(
|
|
f"no flows declared in {', '.join(targets)} — a flow is a `Flow(...)` "
|
|
"at module level. Name the package if it is somewhere else: "
|
|
"`fluksio sync src/myresearch`."
|
|
)
|
|
|
|
repo = repo_root(targets[0])
|
|
origin = origin_of(repo)
|
|
if origin["dirty"]:
|
|
_say(f"warning: {repo} has uncommitted changes, so the stamp says -dirty")
|
|
if not origin["commit"]:
|
|
_say(f"warning: {repo} is not a git repository, so runs cannot name a commit")
|
|
|
|
if args.dry_run:
|
|
for target in flows:
|
|
_say(f"=== flow {target.name}")
|
|
_say(json.dumps(target.document(origin), indent=2))
|
|
for node_id, code in target.shims().items():
|
|
_say(f"=== {target.name}.{node_id}")
|
|
_say(code)
|
|
return 0
|
|
|
|
try:
|
|
client = Client(url=args.url, token=args.token)
|
|
reports = sync(
|
|
flows,
|
|
client,
|
|
origin=origin,
|
|
publish=not args.no_publish,
|
|
force=args.force,
|
|
)
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc, "Nothing was published; sync again when it is back.")
|
|
|
|
for report in reports:
|
|
if report.unchanged:
|
|
_say(f" {report.flow}: unchanged")
|
|
continue
|
|
what = "created" if report.created else "updated"
|
|
detail = ", ".join(report.changed)
|
|
# Nothing to publish is not the same as left as a draft, and saying
|
|
# the second when it was the first reads as work not finished.
|
|
state = "published" if report.published else "draft" if report.drafted else ""
|
|
_say(f" {report.flow}: {what} ({detail})" + (f" — {state}" if state else ""))
|
|
if any(report.forgot_code for report in reports):
|
|
engine = _engine_version(client)
|
|
_say(
|
|
" note: this engine did not store what each node's code reaches, "
|
|
"so its cache is still keyed on the whole repository. It is "
|
|
f"{engine or 'older'} and this client is {__version__} — "
|
|
"`pip install -U fluksio` there."
|
|
)
|
|
stamp = origin["commit"][:7] + ("-dirty" if origin["dirty"] else "")
|
|
_say(f"Stamped with {stamp or 'no commit'} from {repo}.")
|
|
return 0
|
|
|
|
|
|
def _coerce(value: str, dtype: str) -> Any:
|
|
if value.startswith("@run:") or value.startswith("sha256:"):
|
|
# A run's output, named rather than typed out. Sent as it stands: the
|
|
# engine turns it into the value itself, whatever type that is. Passing
|
|
# the whole thing as JSON still works, and is what a script that
|
|
# already has it in hand would do.
|
|
return value
|
|
if dtype == "int":
|
|
return int(value)
|
|
if dtype == "float":
|
|
return float(value)
|
|
if dtype == "bool":
|
|
return value.lower() in ("true", "1", "yes", "on")
|
|
if dtype == "str":
|
|
return value
|
|
return json.loads(value)
|
|
|
|
|
|
def _input_types(definition: dict[str, Any]) -> dict[str, str]:
|
|
"""What each of a flow's inputs is declared to be."""
|
|
return {
|
|
str(entry["spec"]["name"]): str(entry["spec"].get("dtype", "float"))
|
|
for entry in definition.get("inputs") or []
|
|
}
|
|
|
|
|
|
def _ask_params(definition: dict[str, Any]) -> dict[str, Any]:
|
|
"""The run dialog, in a terminal: one line per declared input.
|
|
|
|
An empty answer leaves the input out, which is what keeps its declared
|
|
value — the same thing the browser's dialog does with a field nobody
|
|
filled in. So pressing Enter through the lot runs the defaults.
|
|
"""
|
|
params: dict[str, Any] = {}
|
|
for entry in definition.get("inputs") or []:
|
|
spec = entry.get("spec") or {}
|
|
name = str(spec.get("name", ""))
|
|
if not name:
|
|
continue
|
|
dtype = str(spec.get("dtype", "float"))
|
|
initial = entry.get("initial")
|
|
shown = "" if initial is None else json.dumps(initial)
|
|
prompt = f"{name} ({dtype})" + (f" [{shown}]" if shown else "") + ": "
|
|
answer = input(prompt).strip()
|
|
if not answer:
|
|
continue
|
|
try:
|
|
params[name] = _coerce(answer, dtype)
|
|
except ValueError as exc:
|
|
raise SyncError(f"'{name}' takes {dtype}: {exc}") from exc
|
|
return params
|
|
|
|
|
|
def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
|
|
"""Turn `--lr 0.05` into a typed parameter, using the flow's own inputs."""
|
|
types = _input_types(definition)
|
|
# Collected as written and typed afterwards, so a name this flow does not
|
|
# have is refused by name rather than by whatever its value failed to parse
|
|
# as: `--param lr=0.002` is a sweep's spelling, and said so it reads as an
|
|
# input called `param` holding unparseable json.
|
|
raw: dict[str, str | bool] = {}
|
|
pending: str | None = None
|
|
for token in rest:
|
|
if token.startswith("--"):
|
|
if pending is not None:
|
|
# A flag with no value is a flag: `--resume` means true.
|
|
raw[pending] = True
|
|
name, sep, value = token[2:].partition("=")
|
|
# Only the name is spelled with dashes; a value may hold one, and
|
|
# `--lr=1e-4` is the case that says so.
|
|
pending = name.replace("-", "_")
|
|
if sep:
|
|
raw[pending] = value
|
|
pending = None
|
|
continue
|
|
if pending is None:
|
|
raise SyncError(f"unexpected argument '{token}'")
|
|
raw[pending] = token
|
|
pending = None
|
|
if pending is not None:
|
|
raw[pending] = True
|
|
unknown = sorted(set(raw) - set(types))
|
|
if unknown:
|
|
hint = ""
|
|
if unknown[0] == "param":
|
|
hint = (
|
|
" — one value is `--<name> <value>`; several is a sweep: "
|
|
"`fluksio sweep --param name=v1,v2`"
|
|
)
|
|
raise SyncError(
|
|
f"'{unknown[0]}' is not an input of this flow (it takes "
|
|
f"{', '.join(sorted(types)) or 'none'}){hint}"
|
|
)
|
|
params: dict[str, Any] = {}
|
|
for key, written in raw.items():
|
|
if isinstance(written, bool):
|
|
params[key] = written
|
|
continue
|
|
try:
|
|
params[key] = _coerce(written, types[key])
|
|
except ValueError as exc:
|
|
raise SyncError(f"'{key}' takes {types[key]}: {exc}") from exc
|
|
return params
|
|
|
|
|
|
def _sync_first(client: Client, targets: list[str] | None = None) -> None:
|
|
"""Upload what the working directory declares, before running it.
|
|
|
|
The reason a run exists is usually the edit that came before it, and
|
|
remembering to sync is remembering to do the thing the computer could have
|
|
done. So `run` syncs by default — including the worker refresh, which is
|
|
what makes an edit to your own package take effect at all.
|
|
|
|
The whole directory, downwards: from a repository root the flow is usually
|
|
in a study folder below, and the alternative is syncing by hand and then
|
|
running with `--no-sync`. The upload is already a no-op for a flow nothing
|
|
changed in, so the cost is importing the other studies — which `--sync`
|
|
narrows when that is not free.
|
|
|
|
A directory that declares nothing is not an error: a flow drawn on the
|
|
canvas is run the same way, and has nothing to upload.
|
|
"""
|
|
try:
|
|
flows = discover(targets or ["."], keep_going=True)
|
|
except (ImportError, SyncError) as exc:
|
|
# Do not fail a run for a module the run may not even need.
|
|
_say(f"warning: nothing synced — {exc}")
|
|
return
|
|
if not flows:
|
|
return
|
|
repo = repo_root((targets or ["."])[0])
|
|
reports = sync(flows, client, origin=origin_of(repo))
|
|
changed = [r for r in reports if not r.unchanged]
|
|
if changed:
|
|
_say(f"synced {', '.join(r.flow for r in changed)}")
|
|
|
|
|
|
def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
|
|
"""Print a run's numbers as they arrive, until it is over.
|
|
|
|
Polled rather than pushed: the engine writes a metric down when it is
|
|
reported, so asking once a second draws the same curve a socket would
|
|
have, without either side holding a connection open. The status is read
|
|
before the numbers, so the last batch is never the one that gets missed.
|
|
"""
|
|
seen: set[tuple[str, int]] = set()
|
|
failures = 0
|
|
while True:
|
|
try:
|
|
done = handle.refresh().done
|
|
points = client.metrics(handle.id)
|
|
except (httpx.HTTPError, ApiError) as exc:
|
|
# Following an eight-hour run must not end because one poll of it
|
|
# did. The run is still going; only this side lost sight of it.
|
|
if isinstance(exc, ApiError) and exc.status < 500:
|
|
raise
|
|
failures += 1
|
|
if failures >= WAIT_TOLERANCE:
|
|
raise
|
|
time.sleep(poll)
|
|
continue
|
|
failures = 0
|
|
for point in points:
|
|
mark = (str(point.get("name", "")), int(point.get("step", -1)))
|
|
if mark in seen:
|
|
continue
|
|
seen.add(mark)
|
|
_say(f" {mark[0]}[{mark[1]}] = {point['value']:g}")
|
|
if done:
|
|
return
|
|
time.sleep(poll)
|
|
|
|
|
|
def _cancel(client: Client, handle: RunHandle) -> int:
|
|
"""Ctrl-C means stop the run, not just stop watching it."""
|
|
try:
|
|
client.cancel(handle.id)
|
|
except (SyncError, ApiError, httpx.HTTPError) as exc:
|
|
return _fail(f"could not cancel {handle.id}: {exc}")
|
|
_say(f"{handle.id} {_status('cancelled')}")
|
|
return 130
|
|
|
|
|
|
def _report_failures(handle: RunHandle) -> None:
|
|
"""What each failed node said, traceback included.
|
|
|
|
The alternative is reproducing the run under ``--local`` to see it, which
|
|
is the whole reason the traceback is stored.
|
|
"""
|
|
for failed in handle.failures:
|
|
_say(
|
|
f" {failed.get('node', '')} {_status('error')} {failed.get('error', '')}"
|
|
)
|
|
for line in str(failed.get("logs") or "").splitlines():
|
|
_say(f" {line}")
|
|
|
|
|
|
def _cached_note(client: Client, handle: RunHandle) -> str:
|
|
"""How much of the run earlier ones had already answered."""
|
|
try:
|
|
nodes = client.run(handle.id).get("nodes") or []
|
|
except (SyncError, ApiError, httpx.HTTPError):
|
|
return ""
|
|
cached = sum(1 for node in nodes if node.get("status") == "cached")
|
|
return f" ({cached}/{len(nodes)} {_status('cached')})" if cached else ""
|
|
|
|
|
|
def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
|
|
# An in-process engine lives exactly as long as this command, so a run
|
|
# nobody waits for would be thrown away with the queue holding it.
|
|
wait = args.wait or args.follow or args.local
|
|
handle: RunHandle | None = None
|
|
try:
|
|
with _client_for(args) as client:
|
|
if not args.no_sync:
|
|
_sync_first(client, args.sync)
|
|
stored = client.get_flow(args.flow)
|
|
if stored is None:
|
|
return _fail(f"no flow '{args.flow}' on that engine")
|
|
definition = stored.get("definition") or {}
|
|
params = _params(definition, rest)
|
|
# Nothing on the command line and somebody watching: ask, the way
|
|
# pressing Run in the browser asks. A scripted run — piped, or
|
|
# carrying parameters already — is left exactly as it was.
|
|
if (
|
|
not params
|
|
and not args.defaults
|
|
and (definition.get("inputs") or [])
|
|
and sys.stdin.isatty()
|
|
):
|
|
params = _ask_params(definition)
|
|
handle = client.submit(
|
|
args.flow,
|
|
params,
|
|
seed=args.seed,
|
|
no_cache=args.no_cache,
|
|
cause="cli",
|
|
)
|
|
_say(f"{handle.id} queued {json.dumps(params)}")
|
|
if not wait:
|
|
return 0
|
|
try:
|
|
if args.follow:
|
|
_follow(client, handle)
|
|
else:
|
|
handle.wait(timeout=args.timeout)
|
|
except KeyboardInterrupt:
|
|
return _cancel(client, handle)
|
|
_say(
|
|
f"{handle.id} {_status(handle.status)} "
|
|
f"{json.dumps(handle.result)}{_cached_note(client, handle)}"
|
|
)
|
|
if handle.status == "ok":
|
|
return 0
|
|
_report_failures(handle)
|
|
return 1
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
# The run is the engine's, not this command's: it carries on, and its
|
|
# id is how to find it again.
|
|
return _unreachable(
|
|
exc, f"Run {handle.id} is still on the engine." if handle else NO_ENGINE
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Status
|
|
#
|
|
# The home screen's top half, in a terminal: how the engine is, what each flow
|
|
# is doing, and what failed recently. Drawn with rich, which is what the serve
|
|
# dashboard renders this same group into.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
#: How often `--watch` asks again. The dashboard polls its summary every ten
|
|
#: seconds; nothing here moves faster than that.
|
|
WATCH_INTERVAL_S = 5.0
|
|
|
|
|
|
def _flow_state(flow: dict[str, Any]) -> str:
|
|
if flow.get("quarantined"):
|
|
return "quarantined"
|
|
if not flow.get("enabled", True):
|
|
return "stopped"
|
|
if flow.get("paused"):
|
|
return "paused"
|
|
return "running"
|
|
|
|
|
|
def _portal_phrase(portal: dict[str, Any]) -> tuple[str, str]:
|
|
"""Where this instance stands with its portal, and how to colour it.
|
|
|
|
Three states worth telling apart: never paired, paired and linked, and
|
|
paired but not reaching it — the last being the one somebody needs to know
|
|
about, since the dashboard is served from the other end.
|
|
"""
|
|
if not portal.get("enrolled"):
|
|
return "no portal", "dim"
|
|
if portal.get("connected"):
|
|
host = str(portal.get("portal_url") or "").split("//")[-1].rstrip("/")
|
|
return f"portal {host}", "green"
|
|
trouble = str(portal.get("last_error") or "").strip()
|
|
return "portal unreachable" + (f" ({trouble[:60]})" if trouble else ""), "red"
|
|
|
|
|
|
def _resource_lines(machines: dict[str, Any]) -> list[str]:
|
|
"""One line per machine: what it holds, and what is queued for it."""
|
|
targets = machines.get("targets") or []
|
|
if not targets:
|
|
return []
|
|
waiting = len(machines.get("waiting") or [])
|
|
lines = []
|
|
for target in targets:
|
|
parts = []
|
|
for label, level in (
|
|
("cpu", target.get("cpus")),
|
|
("gpu", target.get("gpus")),
|
|
):
|
|
# A machine with no card says nothing about cards.
|
|
if level and level.get("total"):
|
|
parts.append(
|
|
f"{label} {level['total'] - level['free']}/{level['total']}"
|
|
)
|
|
ram = target.get("ram_mb")
|
|
if ram and ram.get("total"):
|
|
parts.append(
|
|
f"ram {(ram['total'] - ram['free']) // 1024}/{ram['total'] // 1024}G"
|
|
)
|
|
lines.append(f"{str(target.get('target', '')):<16}" + " · ".join(parts))
|
|
if waiting:
|
|
lines[-1] += f" {waiting} waiting"
|
|
return [f"resources {lines[0]}"] + [f" {line}" for line in lines[1:]]
|
|
|
|
|
|
def _status_screen(client: Client) -> Any:
|
|
"""One frame: health, the flows, and the failures under them."""
|
|
from rich.console import Group
|
|
from rich.table import Table
|
|
from rich.text import Text
|
|
|
|
summary = client.summary()
|
|
flows = client.flows()
|
|
# Both, because an instance is usually one or the other: a live flow
|
|
# fails as an engine event, while a batch run fails on its own row.
|
|
failures = client.events(kind="failure", limit=5)
|
|
runs = client.runs(limit=5)
|
|
try:
|
|
portal = client.cloud_status()
|
|
except (SyncError, ApiError):
|
|
# Never enrolled, or an engine too old to answer. Neither is worth
|
|
# failing a status screen over.
|
|
portal = {}
|
|
try:
|
|
machines = client.resources()
|
|
except (SyncError, ApiError):
|
|
# An engine that accounts nothing, or one too old to answer.
|
|
machines = {}
|
|
|
|
healthy = summary.get("status") == "ok"
|
|
head = Text()
|
|
head.append(
|
|
"healthy" if healthy else str(summary.get("status", "unknown")),
|
|
style="green" if healthy else "yellow",
|
|
)
|
|
problems = summary.get("problems") or []
|
|
if problems:
|
|
head.append(" " + " · ".join(str(p) for p in problems), style="yellow")
|
|
phrase, style = _portal_phrase(portal)
|
|
head.append(" " + phrase, style=style)
|
|
# What it is running, since a client is upgraded on its own and a route
|
|
# this one knows may not be there. Absent from an engine older than the
|
|
# field itself, which is the answer in its own way.
|
|
head.append(f" v{summary.get('version') or '?'}", style="dim")
|
|
|
|
counts = summary.get("flows") or {}
|
|
nodes = summary.get("nodes") or {}
|
|
queue = summary.get("queue") or {}
|
|
lag = summary.get("loop_lag") or {}
|
|
facts = Text(
|
|
f"{counts.get('running', 0)}/{counts.get('total', 0)} flows running "
|
|
f"{nodes.get('total', 0)} nodes"
|
|
+ (f", {nodes['error']} failed" if nodes.get("error") else "")
|
|
+ f" queue {queue.get('pending', 0)} pending"
|
|
+ (f", {queue['parked']} parked" if queue.get("parked") else "")
|
|
+ f" loop lag {lag.get('ewma', 0.0) * 1000:.0f}ms",
|
|
style="dim",
|
|
)
|
|
|
|
table = Table(box=None, pad_edge=False, header_style="dim")
|
|
table.add_column("flow")
|
|
table.add_column("state")
|
|
table.add_column("nodes", justify="right")
|
|
table.add_column("")
|
|
for flow in sorted(flows, key=lambda one: str(one.get("name", ""))):
|
|
state = _flow_state(flow)
|
|
notes = []
|
|
if flow.get("error_count"):
|
|
notes.append(f"{flow['error_count']} in error")
|
|
if flow.get("has_draft"):
|
|
notes.append("unpublished changes")
|
|
table.add_row(
|
|
str(flow.get("title") or flow.get("name", "")),
|
|
Text(
|
|
state,
|
|
style={"running": "green", "paused": "yellow"}.get(state, "red"),
|
|
),
|
|
str(flow.get("node_count", 0)),
|
|
Text(" · ".join(notes), style="red" if flow.get("error_count") else "dim"),
|
|
)
|
|
|
|
parts: list[Any] = [head, facts]
|
|
parts += [Text(line, style="dim") for line in _resource_lines(machines)]
|
|
parts.append("")
|
|
parts.append(
|
|
table if flows else Text("No flows yet. `fluksio sync` uploads yours.", "dim")
|
|
)
|
|
# Tables rather than padded strings: a flow named longer than the column
|
|
# used to push everything after it out of line.
|
|
if runs:
|
|
recent = Table(box=None, pad_edge=False, show_header=False)
|
|
recent.add_column("")
|
|
# Minimums rather than widths: the column keeps its shape when every
|
|
# value is short and grows for the one that is not.
|
|
recent.add_column("", min_width=9)
|
|
recent.add_column("", min_width=14)
|
|
recent.add_column("", justify="right", min_width=7)
|
|
recent.add_column("", justify="right", min_width=8)
|
|
for row in runs:
|
|
state = str(row.get("status", ""))
|
|
recent.add_row(
|
|
f" {str(row.get('id', ''))[-8:]}",
|
|
Text(
|
|
state,
|
|
style={"ok": "green", "cached": "cyan", "running": "cyan"}.get(
|
|
state, "red" if state == "error" else "yellow"
|
|
),
|
|
),
|
|
Text(str(row.get("flow", "")), style="dim"),
|
|
Text(_dur(row.get("duration_ms")), style="dim"),
|
|
Text(
|
|
_ago(row.get("finished_at") or row.get("created_at")), style="dim"
|
|
),
|
|
)
|
|
parts += ["", Text("recent runs", style="dim"), recent]
|
|
if failures:
|
|
broken = Table(box=None, pad_edge=False, show_header=False)
|
|
broken.add_column("")
|
|
broken.add_column("", justify="right")
|
|
broken.add_column("")
|
|
for event in failures:
|
|
where = " ".join(
|
|
str(event.get(key, "")) for key in ("flow", "node") if event.get(key)
|
|
)
|
|
broken.add_row(
|
|
Text(f" {where}", style="red"),
|
|
Text(_ago(event.get("ts")), style="dim"),
|
|
Text(str(event.get("detail", ""))[:100], style="dim"),
|
|
)
|
|
parts += ["", Text("recent failures", style="dim"), broken]
|
|
return Group(*parts)
|
|
|
|
|
|
def cmd_status(args: argparse.Namespace) -> int:
|
|
"""How the engine is doing, once or until Ctrl-C."""
|
|
from rich.console import Console
|
|
from rich.live import Live
|
|
|
|
console = Console()
|
|
try:
|
|
# A watch outlives a blip and should ride one out; a single screen is
|
|
# a read, and says "not answering" at once.
|
|
with _client_for(args, retries=RETRIES if args.watch else 0) as client:
|
|
if not args.watch:
|
|
console.print(_status_screen(client))
|
|
return 0
|
|
if not sys.stdout.isatty():
|
|
return _fail("--watch needs a terminal; without one, drop it")
|
|
if getattr(args, "local", False):
|
|
# An in-process engine is this command and nothing else, so
|
|
# nothing can change under it while it watches.
|
|
return _fail("--watch wants an engine that outlives it; drop --local")
|
|
with Live(_status_screen(client), console=console) as live:
|
|
while True:
|
|
time.sleep(WATCH_INTERVAL_S)
|
|
live.update(_status_screen(client))
|
|
except KeyboardInterrupt:
|
|
return 130
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc)
|
|
|
|
|
|
#: How much of a run's inputs the list shows when nothing says how wide the
|
|
#: terminal is. `client.runs()` and `fluksio export runs` are where the whole
|
|
#: value is read.
|
|
PARAMS_WIDTH = 80
|
|
|
|
#: What the columns before the inputs take: the id, status, flow, duration,
|
|
#: age and stamp, with their spacing.
|
|
LISTING_WIDTH = 95
|
|
|
|
|
|
def _declared(client: Client, flows: Iterable[str]) -> dict[str, dict[str, Any]]:
|
|
"""What each flow declares its inputs to be, by flow.
|
|
|
|
So the listing can leave out an input that is simply the declared value:
|
|
what a reader is looking for across a page of runs is where they differ,
|
|
and a flow taking a few kB of JSON crowds that off the line.
|
|
"""
|
|
known: dict[str, dict[str, Any]] = {}
|
|
for name in flows:
|
|
try:
|
|
stored = client.get_flow(name) or {}
|
|
except (SyncError, ApiError, httpx.HTTPError):
|
|
# A flow deleted since its runs were recorded still lists them.
|
|
stored = {}
|
|
known[name] = {
|
|
str((entry.get("spec") or {}).get("name", "")): entry.get("initial")
|
|
for entry in (stored.get("definition") or {}).get("inputs") or []
|
|
}
|
|
return known
|
|
|
|
|
|
def _dur(ms: Any) -> str:
|
|
"""How long something took, in the same notation `_ago` reads an age in.
|
|
|
|
A run measured in hours used to print four digits of seconds, because the
|
|
three places that formatted a duration each did it their own way.
|
|
"""
|
|
seconds = max(float(ms or 0.0), 0.0) / 1000
|
|
for span, unit in ((86400, "d"), (3600, "h"), (60, "min")):
|
|
if seconds >= span:
|
|
return f"{seconds / span:.1f}{unit}"
|
|
return f"{seconds:.1f}s"
|
|
|
|
|
|
def _ago(stamp: Any) -> str:
|
|
"""How long ago something happened, in the notation the screens use.
|
|
|
|
A duration and an age read together, so they are spelled the same way:
|
|
seconds, then minutes, hours, days.
|
|
"""
|
|
if not stamp:
|
|
return ""
|
|
try:
|
|
then = datetime.fromisoformat(str(stamp))
|
|
except ValueError:
|
|
return ""
|
|
if then.tzinfo is None:
|
|
# Everything the engine records is UTC; only some spellings say so.
|
|
then = then.replace(tzinfo=UTC)
|
|
seconds = max((datetime.now(UTC) - then).total_seconds(), 0)
|
|
for span, unit in ((86400, "d"), (3600, "h"), (60, "min")):
|
|
if seconds >= span:
|
|
return f"{seconds / span:.0f}{unit} ago"
|
|
return f"{seconds:.0f}s ago"
|
|
|
|
|
|
def _stamp(row: dict[str, Any]) -> str:
|
|
"""What code a run ran: the commit, whether it was dirty, and the digest.
|
|
|
|
The digest is the part that separates two runs of one uncommitted tree —
|
|
the commit says `-dirty` for both, however much changed in between.
|
|
"""
|
|
commit, dirty, _ = (row.get("origin_commit") or "").partition("-dirty")
|
|
digest = row.get("code_digest") or ""
|
|
return (
|
|
commit[:7] + ("-dirty" if dirty else "") + (f"+{digest[:7]}" if digest else "")
|
|
)
|
|
|
|
|
|
def cmd_runs(args: argparse.Namespace) -> int:
|
|
try:
|
|
with _client_for(args, retries=0) as client:
|
|
rows = client.runs(flow=args.flow, limit=args.limit)
|
|
declared = _declared(client, {str(row["flow"]) for row in rows})
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc)
|
|
# Whatever is left of the line after the fixed columns; the fallback is
|
|
# what a pipe gets, since there is no width to ask for there.
|
|
room = max(
|
|
20,
|
|
shutil.get_terminal_size((LISTING_WIDTH + PARAMS_WIDTH, 24)).columns
|
|
- LISTING_WIDTH,
|
|
)
|
|
for row in rows:
|
|
given = row["params"] or {}
|
|
defaults = declared.get(str(row["flow"])) or {}
|
|
shown = {
|
|
name: value
|
|
for name, value in given.items()
|
|
if name not in defaults or defaults[name] != value
|
|
}
|
|
params = json.dumps(shown)
|
|
if len(params) > room:
|
|
params = params[: room - 3] + "..."
|
|
_say(
|
|
f"{row['id']} {_status(row['status'], 9)} {row['flow']:<16} "
|
|
f"{_dur(row['duration_ms']):>8} {_ago(row.get('created_at')):>9} "
|
|
f"{_stamp(row):<22} {params}"
|
|
)
|
|
return 0
|
|
|
|
|
|
#: The most runs of one group a retry looks at, which is the list route's own
|
|
#: ceiling. ponytail: a sweep larger than this needs paging, not a bigger number.
|
|
MAX_GROUP = 500
|
|
|
|
#: What a retry leaves alone. Everything else in a group — error, cancelled,
|
|
#: abandoned — is what "the ones that did not make it" means.
|
|
KEPT = frozenset({"ok", "cached", "queued", "running"})
|
|
|
|
|
|
def cmd_retry(args: argparse.Namespace) -> int:
|
|
"""Run the same thing again, one run or a group's unfinished ones."""
|
|
try:
|
|
with _client_for(args) as client:
|
|
ids = list(args.run_id)
|
|
if args.group:
|
|
ids += [
|
|
str(row["id"])
|
|
for row in client.runs(limit=MAX_GROUP, group=args.group)
|
|
if str(row["status"]) not in KEPT
|
|
]
|
|
if not ids:
|
|
return _fail("nothing to retry; name a run or a group with runs in it")
|
|
for run_id in ids:
|
|
made = client.retry(run_id)
|
|
_say(f"{made['id']} queued (retry of {run_id})")
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc)
|
|
return 0
|
|
|
|
|
|
def _artifacts(client: Client, args: argparse.Namespace) -> int:
|
|
"""List a run's files, or write one of them here."""
|
|
handle = RunHandle(client, args.run_id, client.run(args.run_id))
|
|
rows = handle.artifacts
|
|
if not args.name:
|
|
if not rows:
|
|
_say("This run produced no artifacts.")
|
|
return 0
|
|
for row in rows:
|
|
named = row.get("filename") or ""
|
|
_say(
|
|
f"{str(row['name']):<24} {row['size']:>10} B "
|
|
f"{row.get('media_type', ''):<24} {named}"
|
|
)
|
|
return 0
|
|
data = handle.download(args.name)
|
|
# The name it was written under reads better than the message's, which is
|
|
# chosen for the graph; `--out` beats both.
|
|
match = next((row for row in rows if row.get("name") == args.name), {})
|
|
out = Path(args.out or match.get("filename") or args.name)
|
|
out.write_bytes(data)
|
|
_say(f"{out} {len(data)} bytes")
|
|
return 0
|
|
|
|
|
|
def cmd_artifacts(args: argparse.Namespace) -> int:
|
|
try:
|
|
with _client_for(args, retries=0) as client:
|
|
return _artifacts(client, args)
|
|
except KeyError as exc:
|
|
return _fail(str(exc.args[0]))
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc)
|
|
|
|
|
|
def cmd_flavors(args: argparse.Namespace) -> int:
|
|
"""The named sizes a node can ask for."""
|
|
try:
|
|
with _client_for(args, retries=0) as client:
|
|
rows = client.flavors()
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc)
|
|
if not rows:
|
|
_say("No flavors. A node can still say cpus and gpus itself.")
|
|
return 0
|
|
for row in rows:
|
|
cards = f"{row['gpus']:>3} gpu" if row.get("gpus") else " " * 7
|
|
_say(
|
|
f"{row['name']:<14}{row['cpus']:>3} cpu {row['ram'] // 1024:>5} GB"
|
|
f"{cards} {row.get('description', '')}"
|
|
)
|
|
return 0
|
|
|
|
|
|
def _grid(
|
|
definition: dict[str, Any], values: list[str], seed: int | None
|
|
) -> list[dict[str, Any]]:
|
|
"""`--param lr=0.1,0.01 --param epochs=10,50` — every combination of them."""
|
|
types = _input_types(definition)
|
|
names: list[str] = []
|
|
columns: list[list[Any]] = []
|
|
for raw in values:
|
|
name, sep, listed = raw.partition("=")
|
|
name = name.replace("-", "_")
|
|
if not sep or not listed:
|
|
raise SyncError(f"--param takes name=value,value — got '{raw}'")
|
|
if name not in types:
|
|
raise SyncError(
|
|
f"'{name}' is not an input of this flow (it takes "
|
|
f"{', '.join(sorted(types)) or 'none'})"
|
|
)
|
|
names.append(name)
|
|
columns.append([_coerce(item, types[name]) for item in listed.split(",")])
|
|
return [
|
|
{"params": dict(zip(names, combination, strict=True)), "seed": seed}
|
|
for combination in itertools.product(*columns)
|
|
]
|
|
|
|
|
|
def cmd_sweep(args: argparse.Namespace) -> int:
|
|
wait = args.wait or args.local
|
|
try:
|
|
with _client_for(args) as client:
|
|
if not args.no_sync:
|
|
_sync_first(client, args.sync)
|
|
stored = client.get_flow(args.flow)
|
|
if stored is None:
|
|
return _fail(f"no flow '{args.flow}' on that engine")
|
|
entries = _grid(stored.get("definition") or {}, args.param, args.seed)
|
|
handles = client.sweep(args.flow, entries, no_cache=args.no_cache)
|
|
for handle, entry in zip(handles, entries, strict=True):
|
|
_say(f"{handle.id} queued {json.dumps(entry['params'])}")
|
|
if not wait:
|
|
return 0
|
|
failed = 0
|
|
try:
|
|
for handle in handles:
|
|
handle.wait(timeout=args.timeout)
|
|
_say(
|
|
f"{handle.id} {_status(handle.status)} "
|
|
f"{json.dumps(handle.result)}"
|
|
)
|
|
failed += handle.status != "ok"
|
|
except KeyboardInterrupt:
|
|
for handle in handles:
|
|
if not handle.refresh().done:
|
|
_cancel(client, handle)
|
|
return 130
|
|
return 1 if failed else 0
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc, "The runs are still on the engine; `fluksio runs`.")
|
|
|
|
|
|
def _selection(args: argparse.Namespace) -> dict[str, Any]:
|
|
"""The filters both exports share, minus the ones left empty."""
|
|
return {
|
|
key: value
|
|
for key, value in (
|
|
("status", args.status),
|
|
("group", args.group),
|
|
("since", args.since),
|
|
("until", args.until),
|
|
)
|
|
if value
|
|
}
|
|
|
|
|
|
def _write_rows(rows: list[dict[str, Any]], fmt: str, out: str, hint: str = "") -> int:
|
|
"""The exported rows, in the format asked for, to a file or to stdout.
|
|
|
|
The engine settles the columns over the whole selection before it sends
|
|
anything, so every row carries the same keys in the same order and the
|
|
first row's are the header.
|
|
"""
|
|
if not rows:
|
|
empty = "fluksio: nothing matched, so nothing was written"
|
|
print(empty + (f". {hint}" if hint else ""), file=sys.stderr)
|
|
return 0
|
|
if fmt == "parquet":
|
|
try:
|
|
import pyarrow
|
|
import pyarrow.parquet
|
|
except ImportError:
|
|
return _fail(
|
|
"parquet needs pyarrow — `pip install 'fluksio[parquet]'`, or "
|
|
"export csv and convert it"
|
|
)
|
|
pyarrow.parquet.write_table(pyarrow.Table.from_pylist(rows), out)
|
|
return 0
|
|
with open(out, "w", newline="") if out else nullcontext(sys.stdout) as handle:
|
|
if fmt == "jsonl":
|
|
handle.writelines(json.dumps(row) + "\n" for row in rows)
|
|
else:
|
|
# The wire is jsonl either way (see `Client._export`), so a record
|
|
# a jsonl row keeps as a value is still one here — stringify it
|
|
# for the csv cell it has to sit in.
|
|
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
|
writer.writeheader()
|
|
writer.writerows(
|
|
{
|
|
key: value
|
|
if value is None or isinstance(value, (str, int, float, bool))
|
|
else json.dumps(value)
|
|
for key, value in row.items()
|
|
}
|
|
for row in rows
|
|
)
|
|
return 0
|
|
|
|
|
|
#: How many runs `--list` reads before giving up on finding a metric name.
|
|
#: The names belong to the flow's nodes rather than to a run, so the newest
|
|
#: one that recorded any is the whole vocabulary — the rest are for a
|
|
#: selection whose newest runs failed before they measured anything.
|
|
# ponytail: the first run with names wins; a name only an older run recorded
|
|
# is not listed. A `distinct` over the selection would be exact and is a route
|
|
# of its own.
|
|
LIST_SCAN = 10
|
|
|
|
|
|
def metric_names(client: Client, ids: Iterable[str]) -> list[str]:
|
|
"""The metric names these runs carry, since a name is flow-qualified.
|
|
|
|
`train_loss` is recorded as `train.train_loss`, and asking for the bare
|
|
one matches nothing — so this is the answer to "what would match".
|
|
"""
|
|
for run_id in list(ids)[:LIST_SCAN]:
|
|
names = sorted({point["name"] for point in client.metrics(run_id)})
|
|
if names:
|
|
return names
|
|
return []
|
|
|
|
|
|
def _list_names(client: Client, args: argparse.Namespace) -> int:
|
|
filters = _selection(args)
|
|
if "until" in filters:
|
|
# The history spells the same bound `before`, where it is also the
|
|
# cursor a page is taken from.
|
|
filters["before"] = filters.pop("until")
|
|
ids = args.run or [
|
|
row["id"] for row in client.runs(flow=args.flow, limit=LIST_SCAN, **filters)
|
|
]
|
|
names = metric_names(client, ids)
|
|
_say("\n".join(names) if names else "No metrics recorded by these runs.")
|
|
return 0
|
|
|
|
|
|
def _engine_version(client: Client) -> str:
|
|
"""What the engine says it is, or empty if it is older than saying so."""
|
|
try:
|
|
return str((client.summary() or {}).get("version") or "")
|
|
except (SyncError, ApiError, httpx.HTTPError):
|
|
return ""
|
|
|
|
|
|
def _too_old(client: Client) -> str:
|
|
"""A route this client knows and the engine does not."""
|
|
version = _engine_version(client)
|
|
engine = f"the engine is {version}" if version else "the engine is older"
|
|
return (
|
|
f"this engine has no export endpoints — {engine} and this client is "
|
|
f"{__version__}. Upgrade it: `pip install -U fluksio`"
|
|
)
|
|
|
|
|
|
def _export(
|
|
args: argparse.Namespace, fetch: Callable[[Client], list[dict[str, Any]]]
|
|
) -> int:
|
|
"""Both exports: fetch what was asked for, then write it."""
|
|
if args.format == "parquet" and not args.out:
|
|
return _fail("--format parquet writes a file; name it with -o FILE")
|
|
hint = (
|
|
"`--list` names the metrics these runs carry"
|
|
if getattr(args, "name", "")
|
|
else ""
|
|
)
|
|
try:
|
|
with _client_for(args, retries=0) as client:
|
|
if getattr(args, "list_names", False):
|
|
return _list_names(client, args)
|
|
try:
|
|
rows = fetch(client)
|
|
except ApiError as exc:
|
|
if exc.status != 404:
|
|
raise
|
|
return _fail(_too_old(client))
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc)
|
|
return _write_rows(rows, args.format, args.out, hint)
|
|
|
|
|
|
def cmd_export_metrics(args: argparse.Namespace) -> int:
|
|
"""Every selected run's numbers as one long table."""
|
|
return _export(
|
|
args,
|
|
lambda client: client.export_metrics(
|
|
flow=args.flow,
|
|
ids=args.run,
|
|
name=args.name,
|
|
stride=args.stride,
|
|
**_selection(args),
|
|
),
|
|
)
|
|
|
|
|
|
def cmd_export_runs(args: argparse.Namespace) -> int:
|
|
"""One row per run: its inputs, its final numbers, what it ran."""
|
|
return _export(
|
|
args,
|
|
lambda client: client.export_runs(
|
|
flow=args.flow,
|
|
ids=args.run,
|
|
params=args.params,
|
|
metrics=args.metrics,
|
|
**_selection(args),
|
|
),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Wiring
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def add_parsers(subparsers: Any) -> None:
|
|
"""Register the client commands on `fluksio`'s parser."""
|
|
|
|
def with_engine(sub: argparse.ArgumentParser, local: bool = False) -> None:
|
|
sub.add_argument(
|
|
"--url", default="", help="the engine (default: the last login)"
|
|
)
|
|
sub.add_argument("--token", default="", help="override the stored token")
|
|
if local:
|
|
sub.add_argument(
|
|
"--local",
|
|
action="store_true",
|
|
help="boot the engine in this process instead of talking to one",
|
|
)
|
|
|
|
parser = subparsers.add_parser(
|
|
"login", help="store a token for an engine elsewhere"
|
|
)
|
|
parser.add_argument("--url", default="http://localhost:8000")
|
|
parser.add_argument("--email", default="")
|
|
parser.add_argument("--password", default="")
|
|
parser.add_argument(
|
|
"--global",
|
|
dest="shared",
|
|
action="store_true",
|
|
help="store it for the machine rather than this project",
|
|
)
|
|
parser.set_defaults(func=cmd_login)
|
|
|
|
parser = subparsers.add_parser(
|
|
"sync", help="upload the flows declared in your own code"
|
|
)
|
|
parser.add_argument(
|
|
"targets",
|
|
nargs="*",
|
|
help="modules, packages or directories to import (default: .)",
|
|
)
|
|
parser.add_argument(
|
|
"--dry-run", action="store_true", help="print what would be uploaded"
|
|
)
|
|
parser.add_argument(
|
|
"--no-publish", action="store_true", help="leave the changes as a draft"
|
|
)
|
|
parser.add_argument(
|
|
"--force", action="store_true", help="overwrite work done on the canvas"
|
|
)
|
|
with_engine(parser)
|
|
parser.set_defaults(func=cmd_sync)
|
|
|
|
parser = subparsers.add_parser(
|
|
"run", help="sync this directory, then start a run of one of its flows"
|
|
)
|
|
parser.add_argument("flow")
|
|
parser.add_argument(
|
|
"--seed",
|
|
type=int,
|
|
default=None,
|
|
help=(
|
|
"the run's seed: recorded on it, part of what tells two runs of one "
|
|
"configuration apart, and passed to an input named 'seed' when the "
|
|
"flow declares one"
|
|
),
|
|
)
|
|
parser.add_argument("--wait", action="store_true", help="block until it finishes")
|
|
parser.add_argument(
|
|
"--follow",
|
|
action="store_true",
|
|
help="wait, printing the numbers it reports as they arrive",
|
|
)
|
|
parser.add_argument(
|
|
"--timeout",
|
|
type=float,
|
|
default=0.0,
|
|
metavar="SECONDS",
|
|
help=(
|
|
"give up waiting after this long and leave the run going; 0, the "
|
|
"default, waits as long as it takes. Not the node timeout"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--no-sync",
|
|
action="store_true",
|
|
help="run what is already on the engine, without uploading first",
|
|
)
|
|
parser.add_argument(
|
|
"--sync",
|
|
action="append",
|
|
default=[],
|
|
metavar="PATH",
|
|
help="what to sync first (default: this directory, downwards)",
|
|
)
|
|
parser.add_argument(
|
|
"--no-cache",
|
|
action="store_true",
|
|
help="execute every node, even one an earlier run already answered",
|
|
)
|
|
parser.add_argument(
|
|
"--defaults",
|
|
action="store_true",
|
|
help="take every input's declared value instead of asking for it",
|
|
)
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_run)
|
|
|
|
parser = subparsers.add_parser(
|
|
"status", help="how the engine is doing, and what each flow is up to"
|
|
)
|
|
parser.add_argument(
|
|
"--watch",
|
|
action="store_true",
|
|
help="keep it on screen, refreshed until Ctrl-C",
|
|
)
|
|
# `--local` for the same reason `runs` has it: the flows and the history are
|
|
# in this directory, and reading them should not need a server.
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_status)
|
|
|
|
parser = subparsers.add_parser("runs", help="the runs an engine has recorded")
|
|
parser.add_argument("--flow", default="")
|
|
parser.add_argument("--limit", type=int, default=20)
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_runs)
|
|
|
|
parser = subparsers.add_parser(
|
|
"retry", help="run something again: one run, or a group's unfinished ones"
|
|
)
|
|
parser.add_argument("run_id", nargs="*", help="the runs to retry")
|
|
parser.add_argument(
|
|
"--group",
|
|
default="",
|
|
metavar="ID",
|
|
help="every run of this sweep that did not end ok",
|
|
)
|
|
with_engine(parser)
|
|
parser.set_defaults(func=cmd_retry)
|
|
|
|
parser = subparsers.add_parser(
|
|
"artifacts", help="the files a run produced; name one to download it"
|
|
)
|
|
parser.add_argument("run_id")
|
|
parser.add_argument("name", nargs="?", default="")
|
|
parser.add_argument(
|
|
"-o",
|
|
"--out",
|
|
default="",
|
|
metavar="PATH",
|
|
help="where to write it (default: the name it was saved under)",
|
|
)
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_artifacts)
|
|
|
|
parser = subparsers.add_parser(
|
|
"flavors", help="the named resource sizes a node can ask for"
|
|
)
|
|
with_engine(parser)
|
|
parser.set_defaults(func=cmd_flavors)
|
|
|
|
parser = subparsers.add_parser(
|
|
"sweep", help="one flow, once per combination of the parameters given"
|
|
)
|
|
parser.add_argument("flow")
|
|
parser.add_argument(
|
|
"--param",
|
|
action="append",
|
|
default=[],
|
|
metavar="NAME=V1,V2",
|
|
help="an input and the values to try; repeat for a grid",
|
|
)
|
|
parser.add_argument(
|
|
"--seed",
|
|
type=int,
|
|
default=None,
|
|
help="the seed every run in the sweep gets; vary it with --param seed=1,2",
|
|
)
|
|
parser.add_argument("--wait", action="store_true", help="block until all finish")
|
|
parser.add_argument(
|
|
"--timeout",
|
|
type=float,
|
|
default=0.0,
|
|
metavar="SECONDS",
|
|
help="give up waiting after this long; 0, the default, waits them out",
|
|
)
|
|
parser.add_argument(
|
|
"--no-sync",
|
|
action="store_true",
|
|
help="run what is already on the engine, without uploading first",
|
|
)
|
|
parser.add_argument(
|
|
"--sync",
|
|
action="append",
|
|
default=[],
|
|
metavar="PATH",
|
|
help="what to sync first (default: this directory, downwards)",
|
|
)
|
|
parser.add_argument(
|
|
"--no-cache",
|
|
action="store_true",
|
|
help="execute every node, even one an earlier run already answered",
|
|
)
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_sweep)
|
|
|
|
parser = subparsers.add_parser(
|
|
"export", help="runs and their numbers as a table an analysis reads"
|
|
)
|
|
exports = parser.add_subparsers(dest="table", required=True)
|
|
|
|
def with_selection(sub: argparse.ArgumentParser) -> None:
|
|
sub.add_argument("--flow", default="", help="only runs of this flow")
|
|
sub.add_argument(
|
|
"--run",
|
|
action="append",
|
|
default=[],
|
|
metavar="ID",
|
|
help="only this run; repeat for several",
|
|
)
|
|
sub.add_argument("--group", default="", help="only the runs of this sweep")
|
|
sub.add_argument("--status", default="", help="only runs that ended this way")
|
|
sub.add_argument(
|
|
"--since", default="", metavar="TS", help="only runs created at or after"
|
|
)
|
|
sub.add_argument(
|
|
"--until", default="", metavar="TS", help="only runs created before"
|
|
)
|
|
sub.add_argument("--format", default="csv", choices=("csv", "jsonl", "parquet"))
|
|
sub.add_argument(
|
|
"-o", "--out", default="", metavar="FILE", help="write here, not to stdout"
|
|
)
|
|
with_engine(sub, local=True)
|
|
|
|
sub = exports.add_parser(
|
|
"metrics", help="one row per run, metric and step — the tidy shape"
|
|
)
|
|
with_selection(sub)
|
|
sub.add_argument("--name", default="", metavar="A,B", help="only these metrics")
|
|
sub.add_argument(
|
|
"--stride",
|
|
type=int,
|
|
default=1,
|
|
help="keep every Nth point of each curve",
|
|
)
|
|
sub.add_argument(
|
|
"--list",
|
|
dest="list_names",
|
|
action="store_true",
|
|
help="print the metric names these runs carry, and stop",
|
|
)
|
|
sub.set_defaults(func=cmd_export_metrics)
|
|
|
|
sub = exports.add_parser(
|
|
"runs", help="one row per run: its inputs, its final numbers, its status"
|
|
)
|
|
with_selection(sub)
|
|
sub.add_argument(
|
|
"--params",
|
|
default="",
|
|
metavar="A,B",
|
|
help=(
|
|
"the inputs to put in columns, dotted into a record "
|
|
"(default: every input the runs recorded)"
|
|
),
|
|
)
|
|
sub.add_argument(
|
|
"--metrics",
|
|
default="",
|
|
metavar="A,B",
|
|
help="the final numbers to keep, e.g. final_metrics.train_loss",
|
|
)
|
|
sub.set_defaults(func=cmd_export_runs)
|