stroblmeandClaude Opus 5
4479eeb726
Docs / docs (push) Successful in 38s
Playwright Tests / test-playwright (1, 2) (push) Successful in 2m56s
Playwright Tests / test-playwright (2, 2) (push) Successful in 2m7s
pre-commit / pre-commit (push) Failing after 2m17s
Test Backend / test-backend (push) Successful in 2m54s
Compose Smoke Test / test-compose (push) Successful in 44s
Playwright Tests / merge-reports (push) Successful in 1m17s
Three things the first export pass got wrong for a real study. **Dotted paths.** A node returns a record, not a scalar — the numbers arrive inside `final_metrics` — so `--metrics final_metrics.train_loss` yielded an empty column and `--metrics final_metrics` yielded the whole record in one cell. Both sides of the wide table now take dotted paths, and the defaults reach the same depth: every number a result carries is a column named by its path, and inputs are compared leaf by leaf, so two configurations differing in one field give that field as the axis rather than two blobs that are merely not equal. Lists stay whole — a curve belongs in the long table. **`--list`.** Metric names are flow-qualified, so `--name train_loss` matched nothing and said only that. `fluksio export metrics --list` prints the names the selection carries, and an empty export made with `--name` points at it. **A version to compare.** The CLI ships ahead of the engine and a stale one answered a flat 404 with nothing anywhere in the API to tell how old it was. The engine reports `version` on `/observability/summary`, `fluksio status` prints it, and a 404 from export now names both versions — or says "older" when the field itself predates the engine. Bumped to 0.1.5, which is what makes the number worth reading. Also formats `flow/metrics.py`, which had been committed unformatted and was the last `ruff format --check` failure. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01A9Hdrmf2cwNABCnE5x9UJa
1247 lines
44 KiB
Python
1247 lines
44 KiB
Python
"""The `fluksio sync`, `run`, `runs`, `sweep`, `status` and `login` commands.
|
|
|
|
Kept beside the SDK rather than in `fluksio.cli`: these are the client half of
|
|
the tool, and none of them needs the engine to be importable — `--local`, which
|
|
does, imports it inside the branch that asked for it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import getpass
|
|
import importlib
|
|
import itertools
|
|
import json
|
|
import pkgutil
|
|
import sys
|
|
import time
|
|
from collections.abc import Callable, Iterator
|
|
from contextlib import contextmanager, nullcontext
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from fluksio import __version__
|
|
from fluksio.sdk import FLOWS, Flow, SyncError
|
|
from fluksio.sdk.client import (
|
|
GLOBAL_DATA_DIR,
|
|
RETRIES,
|
|
WAIT_TOLERANCE,
|
|
ApiError,
|
|
Client,
|
|
RunHandle,
|
|
data_dir,
|
|
login,
|
|
origin_of,
|
|
repo_root,
|
|
sync,
|
|
)
|
|
|
|
__all__ = ["add_parsers", "discover"]
|
|
|
|
#: Statuses worth a colour, and the SGR code each gets.
|
|
_COLORS = {
|
|
"ok": "32",
|
|
"error": "31",
|
|
"cached": "36",
|
|
"cancelled": "33",
|
|
"abandoned": "31",
|
|
"running": "36",
|
|
"queued": "33",
|
|
}
|
|
|
|
|
|
def _say(message: str = "") -> None:
|
|
print(message)
|
|
|
|
|
|
def _fail(message: str) -> int:
|
|
print(f"fluksio: {message}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
#: What to try when nothing answered at all. By far the commonest reason is
|
|
#: that no engine is running, and the message said only that it was not.
|
|
NO_ENGINE = "Is one running? `fluksio serve` starts one."
|
|
|
|
|
|
def _unreachable(exc: Exception, note: str = NO_ENGINE) -> int:
|
|
"""The engine did not answer. Say so as a sentence, not a traceback."""
|
|
return _fail(
|
|
f"engine not answering ({type(exc).__name__}: {exc})"
|
|
+ (f". {note}" if note else "")
|
|
)
|
|
|
|
|
|
def _status(text: str, width: int = 0) -> str:
|
|
"""A status, coloured when a terminal is reading it.
|
|
|
|
Padded before it is coloured: the escape sequences are characters as far
|
|
as `str.format` is concerned, and a column that lines up in a pipe would
|
|
not line up on screen.
|
|
"""
|
|
body = f"{text:<{width}}" if width else text
|
|
code = _COLORS.get(text)
|
|
if not code or not sys.stdout.isatty():
|
|
return body
|
|
return f"\033[{code}m{body}\033[0m"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Discovery
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _package_of(directory: Path) -> tuple[str, str]:
|
|
"""The path root and dotted name of a package directory."""
|
|
parts = [directory.name]
|
|
parent = directory.parent
|
|
while (parent / "__init__.py").exists():
|
|
parts.append(parent.name)
|
|
parent = parent.parent
|
|
return str(parent), ".".join(reversed(parts))
|
|
|
|
|
|
def _module_of(path: Path) -> tuple[str, str]:
|
|
"""The path root and dotted name of a module file."""
|
|
parts = [path.stem]
|
|
directory = path.parent
|
|
while (directory / "__init__.py").exists():
|
|
parts.append(directory.name)
|
|
directory = directory.parent
|
|
return str(directory), ".".join(reversed(parts))
|
|
|
|
|
|
def _import(root: str, dotted: str) -> None:
|
|
if root not in sys.path:
|
|
sys.path.insert(0, root)
|
|
importlib.import_module(dotted)
|
|
|
|
|
|
def discover(targets: list[str]) -> list[Flow]:
|
|
"""Import what was named and hand back the flows it declared.
|
|
|
|
Imported by dotted name with its root on the path, never from a file
|
|
location: the generated node bodies import the same way, and a module
|
|
loaded under a different name would generate an import that does not
|
|
resolve.
|
|
"""
|
|
for target in targets:
|
|
path = Path(target)
|
|
if not path.exists():
|
|
_import(str(Path.cwd()), target)
|
|
continue
|
|
path = path.resolve()
|
|
if path.is_file():
|
|
_import(*_module_of(path))
|
|
continue
|
|
if (path / "__init__.py").exists():
|
|
root, dotted = _package_of(path)
|
|
_import(root, dotted)
|
|
package = sys.modules[dotted]
|
|
for info in pkgutil.walk_packages(package.__path__, f"{dotted}."):
|
|
importlib.import_module(info.name)
|
|
continue
|
|
# A plain directory — a repository root, usually. Its own modules,
|
|
# and the packages inside it: `myresearch/` beside a `README` is the
|
|
# ordinary shape, and naming it explicitly should not be the price of
|
|
# keeping your code in a package.
|
|
for module in sorted(path.glob("*.py")):
|
|
_import(*_module_of(module))
|
|
for child in sorted(path.iterdir()):
|
|
if child.name.startswith(".") or not (child / "__init__.py").exists():
|
|
continue
|
|
root, dotted = _package_of(child)
|
|
_import(root, dotted)
|
|
for info in pkgutil.walk_packages(
|
|
sys.modules[dotted].__path__, f"{dotted}."
|
|
):
|
|
importlib.import_module(info.name)
|
|
return list(FLOWS.values())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Which engine
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@contextmanager
|
|
def _engine_client() -> Iterator[Client]:
|
|
"""The engine itself, in this process, behind the ordinary client.
|
|
|
|
Everything `fluksio serve` does apart from listening on a socket: the same
|
|
data directory, the same database, the same admin, the same lifespan. The
|
|
app is driven through its ASGI interface, so a run costs what it costs on
|
|
a served engine and lands in the same history — which is what makes the
|
|
stage cache carry across the two.
|
|
|
|
The engine only exists for the length of the command, so nothing here is
|
|
written back as a login: a stored token belongs to whichever engine
|
|
`fluksio login` was pointed at.
|
|
"""
|
|
# Imported here, not at module scope: everything else in this file is the
|
|
# client half and must keep working with no engine installed.
|
|
import logging
|
|
|
|
from fluksio.cli import _data_dir, _prepare, _print_new_admin
|
|
|
|
directory = _data_dir(None)
|
|
# Settings are read when the app is imported, so this comes first.
|
|
_prepare(directory)
|
|
# Nothing here goes over a network, so httpx logging each call as a
|
|
# request to "testserver" is noise that also happens to be untrue.
|
|
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
|
|
from datetime import timedelta
|
|
|
|
from fastapi.testclient import TestClient
|
|
from sqlmodel import Session
|
|
|
|
from fluksio.core import security
|
|
from fluksio.core.bootstrap import ensure_superuser
|
|
from fluksio.core.db import engine
|
|
|
|
with Session(engine) as session:
|
|
admin, generated = ensure_superuser(session)
|
|
admin_id = admin.id
|
|
if generated:
|
|
_print_new_admin(admin.email, generated)
|
|
token = security.create_access_token(admin_id, expires_delta=timedelta(hours=12))
|
|
|
|
from fluksio.main import app
|
|
|
|
# Entering the client is what runs the lifespan: the worker pool, the
|
|
# controller and the run service all start here and stop on the way out.
|
|
with TestClient(app) as http:
|
|
yield Client(http=http, token=token)
|
|
|
|
|
|
@contextmanager
|
|
def _client_for(args: argparse.Namespace, retries: int = RETRIES) -> Iterator[Client]:
|
|
"""The engine this command talks to: one running somewhere, or this one.
|
|
|
|
``retries=0`` is what a read passes: an engine that is not there otherwise
|
|
takes the backoff — seconds — to say so, and nothing in a read is worth
|
|
waiting out a restart for. A command that submits keeps them.
|
|
"""
|
|
if getattr(args, "local", False):
|
|
with _engine_client() as client:
|
|
yield client
|
|
else:
|
|
yield Client(url=args.url, token=args.token, retries=retries)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Commands
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def cmd_login(args: argparse.Namespace) -> int:
|
|
"""Only needed for an engine somewhere else — `serve` signs you in here."""
|
|
email = args.email or input("Email: ")
|
|
password = args.password or getpass.getpass("Password: ")
|
|
directory = GLOBAL_DATA_DIR.expanduser() if args.shared else data_dir()
|
|
try:
|
|
path = login(args.url, email, password, directory=directory)
|
|
except ApiError as exc:
|
|
return _fail(f"could not log in: {exc.detail}")
|
|
_say(f"Logged in to {args.url}; the token is in {path}.")
|
|
return 0
|
|
|
|
|
|
def cmd_sync(args: argparse.Namespace) -> int:
|
|
targets = args.targets or ["."]
|
|
try:
|
|
flows = discover(targets)
|
|
except (ImportError, SyncError) as exc:
|
|
return _fail(str(exc))
|
|
if not flows:
|
|
return _fail(
|
|
f"no flows declared in {', '.join(targets)} — a flow is a `Flow(...)` "
|
|
"at module level. Name the package if it is somewhere else: "
|
|
"`fluksio sync src/myresearch`."
|
|
)
|
|
|
|
repo = repo_root(targets[0])
|
|
origin = origin_of(repo)
|
|
if origin["dirty"]:
|
|
_say(f"warning: {repo} has uncommitted changes, so the stamp says -dirty")
|
|
if not origin["commit"]:
|
|
_say(f"warning: {repo} is not a git repository, so runs cannot name a commit")
|
|
|
|
if args.dry_run:
|
|
for target in flows:
|
|
_say(f"=== flow {target.name}")
|
|
_say(json.dumps(target.document(origin), indent=2))
|
|
for node_id, code in target.shims().items():
|
|
_say(f"=== {target.name}.{node_id}")
|
|
_say(code)
|
|
return 0
|
|
|
|
try:
|
|
client = Client(url=args.url, token=args.token)
|
|
reports = sync(
|
|
flows,
|
|
client,
|
|
origin=origin,
|
|
publish=not args.no_publish,
|
|
force=args.force,
|
|
)
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc, "Nothing was published; sync again when it is back.")
|
|
|
|
for report in reports:
|
|
if report.unchanged:
|
|
_say(f" {report.flow}: unchanged")
|
|
continue
|
|
what = "created" if report.created else "updated"
|
|
detail = ", ".join(report.changed)
|
|
state = "published" if report.published else "draft"
|
|
_say(f" {report.flow}: {what} ({detail}) — {state}")
|
|
stamp = origin["commit"][:7] + ("-dirty" if origin["dirty"] else "")
|
|
_say(f"Stamped with {stamp or 'no commit'} from {repo}.")
|
|
return 0
|
|
|
|
|
|
def _coerce(value: str, dtype: str) -> Any:
|
|
if value.startswith("@run:") or value.startswith("sha256:"):
|
|
# A run's output, named rather than typed out. Sent as it stands: the
|
|
# engine turns it into the value itself, whatever type that is. Passing
|
|
# the whole thing as JSON still works, and is what a script that
|
|
# already has it in hand would do.
|
|
return value
|
|
if dtype == "int":
|
|
return int(value)
|
|
if dtype == "float":
|
|
return float(value)
|
|
if dtype == "bool":
|
|
return value.lower() in ("true", "1", "yes", "on")
|
|
if dtype == "str":
|
|
return value
|
|
return json.loads(value)
|
|
|
|
|
|
def _input_types(definition: dict[str, Any]) -> dict[str, str]:
|
|
"""What each of a flow's inputs is declared to be."""
|
|
return {
|
|
str(entry["spec"]["name"]): str(entry["spec"].get("dtype", "float"))
|
|
for entry in definition.get("inputs") or []
|
|
}
|
|
|
|
|
|
def _ask_params(definition: dict[str, Any]) -> dict[str, Any]:
|
|
"""The run dialog, in a terminal: one line per declared input.
|
|
|
|
An empty answer leaves the input out, which is what keeps its declared
|
|
value — the same thing the browser's dialog does with a field nobody
|
|
filled in. So pressing Enter through the lot runs the defaults.
|
|
"""
|
|
params: dict[str, Any] = {}
|
|
for entry in definition.get("inputs") or []:
|
|
spec = entry.get("spec") or {}
|
|
name = str(spec.get("name", ""))
|
|
if not name:
|
|
continue
|
|
dtype = str(spec.get("dtype", "float"))
|
|
initial = entry.get("initial")
|
|
shown = "" if initial is None else json.dumps(initial)
|
|
prompt = f"{name} ({dtype})" + (f" [{shown}]" if shown else "") + ": "
|
|
answer = input(prompt).strip()
|
|
if not answer:
|
|
continue
|
|
try:
|
|
params[name] = _coerce(answer, dtype)
|
|
except ValueError as exc:
|
|
raise SyncError(f"'{name}' takes {dtype}: {exc}") from exc
|
|
return params
|
|
|
|
|
|
def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
|
|
"""Turn `--lr 0.05` into a typed parameter, using the flow's own inputs."""
|
|
types = _input_types(definition)
|
|
params: dict[str, Any] = {}
|
|
pending: str | None = None
|
|
for token in rest:
|
|
if token.startswith("--"):
|
|
if pending is not None:
|
|
# A flag with no value is a flag: `--resume` means true.
|
|
params[pending] = True
|
|
name, sep, value = token[2:].partition("=")
|
|
# Only the name is spelled with dashes; a value may hold one, and
|
|
# `--lr=1e-4` is the case that says so.
|
|
pending = name.replace("-", "_")
|
|
if sep:
|
|
params[pending] = _coerce(value, types.get(pending, "json"))
|
|
pending = None
|
|
continue
|
|
if pending is None:
|
|
raise SyncError(f"unexpected argument '{token}'")
|
|
params[pending] = _coerce(token, types.get(pending, "json"))
|
|
pending = None
|
|
if pending is not None:
|
|
params[pending] = True
|
|
unknown = sorted(set(params) - set(types))
|
|
if unknown:
|
|
raise SyncError(
|
|
f"'{unknown[0]}' is not an input of this flow (it takes "
|
|
f"{', '.join(sorted(types)) or 'none'})"
|
|
)
|
|
return params
|
|
|
|
|
|
def _sync_first(client: Client) -> None:
|
|
"""Upload what the working directory declares, before running it.
|
|
|
|
The reason a run exists is usually the edit that came before it, and
|
|
remembering to sync is remembering to do the thing the computer could have
|
|
done. So `run` syncs by default — including the worker refresh, which is
|
|
what makes an edit to your own package take effect at all.
|
|
|
|
A directory that declares nothing is not an error: a flow drawn on the
|
|
canvas is run the same way, and has nothing to upload.
|
|
"""
|
|
try:
|
|
flows = discover(["."])
|
|
except (ImportError, SyncError) as exc:
|
|
# Do not fail a run for a module the run may not even need.
|
|
_say(f"warning: nothing synced — {exc}")
|
|
return
|
|
if not flows:
|
|
return
|
|
repo = repo_root(".")
|
|
reports = sync(flows, client, origin=origin_of(repo))
|
|
changed = [r for r in reports if not r.unchanged]
|
|
if changed:
|
|
_say(f"synced {', '.join(r.flow for r in changed)}")
|
|
|
|
|
|
def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
|
|
"""Print a run's numbers as they arrive, until it is over.
|
|
|
|
Polled rather than pushed: the engine writes a metric down when it is
|
|
reported, so asking once a second draws the same curve a socket would
|
|
have, without either side holding a connection open. The status is read
|
|
before the numbers, so the last batch is never the one that gets missed.
|
|
"""
|
|
seen: set[tuple[str, int]] = set()
|
|
failures = 0
|
|
while True:
|
|
try:
|
|
done = handle.refresh().done
|
|
points = client.metrics(handle.id)
|
|
except (httpx.HTTPError, ApiError) as exc:
|
|
# Following an eight-hour run must not end because one poll of it
|
|
# did. The run is still going; only this side lost sight of it.
|
|
if isinstance(exc, ApiError) and exc.status < 500:
|
|
raise
|
|
failures += 1
|
|
if failures >= WAIT_TOLERANCE:
|
|
raise
|
|
time.sleep(poll)
|
|
continue
|
|
failures = 0
|
|
for point in points:
|
|
mark = (str(point.get("name", "")), int(point.get("step", -1)))
|
|
if mark in seen:
|
|
continue
|
|
seen.add(mark)
|
|
_say(f" {mark[0]}[{mark[1]}] = {point['value']:g}")
|
|
if done:
|
|
return
|
|
time.sleep(poll)
|
|
|
|
|
|
def _cancel(client: Client, handle: RunHandle) -> int:
|
|
"""Ctrl-C means stop the run, not just stop watching it."""
|
|
try:
|
|
client.cancel(handle.id)
|
|
except (SyncError, ApiError, httpx.HTTPError) as exc:
|
|
return _fail(f"could not cancel {handle.id}: {exc}")
|
|
_say(f"{handle.id} {_status('cancelled')}")
|
|
return 130
|
|
|
|
|
|
def _cached_note(client: Client, handle: RunHandle) -> str:
|
|
"""How much of the run earlier ones had already answered."""
|
|
try:
|
|
nodes = client.run(handle.id).get("nodes") or []
|
|
except (SyncError, ApiError, httpx.HTTPError):
|
|
return ""
|
|
cached = sum(1 for node in nodes if node.get("status") == "cached")
|
|
return f" ({cached}/{len(nodes)} {_status('cached')})" if cached else ""
|
|
|
|
|
|
def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
|
|
# An in-process engine lives exactly as long as this command, so a run
|
|
# nobody waits for would be thrown away with the queue holding it.
|
|
wait = args.wait or args.follow or args.local
|
|
handle: RunHandle | None = None
|
|
try:
|
|
with _client_for(args) as client:
|
|
if not args.no_sync:
|
|
_sync_first(client)
|
|
stored = client.get_flow(args.flow)
|
|
if stored is None:
|
|
return _fail(f"no flow '{args.flow}' on that engine")
|
|
definition = stored.get("definition") or {}
|
|
params = _params(definition, rest)
|
|
# Nothing on the command line and somebody watching: ask, the way
|
|
# pressing Run in the browser asks. A scripted run — piped, or
|
|
# carrying parameters already — is left exactly as it was.
|
|
if (
|
|
not params
|
|
and not args.defaults
|
|
and (definition.get("inputs") or [])
|
|
and sys.stdin.isatty()
|
|
):
|
|
params = _ask_params(definition)
|
|
handle = client.submit(
|
|
args.flow,
|
|
params,
|
|
seed=args.seed,
|
|
no_cache=args.no_cache,
|
|
cause="cli",
|
|
)
|
|
_say(f"{handle.id} queued {json.dumps(params)}")
|
|
if not wait:
|
|
return 0
|
|
try:
|
|
if args.follow:
|
|
_follow(client, handle)
|
|
else:
|
|
handle.wait(timeout=args.timeout)
|
|
except KeyboardInterrupt:
|
|
return _cancel(client, handle)
|
|
_say(
|
|
f"{handle.id} {_status(handle.status)} "
|
|
f"{json.dumps(handle.result)}{_cached_note(client, handle)}"
|
|
)
|
|
return 0 if handle.status == "ok" else 1
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
# The run is the engine's, not this command's: it carries on, and its
|
|
# id is how to find it again.
|
|
return _unreachable(
|
|
exc, f"Run {handle.id} is still on the engine." if handle else NO_ENGINE
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Status
|
|
#
|
|
# The home screen's top half, in a terminal: how the engine is, what each flow
|
|
# is doing, and what failed recently. Drawn with rich, which is already here
|
|
# under fastapi's own CLI.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
#: How often `--watch` asks again. The dashboard polls its summary every ten
|
|
#: seconds; nothing here moves faster than that.
|
|
WATCH_INTERVAL_S = 5.0
|
|
|
|
|
|
def _flow_state(flow: dict[str, Any]) -> str:
|
|
if flow.get("quarantined"):
|
|
return "quarantined"
|
|
if not flow.get("enabled", True):
|
|
return "stopped"
|
|
if flow.get("paused"):
|
|
return "paused"
|
|
return "running"
|
|
|
|
|
|
def _portal_phrase(portal: dict[str, Any]) -> tuple[str, str]:
|
|
"""Where this installation stands with its portal, and how to colour it.
|
|
|
|
Three states worth telling apart: never paired, paired and linked, and
|
|
paired but not reaching it — the last being the one somebody needs to know
|
|
about, since the dashboard is served from the other end.
|
|
"""
|
|
if not portal.get("enrolled"):
|
|
return "no portal", "dim"
|
|
if portal.get("connected"):
|
|
host = str(portal.get("portal_url") or "").split("//")[-1].rstrip("/")
|
|
return f"portal {host}", "green"
|
|
trouble = str(portal.get("last_error") or "").strip()
|
|
return "portal unreachable" + (f" ({trouble[:60]})" if trouble else ""), "red"
|
|
|
|
|
|
def _resource_lines(machines: dict[str, Any]) -> list[str]:
|
|
"""One line per machine: what it holds, and what is queued for it."""
|
|
targets = machines.get("targets") or []
|
|
if not targets:
|
|
return []
|
|
waiting = len(machines.get("waiting") or [])
|
|
lines = []
|
|
for target in targets:
|
|
parts = []
|
|
for label, level in (
|
|
("cpu", target.get("cpus")),
|
|
("gpu", target.get("gpus")),
|
|
):
|
|
# A machine with no card says nothing about cards.
|
|
if level and level.get("total"):
|
|
parts.append(
|
|
f"{label} {level['total'] - level['free']}/{level['total']}"
|
|
)
|
|
ram = target.get("ram_mb")
|
|
if ram and ram.get("total"):
|
|
parts.append(
|
|
f"ram {(ram['total'] - ram['free']) // 1024}/{ram['total'] // 1024}G"
|
|
)
|
|
lines.append(f"{str(target.get('target', '')):<16}" + " · ".join(parts))
|
|
if waiting:
|
|
lines[-1] += f" {waiting} waiting"
|
|
return [f"resources {lines[0]}"] + [f" {line}" for line in lines[1:]]
|
|
|
|
|
|
def _status_screen(client: Client) -> Any:
|
|
"""One frame: health, the flows, and the failures under them."""
|
|
from rich.console import Group
|
|
from rich.table import Table
|
|
from rich.text import Text
|
|
|
|
summary = client.summary()
|
|
flows = client.flows()
|
|
# Both, because an installation is usually one or the other: a live flow
|
|
# fails as an engine event, while a batch run fails on its own row.
|
|
failures = client.events(kind="failure", limit=5)
|
|
runs = client.runs(limit=5)
|
|
try:
|
|
portal = client.cloud_status()
|
|
except (SyncError, ApiError):
|
|
# Never enrolled, or an engine too old to answer. Neither is worth
|
|
# failing a status screen over.
|
|
portal = {}
|
|
try:
|
|
machines = client.resources()
|
|
except (SyncError, ApiError):
|
|
# An engine that accounts nothing, or one too old to answer.
|
|
machines = {}
|
|
|
|
healthy = summary.get("status") == "ok"
|
|
head = Text()
|
|
head.append(
|
|
"healthy" if healthy else str(summary.get("status", "unknown")),
|
|
style="green" if healthy else "yellow",
|
|
)
|
|
problems = summary.get("problems") or []
|
|
if problems:
|
|
head.append(" " + " · ".join(str(p) for p in problems), style="yellow")
|
|
phrase, style = _portal_phrase(portal)
|
|
head.append(" " + phrase, style=style)
|
|
# What it is running, since a client is upgraded on its own and a route
|
|
# this one knows may not be there. Absent from an engine older than the
|
|
# field itself, which is the answer in its own way.
|
|
head.append(f" v{summary.get('version') or '?'}", style="dim")
|
|
|
|
counts = summary.get("flows") or {}
|
|
nodes = summary.get("nodes") or {}
|
|
queue = summary.get("queue") or {}
|
|
lag = summary.get("loop_lag") or {}
|
|
facts = Text(
|
|
f"{counts.get('running', 0)}/{counts.get('total', 0)} flows running "
|
|
f"{nodes.get('total', 0)} nodes"
|
|
+ (f", {nodes['error']} failed" if nodes.get("error") else "")
|
|
+ f" queue {queue.get('pending', 0)} pending"
|
|
+ (f", {queue['parked']} parked" if queue.get("parked") else "")
|
|
+ f" loop lag {lag.get('ewma', 0.0) * 1000:.0f}ms",
|
|
style="dim",
|
|
)
|
|
|
|
table = Table(box=None, pad_edge=False, header_style="dim")
|
|
table.add_column("flow")
|
|
table.add_column("state")
|
|
table.add_column("nodes", justify="right")
|
|
table.add_column("")
|
|
for flow in sorted(flows, key=lambda one: str(one.get("name", ""))):
|
|
state = _flow_state(flow)
|
|
notes = []
|
|
if flow.get("error_count"):
|
|
notes.append(f"{flow['error_count']} in error")
|
|
if flow.get("has_draft"):
|
|
notes.append("unpublished changes")
|
|
table.add_row(
|
|
str(flow.get("title") or flow.get("name", "")),
|
|
Text(
|
|
state,
|
|
style={"running": "green", "paused": "yellow"}.get(state, "red"),
|
|
),
|
|
str(flow.get("node_count", 0)),
|
|
Text(" · ".join(notes), style="red" if flow.get("error_count") else "dim"),
|
|
)
|
|
|
|
parts: list[Any] = [head, facts]
|
|
parts += [Text(line, style="dim") for line in _resource_lines(machines)]
|
|
parts.append("")
|
|
parts.append(
|
|
table if flows else Text("No flows yet. `fluksio sync` uploads yours.", "dim")
|
|
)
|
|
if runs:
|
|
parts += ["", Text("recent runs", style="dim")]
|
|
for row in runs:
|
|
state = str(row.get("status", ""))
|
|
parts.append(
|
|
Text(f" {str(row.get('id', ''))[-8:]} ")
|
|
+ Text(
|
|
f"{state:<9}",
|
|
style={"ok": "green", "cached": "cyan", "running": "cyan"}.get(
|
|
state, "red" if state == "error" else "yellow"
|
|
),
|
|
)
|
|
+ Text(
|
|
f"{str(row.get('flow', '')):<16}"
|
|
f"{(row.get('duration_ms') or 0) / 1000:7.1f}s",
|
|
style="dim",
|
|
)
|
|
)
|
|
if failures:
|
|
parts += ["", Text("recent failures", style="dim")]
|
|
for event in failures:
|
|
where = " ".join(
|
|
str(event.get(key, "")) for key in ("flow", "node") if event.get(key)
|
|
)
|
|
parts.append(
|
|
Text(f" {where} ", style="red")
|
|
+ Text(str(event.get("detail", ""))[:100], style="dim")
|
|
)
|
|
return Group(*parts)
|
|
|
|
|
|
def cmd_status(args: argparse.Namespace) -> int:
|
|
"""How the engine is doing, once or until Ctrl-C."""
|
|
from rich.console import Console
|
|
from rich.live import Live
|
|
|
|
console = Console()
|
|
try:
|
|
# A watch outlives a blip and should ride one out; a single screen is
|
|
# a read, and says "not answering" at once.
|
|
with _client_for(args, retries=RETRIES if args.watch else 0) as client:
|
|
if not args.watch:
|
|
console.print(_status_screen(client))
|
|
return 0
|
|
if not sys.stdout.isatty():
|
|
return _fail("--watch needs a terminal; without one, drop it")
|
|
if getattr(args, "local", False):
|
|
# An in-process engine is this command and nothing else, so
|
|
# nothing can change under it while it watches.
|
|
return _fail("--watch wants an engine that outlives it; drop --local")
|
|
with Live(_status_screen(client), console=console) as live:
|
|
while True:
|
|
time.sleep(WATCH_INTERVAL_S)
|
|
live.update(_status_screen(client))
|
|
except KeyboardInterrupt:
|
|
return 130
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc)
|
|
|
|
|
|
#: How much of a run's inputs the list shows. A flow taking a few kB of JSON
|
|
#: would otherwise make the table unreadable; `client.runs()` is where the
|
|
#: whole value is read.
|
|
PARAMS_WIDTH = 80
|
|
|
|
|
|
def _stamp(row: dict[str, Any]) -> str:
|
|
"""What code a run ran: the commit, whether it was dirty, and the digest.
|
|
|
|
The digest is the part that separates two runs of one uncommitted tree —
|
|
the commit says `-dirty` for both, however much changed in between.
|
|
"""
|
|
commit, dirty, _ = (row.get("origin_commit") or "").partition("-dirty")
|
|
digest = row.get("code_digest") or ""
|
|
return (
|
|
commit[:7] + ("-dirty" if dirty else "") + (f"+{digest[:7]}" if digest else "")
|
|
)
|
|
|
|
|
|
def cmd_runs(args: argparse.Namespace) -> int:
|
|
try:
|
|
with _client_for(args, retries=0) as client:
|
|
rows = client.runs(flow=args.flow, limit=args.limit)
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc)
|
|
for row in rows:
|
|
params = json.dumps(row["params"])
|
|
if len(params) > PARAMS_WIDTH:
|
|
params = params[: PARAMS_WIDTH - 3] + "..."
|
|
_say(
|
|
f"{row['id']} {_status(row['status'], 9)} {row['flow']:<16} "
|
|
f"{row['duration_ms'] / 1000:7.1f}s {_stamp(row):<22} {params}"
|
|
)
|
|
return 0
|
|
|
|
|
|
def cmd_flavors(args: argparse.Namespace) -> int:
|
|
"""The named sizes a node can ask for."""
|
|
try:
|
|
with _client_for(args, retries=0) as client:
|
|
rows = client.flavors()
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc)
|
|
if not rows:
|
|
_say("No flavors. A node can still say cpus and gpus itself.")
|
|
return 0
|
|
for row in rows:
|
|
cards = f"{row['gpus']:>3} gpu" if row.get("gpus") else " " * 7
|
|
_say(
|
|
f"{row['name']:<14}{row['cpus']:>3} cpu {row['ram'] // 1024:>5} GB"
|
|
f"{cards} {row.get('description', '')}"
|
|
)
|
|
return 0
|
|
|
|
|
|
def _grid(
|
|
definition: dict[str, Any], values: list[str], seed: int | None
|
|
) -> list[dict[str, Any]]:
|
|
"""`--param lr=0.1,0.01 --param epochs=10,50` — every combination of them."""
|
|
types = _input_types(definition)
|
|
names: list[str] = []
|
|
columns: list[list[Any]] = []
|
|
for raw in values:
|
|
name, sep, listed = raw.partition("=")
|
|
name = name.replace("-", "_")
|
|
if not sep or not listed:
|
|
raise SyncError(f"--param takes name=value,value — got '{raw}'")
|
|
if name not in types:
|
|
raise SyncError(
|
|
f"'{name}' is not an input of this flow (it takes "
|
|
f"{', '.join(sorted(types)) or 'none'})"
|
|
)
|
|
names.append(name)
|
|
columns.append([_coerce(item, types[name]) for item in listed.split(",")])
|
|
return [
|
|
{"params": dict(zip(names, combination, strict=True)), "seed": seed}
|
|
for combination in itertools.product(*columns)
|
|
]
|
|
|
|
|
|
def cmd_sweep(args: argparse.Namespace) -> int:
|
|
wait = args.wait or args.local
|
|
try:
|
|
with _client_for(args) as client:
|
|
if not args.no_sync:
|
|
_sync_first(client)
|
|
stored = client.get_flow(args.flow)
|
|
if stored is None:
|
|
return _fail(f"no flow '{args.flow}' on that engine")
|
|
entries = _grid(stored.get("definition") or {}, args.param, args.seed)
|
|
handles = client.sweep(args.flow, entries, no_cache=args.no_cache)
|
|
for handle, entry in zip(handles, entries, strict=True):
|
|
_say(f"{handle.id} queued {json.dumps(entry['params'])}")
|
|
if not wait:
|
|
return 0
|
|
failed = 0
|
|
try:
|
|
for handle in handles:
|
|
handle.wait(timeout=args.timeout)
|
|
_say(
|
|
f"{handle.id} {_status(handle.status)} "
|
|
f"{json.dumps(handle.result)}"
|
|
)
|
|
failed += handle.status != "ok"
|
|
except KeyboardInterrupt:
|
|
for handle in handles:
|
|
if not handle.refresh().done:
|
|
_cancel(client, handle)
|
|
return 130
|
|
return 1 if failed else 0
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc, "The runs are still on the engine; `fluksio runs`.")
|
|
|
|
|
|
def _selection(args: argparse.Namespace) -> dict[str, Any]:
|
|
"""The filters both exports share, minus the ones left empty."""
|
|
return {
|
|
key: value
|
|
for key, value in (
|
|
("status", args.status),
|
|
("group", args.group),
|
|
("since", args.since),
|
|
("until", args.until),
|
|
)
|
|
if value
|
|
}
|
|
|
|
|
|
def _write_rows(rows: list[dict[str, Any]], fmt: str, out: str, hint: str = "") -> int:
|
|
"""The exported rows, in the format asked for, to a file or to stdout.
|
|
|
|
The engine settles the columns over the whole selection before it sends
|
|
anything, so every row carries the same keys in the same order and the
|
|
first row's are the header.
|
|
"""
|
|
if not rows:
|
|
empty = "fluksio: nothing matched, so nothing was written"
|
|
print(empty + (f". {hint}" if hint else ""), file=sys.stderr)
|
|
return 0
|
|
if fmt == "parquet":
|
|
try:
|
|
import pyarrow
|
|
import pyarrow.parquet
|
|
except ImportError:
|
|
return _fail(
|
|
"parquet needs pyarrow — `pip install 'fluksio[parquet]'`, or "
|
|
"export csv and convert it"
|
|
)
|
|
pyarrow.parquet.write_table(pyarrow.Table.from_pylist(rows), out)
|
|
return 0
|
|
with open(out, "w", newline="") if out else nullcontext(sys.stdout) as handle:
|
|
if fmt == "jsonl":
|
|
handle.writelines(json.dumps(row) + "\n" for row in rows)
|
|
else:
|
|
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
return 0
|
|
|
|
|
|
#: How many runs `--list` reads before giving up on finding a metric name.
|
|
#: The names belong to the flow's nodes rather than to a run, so the newest
|
|
#: one that recorded any is the whole vocabulary — the rest are for a
|
|
#: selection whose newest runs failed before they measured anything.
|
|
# ponytail: the first run with names wins; a name only an older run recorded
|
|
# is not listed. A `distinct` over the selection would be exact and is a route
|
|
# of its own.
|
|
LIST_SCAN = 10
|
|
|
|
|
|
def _list_names(client: Client, args: argparse.Namespace) -> int:
|
|
"""The metric names these runs carry, since a name is flow-qualified.
|
|
|
|
`train_loss` is recorded as `train.train_loss`, and asking for the bare
|
|
one matches nothing — so this is the answer to "what would match".
|
|
"""
|
|
filters = _selection(args)
|
|
if "until" in filters:
|
|
# The history spells the same bound `before`, where it is also the
|
|
# cursor a page is taken from.
|
|
filters["before"] = filters.pop("until")
|
|
ids = args.run or [
|
|
row["id"] for row in client.runs(flow=args.flow, limit=LIST_SCAN, **filters)
|
|
]
|
|
for run_id in ids[:LIST_SCAN]:
|
|
names = sorted({point["name"] for point in client.metrics(run_id)})
|
|
if names:
|
|
_say("\n".join(names))
|
|
return 0
|
|
_say("No metrics recorded by these runs.")
|
|
return 0
|
|
|
|
|
|
def _too_old(client: Client) -> str:
|
|
"""A route this client knows and the engine does not."""
|
|
try:
|
|
version = (client.summary() or {}).get("version") or ""
|
|
except (ApiError, httpx.HTTPError):
|
|
version = ""
|
|
engine = f"the engine is {version}" if version else "the engine is older"
|
|
return (
|
|
f"this engine has no export endpoints — {engine} and this client is "
|
|
f"{__version__}. Upgrade it: `pip install -U fluksio`"
|
|
)
|
|
|
|
|
|
def _export(
|
|
args: argparse.Namespace, fetch: Callable[[Client], list[dict[str, Any]]]
|
|
) -> int:
|
|
"""Both exports: fetch what was asked for, then write it."""
|
|
if args.format == "parquet" and not args.out:
|
|
return _fail("--format parquet writes a file; name it with -o FILE")
|
|
hint = (
|
|
"`--list` names the metrics these runs carry"
|
|
if getattr(args, "name", "")
|
|
else ""
|
|
)
|
|
try:
|
|
with _client_for(args, retries=0) as client:
|
|
if getattr(args, "list_names", False):
|
|
return _list_names(client, args)
|
|
try:
|
|
rows = fetch(client)
|
|
except ApiError as exc:
|
|
if exc.status != 404:
|
|
raise
|
|
return _fail(_too_old(client))
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
except httpx.HTTPError as exc:
|
|
return _unreachable(exc)
|
|
return _write_rows(rows, args.format, args.out, hint)
|
|
|
|
|
|
def cmd_export_metrics(args: argparse.Namespace) -> int:
|
|
"""Every selected run's numbers as one long table."""
|
|
return _export(
|
|
args,
|
|
lambda client: client.export_metrics(
|
|
flow=args.flow,
|
|
ids=args.run,
|
|
name=args.name,
|
|
stride=args.stride,
|
|
**_selection(args),
|
|
),
|
|
)
|
|
|
|
|
|
def cmd_export_runs(args: argparse.Namespace) -> int:
|
|
"""One row per run: its inputs, its final numbers, what it ran."""
|
|
return _export(
|
|
args,
|
|
lambda client: client.export_runs(
|
|
flow=args.flow,
|
|
ids=args.run,
|
|
params=args.params,
|
|
metrics=args.metrics,
|
|
**_selection(args),
|
|
),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Wiring
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def add_parsers(subparsers: Any) -> None:
|
|
"""Register the client commands on `fluksio`'s parser."""
|
|
|
|
def with_engine(sub: argparse.ArgumentParser, local: bool = False) -> None:
|
|
sub.add_argument(
|
|
"--url", default="", help="the engine (default: the last login)"
|
|
)
|
|
sub.add_argument("--token", default="", help="override the stored token")
|
|
if local:
|
|
sub.add_argument(
|
|
"--local",
|
|
action="store_true",
|
|
help="boot the engine in this process instead of talking to one",
|
|
)
|
|
|
|
parser = subparsers.add_parser(
|
|
"login", help="store a token for an engine elsewhere"
|
|
)
|
|
parser.add_argument("--url", default="http://localhost:8000")
|
|
parser.add_argument("--email", default="")
|
|
parser.add_argument("--password", default="")
|
|
parser.add_argument(
|
|
"--global",
|
|
dest="shared",
|
|
action="store_true",
|
|
help="store it for the machine rather than this project",
|
|
)
|
|
parser.set_defaults(func=cmd_login)
|
|
|
|
parser = subparsers.add_parser(
|
|
"sync", help="upload the flows declared in your own code"
|
|
)
|
|
parser.add_argument(
|
|
"targets",
|
|
nargs="*",
|
|
help="modules, packages or directories to import (default: .)",
|
|
)
|
|
parser.add_argument(
|
|
"--dry-run", action="store_true", help="print what would be uploaded"
|
|
)
|
|
parser.add_argument(
|
|
"--no-publish", action="store_true", help="leave the changes as a draft"
|
|
)
|
|
parser.add_argument(
|
|
"--force", action="store_true", help="overwrite work done on the canvas"
|
|
)
|
|
with_engine(parser)
|
|
parser.set_defaults(func=cmd_sync)
|
|
|
|
parser = subparsers.add_parser(
|
|
"run", help="sync this directory, then start a run of one of its flows"
|
|
)
|
|
parser.add_argument("flow")
|
|
parser.add_argument(
|
|
"--seed",
|
|
type=int,
|
|
default=None,
|
|
help=(
|
|
"the run's seed: recorded on it, part of what tells two runs of one "
|
|
"configuration apart, and passed to an input named 'seed' when the "
|
|
"flow declares one"
|
|
),
|
|
)
|
|
parser.add_argument("--wait", action="store_true", help="block until it finishes")
|
|
parser.add_argument(
|
|
"--follow",
|
|
action="store_true",
|
|
help="wait, printing the numbers it reports as they arrive",
|
|
)
|
|
parser.add_argument(
|
|
"--timeout",
|
|
type=float,
|
|
default=0.0,
|
|
metavar="SECONDS",
|
|
help=(
|
|
"give up waiting after this long and leave the run going; 0, the "
|
|
"default, waits as long as it takes. Not the node timeout"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--no-sync",
|
|
action="store_true",
|
|
help="run what is already on the engine, without uploading first",
|
|
)
|
|
parser.add_argument(
|
|
"--no-cache",
|
|
action="store_true",
|
|
help="execute every node, even one an earlier run already answered",
|
|
)
|
|
parser.add_argument(
|
|
"--defaults",
|
|
action="store_true",
|
|
help="take every input's declared value instead of asking for it",
|
|
)
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_run)
|
|
|
|
parser = subparsers.add_parser(
|
|
"status", help="how the engine is doing, and what each flow is up to"
|
|
)
|
|
parser.add_argument(
|
|
"--watch",
|
|
action="store_true",
|
|
help="keep it on screen, refreshed until Ctrl-C",
|
|
)
|
|
# `--local` for the same reason `runs` has it: the flows and the history are
|
|
# in this directory, and reading them should not need a server.
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_status)
|
|
|
|
parser = subparsers.add_parser("runs", help="the runs an engine has recorded")
|
|
parser.add_argument("--flow", default="")
|
|
parser.add_argument("--limit", type=int, default=20)
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_runs)
|
|
|
|
parser = subparsers.add_parser(
|
|
"flavors", help="the named resource sizes a node can ask for"
|
|
)
|
|
with_engine(parser)
|
|
parser.set_defaults(func=cmd_flavors)
|
|
|
|
parser = subparsers.add_parser(
|
|
"sweep", help="one flow, once per combination of the parameters given"
|
|
)
|
|
parser.add_argument("flow")
|
|
parser.add_argument(
|
|
"--param",
|
|
action="append",
|
|
default=[],
|
|
metavar="NAME=V1,V2",
|
|
help="an input and the values to try; repeat for a grid",
|
|
)
|
|
parser.add_argument(
|
|
"--seed",
|
|
type=int,
|
|
default=None,
|
|
help="the seed every run in the sweep gets; vary it with --param seed=1,2",
|
|
)
|
|
parser.add_argument("--wait", action="store_true", help="block until all finish")
|
|
parser.add_argument(
|
|
"--timeout",
|
|
type=float,
|
|
default=0.0,
|
|
metavar="SECONDS",
|
|
help="give up waiting after this long; 0, the default, waits them out",
|
|
)
|
|
parser.add_argument(
|
|
"--no-sync",
|
|
action="store_true",
|
|
help="run what is already on the engine, without uploading first",
|
|
)
|
|
parser.add_argument(
|
|
"--no-cache",
|
|
action="store_true",
|
|
help="execute every node, even one an earlier run already answered",
|
|
)
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_sweep)
|
|
|
|
parser = subparsers.add_parser(
|
|
"export", help="runs and their numbers as a table an analysis reads"
|
|
)
|
|
exports = parser.add_subparsers(dest="table", required=True)
|
|
|
|
def with_selection(sub: argparse.ArgumentParser) -> None:
|
|
sub.add_argument("--flow", default="", help="only runs of this flow")
|
|
sub.add_argument(
|
|
"--run",
|
|
action="append",
|
|
default=[],
|
|
metavar="ID",
|
|
help="only this run; repeat for several",
|
|
)
|
|
sub.add_argument("--group", default="", help="only the runs of this sweep")
|
|
sub.add_argument("--status", default="", help="only runs that ended this way")
|
|
sub.add_argument(
|
|
"--since", default="", metavar="TS", help="only runs created at or after"
|
|
)
|
|
sub.add_argument(
|
|
"--until", default="", metavar="TS", help="only runs created before"
|
|
)
|
|
sub.add_argument("--format", default="csv", choices=("csv", "jsonl", "parquet"))
|
|
sub.add_argument(
|
|
"-o", "--out", default="", metavar="FILE", help="write here, not to stdout"
|
|
)
|
|
with_engine(sub, local=True)
|
|
|
|
sub = exports.add_parser(
|
|
"metrics", help="one row per run, metric and step — the tidy shape"
|
|
)
|
|
with_selection(sub)
|
|
sub.add_argument("--name", default="", metavar="A,B", help="only these metrics")
|
|
sub.add_argument(
|
|
"--stride",
|
|
type=int,
|
|
default=1,
|
|
help="keep every Nth point of each curve",
|
|
)
|
|
sub.add_argument(
|
|
"--list",
|
|
dest="list_names",
|
|
action="store_true",
|
|
help="print the metric names these runs carry, and stop",
|
|
)
|
|
sub.set_defaults(func=cmd_export_metrics)
|
|
|
|
sub = exports.add_parser(
|
|
"runs", help="one row per run: its inputs, its final numbers, its status"
|
|
)
|
|
with_selection(sub)
|
|
sub.add_argument(
|
|
"--params",
|
|
default="",
|
|
metavar="A,B",
|
|
help=(
|
|
"the inputs to put in columns, dotted into a record "
|
|
"(default: the ones that vary)"
|
|
),
|
|
)
|
|
sub.add_argument(
|
|
"--metrics",
|
|
default="",
|
|
metavar="A,B",
|
|
help="the final numbers to keep, e.g. final_metrics.train_loss",
|
|
)
|
|
sub.set_defaults(func=cmd_export_runs)
|