stroblmeandClaude Opus 5
3c15964364
Docs / docs (push) Successful in 22s
Playwright Tests / test-playwright (1, 2) (push) Failing after 2m42s
Playwright Tests / test-playwright (2, 2) (push) Failing after 1m41s
pre-commit / pre-commit (push) Failing after 2m53s
Test Backend / test-backend (push) Successful in 2m21s
Compose Smoke Test / test-compose (push) Successful in 31s
Playwright Tests / merge-reports (push) Failing after 1m6s
A cache hit still replays no emissions — those values were the story of an
execution that is not happening — but the run they were recorded in is now
written on the row (`run_node.cached_from`), and the metrics endpoints read the
series back from there. So a reused run answers `run.metrics("train.loss")`
with the same points the run that trained did, rather than looking like a run
that produced no numbers at all. Pointed at rather than copied: a sweep of 500
reusing one frozen node would otherwise duplicate its curve 500 times.
That needed the cross-flow restore fixed first. The cache key has no flow in
it while the stored outputs are named for the flow that produced them, so
`quick.prepare` getting a hit from `train` wrote `train.dataset` into `quick`'s
state and the next node was called without its argument. One rule now covers
both halves: `requalify` reads a name owned by one flow as the same name in
another, applied to the restored outputs, to the node id behind the pointer,
and to the series names on the way out. Reuse across flows is kept.
Also: `@run:<id>.<output>` and a bare `sha256:` digest resolve on every input,
not only artifacts. Chaining a run's json config into the next one from a shell
meant pasting the whole object inline, and the CLI could not even send the
spelling — `_coerce` died in `json.loads` before the engine saw it. Both
spellings are reserved on every input now, `str` included, and `_from_run`
returns whatever the run's result holds rather than only a reference.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Dp9L6gakMVro1K2C5zdtBE
884 lines
31 KiB
Python
884 lines
31 KiB
Python
"""The `fluksio sync`, `run`, `runs`, `sweep`, `status` and `login` commands.
|
|
|
|
Kept beside the SDK rather than in `fluksio.cli`: these are the client half of
|
|
the tool, and none of them needs the engine to be importable — `--local`, which
|
|
does, imports it inside the branch that asked for it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import getpass
|
|
import importlib
|
|
import itertools
|
|
import json
|
|
import pkgutil
|
|
import sys
|
|
import time
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from fluksio.sdk import FLOWS, Flow, SyncError
|
|
from fluksio.sdk.client import (
|
|
GLOBAL_DATA_DIR,
|
|
ApiError,
|
|
Client,
|
|
RunHandle,
|
|
data_dir,
|
|
login,
|
|
origin_of,
|
|
repo_root,
|
|
sync,
|
|
)
|
|
|
|
__all__ = ["add_parsers", "discover"]
|
|
|
|
#: Statuses worth a colour, and the SGR code each gets.
|
|
_COLORS = {
|
|
"ok": "32",
|
|
"error": "31",
|
|
"cached": "36",
|
|
"cancelled": "33",
|
|
"abandoned": "31",
|
|
"running": "36",
|
|
"queued": "33",
|
|
}
|
|
|
|
|
|
def _say(message: str = "") -> None:
|
|
print(message)
|
|
|
|
|
|
def _fail(message: str) -> int:
|
|
print(f"fluksio: {message}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
def _status(text: str, width: int = 0) -> str:
|
|
"""A status, coloured when a terminal is reading it.
|
|
|
|
Padded before it is coloured: the escape sequences are characters as far
|
|
as `str.format` is concerned, and a column that lines up in a pipe would
|
|
not line up on screen.
|
|
"""
|
|
body = f"{text:<{width}}" if width else text
|
|
code = _COLORS.get(text)
|
|
if not code or not sys.stdout.isatty():
|
|
return body
|
|
return f"\033[{code}m{body}\033[0m"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Discovery
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _package_of(directory: Path) -> tuple[str, str]:
|
|
"""The path root and dotted name of a package directory."""
|
|
parts = [directory.name]
|
|
parent = directory.parent
|
|
while (parent / "__init__.py").exists():
|
|
parts.append(parent.name)
|
|
parent = parent.parent
|
|
return str(parent), ".".join(reversed(parts))
|
|
|
|
|
|
def _module_of(path: Path) -> tuple[str, str]:
|
|
"""The path root and dotted name of a module file."""
|
|
parts = [path.stem]
|
|
directory = path.parent
|
|
while (directory / "__init__.py").exists():
|
|
parts.append(directory.name)
|
|
directory = directory.parent
|
|
return str(directory), ".".join(reversed(parts))
|
|
|
|
|
|
def _import(root: str, dotted: str) -> None:
|
|
if root not in sys.path:
|
|
sys.path.insert(0, root)
|
|
importlib.import_module(dotted)
|
|
|
|
|
|
def discover(targets: list[str]) -> list[Flow]:
|
|
"""Import what was named and hand back the flows it declared.
|
|
|
|
Imported by dotted name with its root on the path, never from a file
|
|
location: the generated node bodies import the same way, and a module
|
|
loaded under a different name would generate an import that does not
|
|
resolve.
|
|
"""
|
|
for target in targets:
|
|
path = Path(target)
|
|
if not path.exists():
|
|
_import(str(Path.cwd()), target)
|
|
continue
|
|
path = path.resolve()
|
|
if path.is_file():
|
|
_import(*_module_of(path))
|
|
continue
|
|
if (path / "__init__.py").exists():
|
|
root, dotted = _package_of(path)
|
|
_import(root, dotted)
|
|
package = sys.modules[dotted]
|
|
for info in pkgutil.walk_packages(package.__path__, f"{dotted}."):
|
|
importlib.import_module(info.name)
|
|
continue
|
|
# A plain directory — a repository root, usually. Its own modules,
|
|
# and the packages inside it: `myresearch/` beside a `README` is the
|
|
# ordinary shape, and naming it explicitly should not be the price of
|
|
# keeping your code in a package.
|
|
for module in sorted(path.glob("*.py")):
|
|
_import(*_module_of(module))
|
|
for child in sorted(path.iterdir()):
|
|
if child.name.startswith(".") or not (child / "__init__.py").exists():
|
|
continue
|
|
root, dotted = _package_of(child)
|
|
_import(root, dotted)
|
|
for info in pkgutil.walk_packages(
|
|
sys.modules[dotted].__path__, f"{dotted}."
|
|
):
|
|
importlib.import_module(info.name)
|
|
return list(FLOWS.values())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Which engine
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@contextmanager
|
|
def _engine_client() -> Iterator[Client]:
|
|
"""The engine itself, in this process, behind the ordinary client.
|
|
|
|
Everything `fluksio serve` does apart from listening on a socket: the same
|
|
data directory, the same database, the same admin, the same lifespan. The
|
|
app is driven through its ASGI interface, so a run costs what it costs on
|
|
a served engine and lands in the same history — which is what makes the
|
|
stage cache carry across the two.
|
|
|
|
The engine only exists for the length of the command, so nothing here is
|
|
written back as a login: a stored token belongs to whichever engine
|
|
`fluksio login` was pointed at.
|
|
"""
|
|
# Imported here, not at module scope: everything else in this file is the
|
|
# client half and must keep working with no engine installed.
|
|
import logging
|
|
|
|
from fluksio.cli import _data_dir, _prepare, _print_new_admin
|
|
|
|
directory = _data_dir(None)
|
|
# Settings are read when the app is imported, so this comes first.
|
|
_prepare(directory)
|
|
# Nothing here goes over a network, so httpx logging each call as a
|
|
# request to "testserver" is noise that also happens to be untrue.
|
|
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
|
|
from datetime import timedelta
|
|
|
|
from fastapi.testclient import TestClient
|
|
from sqlmodel import Session
|
|
|
|
from fluksio.core import security
|
|
from fluksio.core.bootstrap import ensure_superuser
|
|
from fluksio.core.db import engine
|
|
|
|
with Session(engine) as session:
|
|
admin, generated = ensure_superuser(session)
|
|
admin_id = admin.id
|
|
if generated:
|
|
_print_new_admin(admin.email, generated)
|
|
token = security.create_access_token(admin_id, expires_delta=timedelta(hours=12))
|
|
|
|
from fluksio.main import app
|
|
|
|
# Entering the client is what runs the lifespan: the worker pool, the
|
|
# controller and the run service all start here and stop on the way out.
|
|
with TestClient(app) as http:
|
|
yield Client(http=http, token=token)
|
|
|
|
|
|
@contextmanager
|
|
def _client_for(args: argparse.Namespace) -> Iterator[Client]:
|
|
"""The engine this command talks to: one running somewhere, or this one."""
|
|
if getattr(args, "local", False):
|
|
with _engine_client() as client:
|
|
yield client
|
|
else:
|
|
yield Client(url=args.url, token=args.token)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Commands
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def cmd_login(args: argparse.Namespace) -> int:
|
|
"""Only needed for an engine somewhere else — `serve` signs you in here."""
|
|
email = args.email or input("Email: ")
|
|
password = args.password or getpass.getpass("Password: ")
|
|
directory = GLOBAL_DATA_DIR.expanduser() if args.shared else data_dir()
|
|
try:
|
|
path = login(args.url, email, password, directory=directory)
|
|
except ApiError as exc:
|
|
return _fail(f"could not log in: {exc.detail}")
|
|
_say(f"Logged in to {args.url}; the token is in {path}.")
|
|
return 0
|
|
|
|
|
|
def cmd_sync(args: argparse.Namespace) -> int:
|
|
targets = args.targets or ["."]
|
|
try:
|
|
flows = discover(targets)
|
|
except (ImportError, SyncError) as exc:
|
|
return _fail(str(exc))
|
|
if not flows:
|
|
return _fail(
|
|
f"no flows declared in {', '.join(targets)} — a flow is a `Flow(...)` "
|
|
"at module level. Name the package if it is somewhere else: "
|
|
"`fluksio sync src/myresearch`."
|
|
)
|
|
|
|
repo = repo_root(targets[0])
|
|
origin = origin_of(repo)
|
|
if origin["dirty"]:
|
|
_say(f"warning: {repo} has uncommitted changes, so the stamp says -dirty")
|
|
if not origin["commit"]:
|
|
_say(f"warning: {repo} is not a git repository, so runs cannot name a commit")
|
|
|
|
if args.dry_run:
|
|
for target in flows:
|
|
_say(f"=== flow {target.name}")
|
|
_say(json.dumps(target.document(origin), indent=2))
|
|
for node_id, code in target.shims().items():
|
|
_say(f"=== {target.name}.{node_id}")
|
|
_say(code)
|
|
return 0
|
|
|
|
try:
|
|
client = Client(url=args.url, token=args.token)
|
|
reports = sync(
|
|
flows,
|
|
client,
|
|
origin=origin,
|
|
publish=not args.no_publish,
|
|
force=args.force,
|
|
)
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
|
|
for report in reports:
|
|
if report.unchanged:
|
|
_say(f" {report.flow}: unchanged")
|
|
continue
|
|
what = "created" if report.created else "updated"
|
|
detail = ", ".join(report.changed)
|
|
state = "published" if report.published else "draft"
|
|
_say(f" {report.flow}: {what} ({detail}) — {state}")
|
|
stamp = origin["commit"][:7] + ("-dirty" if origin["dirty"] else "")
|
|
_say(f"Stamped with {stamp or 'no commit'} from {repo}.")
|
|
return 0
|
|
|
|
|
|
def _coerce(value: str, dtype: str) -> Any:
|
|
if value.startswith("@run:") or value.startswith("sha256:"):
|
|
# A run's output, named rather than typed out. Sent as it stands: the
|
|
# engine turns it into the value itself, whatever type that is. Passing
|
|
# the whole thing as JSON still works, and is what a script that
|
|
# already has it in hand would do.
|
|
return value
|
|
if dtype == "int":
|
|
return int(value)
|
|
if dtype == "float":
|
|
return float(value)
|
|
if dtype == "bool":
|
|
return value.lower() in ("true", "1", "yes", "on")
|
|
if dtype == "str":
|
|
return value
|
|
return json.loads(value)
|
|
|
|
|
|
def _input_types(definition: dict[str, Any]) -> dict[str, str]:
|
|
"""What each of a flow's inputs is declared to be."""
|
|
return {
|
|
str(entry["spec"]["name"]): str(entry["spec"].get("dtype", "float"))
|
|
for entry in definition.get("inputs") or []
|
|
}
|
|
|
|
|
|
def _ask_params(definition: dict[str, Any]) -> dict[str, Any]:
|
|
"""The run dialog, in a terminal: one line per declared input.
|
|
|
|
An empty answer leaves the input out, which is what keeps its declared
|
|
value — the same thing the browser's dialog does with a field nobody
|
|
filled in. So pressing Enter through the lot runs the defaults.
|
|
"""
|
|
params: dict[str, Any] = {}
|
|
for entry in definition.get("inputs") or []:
|
|
spec = entry.get("spec") or {}
|
|
name = str(spec.get("name", ""))
|
|
if not name:
|
|
continue
|
|
dtype = str(spec.get("dtype", "float"))
|
|
initial = entry.get("initial")
|
|
shown = "" if initial is None else json.dumps(initial)
|
|
prompt = f"{name} ({dtype})" + (f" [{shown}]" if shown else "") + ": "
|
|
answer = input(prompt).strip()
|
|
if not answer:
|
|
continue
|
|
try:
|
|
params[name] = _coerce(answer, dtype)
|
|
except ValueError as exc:
|
|
raise SyncError(f"'{name}' takes {dtype}: {exc}") from exc
|
|
return params
|
|
|
|
|
|
def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
|
|
"""Turn `--lr 0.05` into a typed parameter, using the flow's own inputs."""
|
|
types = _input_types(definition)
|
|
params: dict[str, Any] = {}
|
|
pending: str | None = None
|
|
for token in rest:
|
|
if token.startswith("--"):
|
|
if pending is not None:
|
|
# A flag with no value is a flag: `--resume` means true.
|
|
params[pending] = True
|
|
name, sep, value = token[2:].partition("=")
|
|
# Only the name is spelled with dashes; a value may hold one, and
|
|
# `--lr=1e-4` is the case that says so.
|
|
pending = name.replace("-", "_")
|
|
if sep:
|
|
params[pending] = _coerce(value, types.get(pending, "json"))
|
|
pending = None
|
|
continue
|
|
if pending is None:
|
|
raise SyncError(f"unexpected argument '{token}'")
|
|
params[pending] = _coerce(token, types.get(pending, "json"))
|
|
pending = None
|
|
if pending is not None:
|
|
params[pending] = True
|
|
unknown = sorted(set(params) - set(types))
|
|
if unknown:
|
|
raise SyncError(
|
|
f"'{unknown[0]}' is not an input of this flow (it takes "
|
|
f"{', '.join(sorted(types)) or 'none'})"
|
|
)
|
|
return params
|
|
|
|
|
|
def _sync_first(client: Client) -> None:
|
|
"""Upload what the working directory declares, before running it.
|
|
|
|
The reason a run exists is usually the edit that came before it, and
|
|
remembering to sync is remembering to do the thing the computer could have
|
|
done. So `run` syncs by default — including the worker refresh, which is
|
|
what makes an edit to your own package take effect at all.
|
|
|
|
A directory that declares nothing is not an error: a flow drawn on the
|
|
canvas is run the same way, and has nothing to upload.
|
|
"""
|
|
try:
|
|
flows = discover(["."])
|
|
except (ImportError, SyncError) as exc:
|
|
# Do not fail a run for a module the run may not even need.
|
|
_say(f"warning: nothing synced — {exc}")
|
|
return
|
|
if not flows:
|
|
return
|
|
repo = repo_root(".")
|
|
reports = sync(flows, client, origin=origin_of(repo))
|
|
changed = [r for r in reports if not r.unchanged]
|
|
if changed:
|
|
_say(f"synced {', '.join(r.flow for r in changed)}")
|
|
|
|
|
|
def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
|
|
"""Print a run's numbers as they arrive, until it is over.
|
|
|
|
Polled rather than pushed: the engine writes a metric down when it is
|
|
reported, so asking once a second draws the same curve a socket would
|
|
have, without either side holding a connection open. The status is read
|
|
before the numbers, so the last batch is never the one that gets missed.
|
|
"""
|
|
seen: set[tuple[str, int]] = set()
|
|
while True:
|
|
done = handle.refresh().done
|
|
for point in client.metrics(handle.id):
|
|
mark = (str(point.get("name", "")), int(point.get("step", -1)))
|
|
if mark in seen:
|
|
continue
|
|
seen.add(mark)
|
|
_say(f" {mark[0]}[{mark[1]}] = {point['value']:g}")
|
|
if done:
|
|
return
|
|
time.sleep(poll)
|
|
|
|
|
|
def _cancel(client: Client, handle: RunHandle) -> int:
|
|
"""Ctrl-C means stop the run, not just stop watching it."""
|
|
try:
|
|
client.cancel(handle.id)
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(f"could not cancel {handle.id}: {exc}")
|
|
_say(f"{handle.id} {_status('cancelled')}")
|
|
return 130
|
|
|
|
|
|
def _cached_note(client: Client, handle: RunHandle) -> str:
|
|
"""How much of the run earlier ones had already answered."""
|
|
try:
|
|
nodes = client.run(handle.id).get("nodes") or []
|
|
except (SyncError, ApiError):
|
|
return ""
|
|
cached = sum(1 for node in nodes if node.get("status") == "cached")
|
|
return f" ({cached}/{len(nodes)} {_status('cached')})" if cached else ""
|
|
|
|
|
|
def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
|
|
# An in-process engine lives exactly as long as this command, so a run
|
|
# nobody waits for would be thrown away with the queue holding it.
|
|
wait = args.wait or args.follow or args.local
|
|
try:
|
|
with _client_for(args) as client:
|
|
if not args.no_sync:
|
|
_sync_first(client)
|
|
stored = client.get_flow(args.flow)
|
|
if stored is None:
|
|
return _fail(f"no flow '{args.flow}' on that engine")
|
|
definition = stored.get("definition") or {}
|
|
params = _params(definition, rest)
|
|
# Nothing on the command line and somebody watching: ask, the way
|
|
# pressing Run in the browser asks. A scripted run — piped, or
|
|
# carrying parameters already — is left exactly as it was.
|
|
if (
|
|
not params
|
|
and not args.defaults
|
|
and (definition.get("inputs") or [])
|
|
and sys.stdin.isatty()
|
|
):
|
|
params = _ask_params(definition)
|
|
handle = client.submit(
|
|
args.flow, params, seed=args.seed, no_cache=args.no_cache
|
|
)
|
|
_say(f"{handle.id} queued {json.dumps(params)}")
|
|
if not wait:
|
|
return 0
|
|
try:
|
|
if args.follow:
|
|
_follow(client, handle)
|
|
else:
|
|
handle.wait(timeout=args.timeout)
|
|
except KeyboardInterrupt:
|
|
return _cancel(client, handle)
|
|
_say(
|
|
f"{handle.id} {_status(handle.status)} "
|
|
f"{json.dumps(handle.result)}{_cached_note(client, handle)}"
|
|
)
|
|
return 0 if handle.status == "ok" else 1
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Status
|
|
#
|
|
# The home screen's top half, in a terminal: how the engine is, what each flow
|
|
# is doing, and what failed recently. Drawn with rich, which is already here
|
|
# under fastapi's own CLI.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
#: How often `--watch` asks again. The dashboard polls its summary every ten
|
|
#: seconds; nothing here moves faster than that.
|
|
WATCH_INTERVAL_S = 5.0
|
|
|
|
|
|
def _flow_state(flow: dict[str, Any]) -> str:
|
|
if flow.get("quarantined"):
|
|
return "quarantined"
|
|
if not flow.get("enabled", True):
|
|
return "stopped"
|
|
if flow.get("paused"):
|
|
return "paused"
|
|
return "running"
|
|
|
|
|
|
def _portal_phrase(portal: dict[str, Any]) -> tuple[str, str]:
|
|
"""Where this installation stands with its portal, and how to colour it.
|
|
|
|
Three states worth telling apart: never paired, paired and linked, and
|
|
paired but not reaching it — the last being the one somebody needs to know
|
|
about, since the dashboard is served from the other end.
|
|
"""
|
|
if not portal.get("enrolled"):
|
|
return "no portal", "dim"
|
|
if portal.get("connected"):
|
|
host = str(portal.get("portal_url") or "").split("//")[-1].rstrip("/")
|
|
return f"portal {host}", "green"
|
|
trouble = str(portal.get("last_error") or "").strip()
|
|
return "portal unreachable" + (f" ({trouble[:60]})" if trouble else ""), "red"
|
|
|
|
|
|
def _status_screen(client: Client) -> Any:
|
|
"""One frame: health, the flows, and the failures under them."""
|
|
from rich.console import Group
|
|
from rich.table import Table
|
|
from rich.text import Text
|
|
|
|
summary = client.summary()
|
|
flows = client.flows()
|
|
# Both, because an installation is usually one or the other: a live flow
|
|
# fails as an engine event, while a batch run fails on its own row.
|
|
failures = client.events(kind="failure", limit=5)
|
|
runs = client.runs(limit=5)
|
|
try:
|
|
portal = client.cloud_status()
|
|
except (SyncError, ApiError):
|
|
# Never enrolled, or an engine too old to answer. Neither is worth
|
|
# failing a status screen over.
|
|
portal = {}
|
|
|
|
healthy = summary.get("status") == "ok"
|
|
head = Text()
|
|
head.append(
|
|
"healthy" if healthy else str(summary.get("status", "unknown")),
|
|
style="green" if healthy else "yellow",
|
|
)
|
|
problems = summary.get("problems") or []
|
|
if problems:
|
|
head.append(" " + " · ".join(str(p) for p in problems), style="yellow")
|
|
phrase, style = _portal_phrase(portal)
|
|
head.append(" " + phrase, style=style)
|
|
|
|
counts = summary.get("flows") or {}
|
|
nodes = summary.get("nodes") or {}
|
|
queue = summary.get("queue") or {}
|
|
lag = summary.get("loop_lag") or {}
|
|
facts = Text(
|
|
f"{counts.get('running', 0)}/{counts.get('total', 0)} flows running "
|
|
f"{nodes.get('total', 0)} nodes"
|
|
+ (f", {nodes['error']} failed" if nodes.get("error") else "")
|
|
+ f" queue {queue.get('pending', 0)} pending"
|
|
+ (f", {queue['parked']} parked" if queue.get("parked") else "")
|
|
+ f" loop lag {lag.get('ewma', 0.0) * 1000:.0f}ms",
|
|
style="dim",
|
|
)
|
|
|
|
table = Table(box=None, pad_edge=False, header_style="dim")
|
|
table.add_column("flow")
|
|
table.add_column("state")
|
|
table.add_column("nodes", justify="right")
|
|
table.add_column("")
|
|
for flow in sorted(flows, key=lambda one: str(one.get("name", ""))):
|
|
state = _flow_state(flow)
|
|
notes = []
|
|
if flow.get("error_count"):
|
|
notes.append(f"{flow['error_count']} in error")
|
|
if flow.get("has_draft"):
|
|
notes.append("unpublished changes")
|
|
table.add_row(
|
|
str(flow.get("title") or flow.get("name", "")),
|
|
Text(
|
|
state,
|
|
style={"running": "green", "paused": "yellow"}.get(state, "red"),
|
|
),
|
|
str(flow.get("node_count", 0)),
|
|
Text(" · ".join(notes), style="red" if flow.get("error_count") else "dim"),
|
|
)
|
|
|
|
parts: list[Any] = [head, facts, ""]
|
|
parts.append(
|
|
table if flows else Text("No flows yet. `fluksio sync` uploads yours.", "dim")
|
|
)
|
|
if runs:
|
|
parts += ["", Text("recent runs", style="dim")]
|
|
for row in runs:
|
|
state = str(row.get("status", ""))
|
|
parts.append(
|
|
Text(f" {str(row.get('id', ''))[-8:]} ")
|
|
+ Text(
|
|
f"{state:<9}",
|
|
style={"ok": "green", "cached": "cyan", "running": "cyan"}.get(
|
|
state, "red" if state == "error" else "yellow"
|
|
),
|
|
)
|
|
+ Text(
|
|
f"{str(row.get('flow', '')):<16}"
|
|
f"{(row.get('duration_ms') or 0) / 1000:7.1f}s",
|
|
style="dim",
|
|
)
|
|
)
|
|
if failures:
|
|
parts += ["", Text("recent failures", style="dim")]
|
|
for event in failures:
|
|
where = " ".join(
|
|
str(event.get(key, "")) for key in ("flow", "node") if event.get(key)
|
|
)
|
|
parts.append(
|
|
Text(f" {where} ", style="red")
|
|
+ Text(str(event.get("detail", ""))[:100], style="dim")
|
|
)
|
|
return Group(*parts)
|
|
|
|
|
|
def cmd_status(args: argparse.Namespace) -> int:
|
|
"""How the engine is doing, once or until Ctrl-C."""
|
|
from rich.console import Console
|
|
from rich.live import Live
|
|
|
|
console = Console()
|
|
try:
|
|
with _client_for(args) as client:
|
|
if not args.watch:
|
|
console.print(_status_screen(client))
|
|
return 0
|
|
if not sys.stdout.isatty():
|
|
return _fail("--watch needs a terminal; without one, drop it")
|
|
if getattr(args, "local", False):
|
|
# An in-process engine is this command and nothing else, so
|
|
# nothing can change under it while it watches.
|
|
return _fail("--watch wants an engine that outlives it; drop --local")
|
|
with Live(_status_screen(client), console=console) as live:
|
|
while True:
|
|
time.sleep(WATCH_INTERVAL_S)
|
|
live.update(_status_screen(client))
|
|
except KeyboardInterrupt:
|
|
return 130
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
|
|
|
|
def cmd_runs(args: argparse.Namespace) -> int:
|
|
try:
|
|
with _client_for(args) as client:
|
|
rows = client.runs(flow=args.flow, limit=args.limit)
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
for row in rows:
|
|
commit = (row.get("origin_commit") or "")[:7]
|
|
_say(
|
|
f"{row['id']} {_status(row['status'], 9)} {row['flow']:<16} "
|
|
f"{row['duration_ms'] / 1000:7.1f}s {commit:<8} {json.dumps(row['params'])}"
|
|
)
|
|
return 0
|
|
|
|
|
|
def _grid(
|
|
definition: dict[str, Any], values: list[str], seed: int | None
|
|
) -> list[dict[str, Any]]:
|
|
"""`--param lr=0.1,0.01 --param epochs=10,50` — every combination of them."""
|
|
types = _input_types(definition)
|
|
names: list[str] = []
|
|
columns: list[list[Any]] = []
|
|
for raw in values:
|
|
name, sep, listed = raw.partition("=")
|
|
name = name.replace("-", "_")
|
|
if not sep or not listed:
|
|
raise SyncError(f"--param takes name=value,value — got '{raw}'")
|
|
if name not in types:
|
|
raise SyncError(
|
|
f"'{name}' is not an input of this flow (it takes "
|
|
f"{', '.join(sorted(types)) or 'none'})"
|
|
)
|
|
names.append(name)
|
|
columns.append([_coerce(item, types[name]) for item in listed.split(",")])
|
|
return [
|
|
{"params": dict(zip(names, combination, strict=True)), "seed": seed}
|
|
for combination in itertools.product(*columns)
|
|
]
|
|
|
|
|
|
def cmd_sweep(args: argparse.Namespace) -> int:
|
|
wait = args.wait or args.local
|
|
try:
|
|
with _client_for(args) as client:
|
|
if not args.no_sync:
|
|
_sync_first(client)
|
|
stored = client.get_flow(args.flow)
|
|
if stored is None:
|
|
return _fail(f"no flow '{args.flow}' on that engine")
|
|
entries = _grid(stored.get("definition") or {}, args.param, args.seed)
|
|
handles = client.sweep(args.flow, entries, no_cache=args.no_cache)
|
|
for handle, entry in zip(handles, entries, strict=True):
|
|
_say(f"{handle.id} queued {json.dumps(entry['params'])}")
|
|
if not wait:
|
|
return 0
|
|
failed = 0
|
|
try:
|
|
for handle in handles:
|
|
handle.wait(timeout=args.timeout)
|
|
_say(
|
|
f"{handle.id} {_status(handle.status)} "
|
|
f"{json.dumps(handle.result)}"
|
|
)
|
|
failed += handle.status != "ok"
|
|
except KeyboardInterrupt:
|
|
for handle in handles:
|
|
if not handle.refresh().done:
|
|
_cancel(client, handle)
|
|
return 130
|
|
return 1 if failed else 0
|
|
except (SyncError, ApiError) as exc:
|
|
return _fail(str(exc))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Wiring
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def add_parsers(subparsers: Any) -> None:
|
|
"""Register the client commands on `fluksio`'s parser."""
|
|
|
|
def with_engine(sub: argparse.ArgumentParser, local: bool = False) -> None:
|
|
sub.add_argument(
|
|
"--url", default="", help="the engine (default: the last login)"
|
|
)
|
|
sub.add_argument("--token", default="", help="override the stored token")
|
|
if local:
|
|
sub.add_argument(
|
|
"--local",
|
|
action="store_true",
|
|
help="boot the engine in this process instead of talking to one",
|
|
)
|
|
|
|
parser = subparsers.add_parser(
|
|
"login", help="store a token for an engine elsewhere"
|
|
)
|
|
parser.add_argument("--url", default="http://localhost:8000")
|
|
parser.add_argument("--email", default="")
|
|
parser.add_argument("--password", default="")
|
|
parser.add_argument(
|
|
"--global",
|
|
dest="shared",
|
|
action="store_true",
|
|
help="store it for the machine rather than this project",
|
|
)
|
|
parser.set_defaults(func=cmd_login)
|
|
|
|
parser = subparsers.add_parser(
|
|
"sync", help="upload the flows declared in your own code"
|
|
)
|
|
parser.add_argument(
|
|
"targets",
|
|
nargs="*",
|
|
help="modules, packages or directories to import (default: .)",
|
|
)
|
|
parser.add_argument(
|
|
"--dry-run", action="store_true", help="print what would be uploaded"
|
|
)
|
|
parser.add_argument(
|
|
"--no-publish", action="store_true", help="leave the changes as a draft"
|
|
)
|
|
parser.add_argument(
|
|
"--force", action="store_true", help="overwrite work done on the canvas"
|
|
)
|
|
with_engine(parser)
|
|
parser.set_defaults(func=cmd_sync)
|
|
|
|
parser = subparsers.add_parser(
|
|
"run", help="sync this directory, then start a run of one of its flows"
|
|
)
|
|
parser.add_argument("flow")
|
|
parser.add_argument(
|
|
"--seed",
|
|
type=int,
|
|
default=None,
|
|
help=(
|
|
"the run's seed: recorded on it, part of what tells two runs of one "
|
|
"configuration apart, and passed to an input named 'seed' when the "
|
|
"flow declares one"
|
|
),
|
|
)
|
|
parser.add_argument("--wait", action="store_true", help="block until it finishes")
|
|
parser.add_argument(
|
|
"--follow",
|
|
action="store_true",
|
|
help="wait, printing the numbers it reports as they arrive",
|
|
)
|
|
parser.add_argument(
|
|
"--timeout",
|
|
type=float,
|
|
default=0.0,
|
|
metavar="SECONDS",
|
|
help=(
|
|
"give up waiting after this long and leave the run going; 0, the "
|
|
"default, waits as long as it takes. Not the node timeout"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--no-sync",
|
|
action="store_true",
|
|
help="run what is already on the engine, without uploading first",
|
|
)
|
|
parser.add_argument(
|
|
"--no-cache",
|
|
action="store_true",
|
|
help="execute every node, even one an earlier run already answered",
|
|
)
|
|
parser.add_argument(
|
|
"--defaults",
|
|
action="store_true",
|
|
help="take every input's declared value instead of asking for it",
|
|
)
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_run)
|
|
|
|
parser = subparsers.add_parser(
|
|
"status", help="how the engine is doing, and what each flow is up to"
|
|
)
|
|
parser.add_argument(
|
|
"--watch",
|
|
action="store_true",
|
|
help="keep it on screen, refreshed until Ctrl-C",
|
|
)
|
|
# `--local` for the same reason `runs` has it: the flows and the history are
|
|
# in this directory, and reading them should not need a server.
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_status)
|
|
|
|
parser = subparsers.add_parser("runs", help="the runs an engine has recorded")
|
|
parser.add_argument("--flow", default="")
|
|
parser.add_argument("--limit", type=int, default=20)
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_runs)
|
|
|
|
parser = subparsers.add_parser(
|
|
"sweep", help="one flow, once per combination of the parameters given"
|
|
)
|
|
parser.add_argument("flow")
|
|
parser.add_argument(
|
|
"--param",
|
|
action="append",
|
|
default=[],
|
|
metavar="NAME=V1,V2",
|
|
help="an input and the values to try; repeat for a grid",
|
|
)
|
|
parser.add_argument(
|
|
"--seed",
|
|
type=int,
|
|
default=None,
|
|
help="the seed every run in the sweep gets; vary it with --param seed=1,2",
|
|
)
|
|
parser.add_argument("--wait", action="store_true", help="block until all finish")
|
|
parser.add_argument(
|
|
"--timeout",
|
|
type=float,
|
|
default=0.0,
|
|
metavar="SECONDS",
|
|
help="give up waiting after this long; 0, the default, waits them out",
|
|
)
|
|
parser.add_argument(
|
|
"--no-sync",
|
|
action="store_true",
|
|
help="run what is already on the engine, without uploading first",
|
|
)
|
|
parser.add_argument(
|
|
"--no-cache",
|
|
action="store_true",
|
|
help="execute every node, even one an earlier run already answered",
|
|
)
|
|
with_engine(parser, local=True)
|
|
parser.set_defaults(func=cmd_sweep)
|