Follow a record into its fields, name the metrics, name the version
Docs / docs (push) Successful in 38s
Playwright Tests / test-playwright (1, 2) (push) Successful in 2m56s
Playwright Tests / test-playwright (2, 2) (push) Successful in 2m7s
pre-commit / pre-commit (push) Failing after 2m17s
Test Backend / test-backend (push) Successful in 2m54s
Compose Smoke Test / test-compose (push) Successful in 44s
Playwright Tests / merge-reports (push) Successful in 1m17s
Docs / docs (push) Successful in 38s
Playwright Tests / test-playwright (1, 2) (push) Successful in 2m56s
Playwright Tests / test-playwright (2, 2) (push) Successful in 2m7s
pre-commit / pre-commit (push) Failing after 2m17s
Test Backend / test-backend (push) Successful in 2m54s
Compose Smoke Test / test-compose (push) Successful in 44s
Playwright Tests / merge-reports (push) Successful in 1m17s
Three things the first export pass got wrong for a real study. **Dotted paths.** A node returns a record, not a scalar — the numbers arrive inside `final_metrics` — so `--metrics final_metrics.train_loss` yielded an empty column and `--metrics final_metrics` yielded the whole record in one cell. Both sides of the wide table now take dotted paths, and the defaults reach the same depth: every number a result carries is a column named by its path, and inputs are compared leaf by leaf, so two configurations differing in one field give that field as the axis rather than two blobs that are merely not equal. Lists stay whole — a curve belongs in the long table. **`--list`.** Metric names are flow-qualified, so `--name train_loss` matched nothing and said only that. `fluksio export metrics --list` prints the names the selection carries, and an empty export made with `--name` points at it. **A version to compare.** The CLI ships ahead of the engine and a stale one answered a flat 404 with nothing anywhere in the API to tell how old it was. The engine reports `version` on `/observability/summary`, `fluksio status` prints it, and a 404 from export now names both versions — or says "older" when the field itself predates the engine. Bumped to 0.1.5, which is what makes the number worth reading. Also formats `flow/metrics.py`, which had been committed unformatted and was the last `ruff format --check` failure. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01A9Hdrmf2cwNABCnE5x9UJa
This commit is contained in:
+111
-31
@@ -16,13 +16,14 @@ import json
|
||||
import pkgutil
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from fluksio import __version__
|
||||
from fluksio.sdk import FLOWS, Flow, SyncError
|
||||
from fluksio.sdk.client import (
|
||||
GLOBAL_DATA_DIR,
|
||||
@@ -634,6 +635,10 @@ def _status_screen(client: Client) -> Any:
|
||||
head.append(" " + " · ".join(str(p) for p in problems), style="yellow")
|
||||
phrase, style = _portal_phrase(portal)
|
||||
head.append(" " + phrase, style=style)
|
||||
# What it is running, since a client is upgraded on its own and a route
|
||||
# this one knows may not be there. Absent from an engine older than the
|
||||
# field itself, which is the answer in its own way.
|
||||
head.append(f" v{summary.get('version') or '?'}", style="dim")
|
||||
|
||||
counts = summary.get("flows") or {}
|
||||
nodes = summary.get("nodes") or {}
|
||||
@@ -873,7 +878,7 @@ def _selection(args: argparse.Namespace) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _write_rows(rows: list[dict[str, Any]], fmt: str, out: str) -> int:
|
||||
def _write_rows(rows: list[dict[str, Any]], fmt: str, out: str, hint: str = "") -> int:
|
||||
"""The exported rows, in the format asked for, to a file or to stdout.
|
||||
|
||||
The engine settles the columns over the whole selection before it sends
|
||||
@@ -881,7 +886,8 @@ def _write_rows(rows: list[dict[str, Any]], fmt: str, out: str) -> int:
|
||||
first row's are the header.
|
||||
"""
|
||||
if not rows:
|
||||
print("fluksio: nothing matched, so nothing was written", file=sys.stderr)
|
||||
empty = "fluksio: nothing matched, so nothing was written"
|
||||
print(empty + (f". {hint}" if hint else ""), file=sys.stderr)
|
||||
return 0
|
||||
if fmt == "parquet":
|
||||
try:
|
||||
@@ -904,44 +910,106 @@ def _write_rows(rows: list[dict[str, Any]], fmt: str, out: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_export_metrics(args: argparse.Namespace) -> int:
|
||||
"""Every selected run's numbers as one long table."""
|
||||
#: How many runs `--list` reads before giving up on finding a metric name.
|
||||
#: The names belong to the flow's nodes rather than to a run, so the newest
|
||||
#: one that recorded any is the whole vocabulary — the rest are for a
|
||||
#: selection whose newest runs failed before they measured anything.
|
||||
# ponytail: the first run with names wins; a name only an older run recorded
|
||||
# is not listed. A `distinct` over the selection would be exact and is a route
|
||||
# of its own.
|
||||
LIST_SCAN = 10
|
||||
|
||||
|
||||
def _list_names(client: Client, args: argparse.Namespace) -> int:
|
||||
"""The metric names these runs carry, since a name is flow-qualified.
|
||||
|
||||
`train_loss` is recorded as `train.train_loss`, and asking for the bare
|
||||
one matches nothing — so this is the answer to "what would match".
|
||||
"""
|
||||
filters = _selection(args)
|
||||
if "until" in filters:
|
||||
# The history spells the same bound `before`, where it is also the
|
||||
# cursor a page is taken from.
|
||||
filters["before"] = filters.pop("until")
|
||||
ids = args.run or [
|
||||
row["id"] for row in client.runs(flow=args.flow, limit=LIST_SCAN, **filters)
|
||||
]
|
||||
for run_id in ids[:LIST_SCAN]:
|
||||
names = sorted({point["name"] for point in client.metrics(run_id)})
|
||||
if names:
|
||||
_say("\n".join(names))
|
||||
return 0
|
||||
_say("No metrics recorded by these runs.")
|
||||
return 0
|
||||
|
||||
|
||||
def _too_old(client: Client) -> str:
|
||||
"""A route this client knows and the engine does not."""
|
||||
try:
|
||||
version = (client.summary() or {}).get("version") or ""
|
||||
except (ApiError, httpx.HTTPError):
|
||||
version = ""
|
||||
engine = f"the engine is {version}" if version else "the engine is older"
|
||||
return (
|
||||
f"this engine has no export endpoints — {engine} and this client is "
|
||||
f"{__version__}. Upgrade it: `pip install -U fluksio`"
|
||||
)
|
||||
|
||||
|
||||
def _export(
|
||||
args: argparse.Namespace, fetch: Callable[[Client], list[dict[str, Any]]]
|
||||
) -> int:
|
||||
"""Both exports: fetch what was asked for, then write it."""
|
||||
if args.format == "parquet" and not args.out:
|
||||
return _fail("--format parquet writes a file; name it with -o FILE")
|
||||
hint = (
|
||||
"`--list` names the metrics these runs carry"
|
||||
if getattr(args, "name", "")
|
||||
else ""
|
||||
)
|
||||
try:
|
||||
with _client_for(args, retries=0) as client:
|
||||
rows = client.export_metrics(
|
||||
flow=args.flow,
|
||||
ids=args.run,
|
||||
name=args.name,
|
||||
stride=args.stride,
|
||||
**_selection(args),
|
||||
)
|
||||
if getattr(args, "list_names", False):
|
||||
return _list_names(client, args)
|
||||
try:
|
||||
rows = fetch(client)
|
||||
except ApiError as exc:
|
||||
if exc.status != 404:
|
||||
raise
|
||||
return _fail(_too_old(client))
|
||||
except (SyncError, ApiError) as exc:
|
||||
return _fail(str(exc))
|
||||
except httpx.HTTPError as exc:
|
||||
return _unreachable(exc)
|
||||
return _write_rows(rows, args.format, args.out)
|
||||
return _write_rows(rows, args.format, args.out, hint)
|
||||
|
||||
|
||||
def cmd_export_metrics(args: argparse.Namespace) -> int:
|
||||
"""Every selected run's numbers as one long table."""
|
||||
return _export(
|
||||
args,
|
||||
lambda client: client.export_metrics(
|
||||
flow=args.flow,
|
||||
ids=args.run,
|
||||
name=args.name,
|
||||
stride=args.stride,
|
||||
**_selection(args),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def cmd_export_runs(args: argparse.Namespace) -> int:
|
||||
"""One row per run: its inputs, its final numbers, what it ran."""
|
||||
if args.format == "parquet" and not args.out:
|
||||
return _fail("--format parquet writes a file; name it with -o FILE")
|
||||
try:
|
||||
with _client_for(args, retries=0) as client:
|
||||
rows = client.export_runs(
|
||||
flow=args.flow,
|
||||
ids=args.run,
|
||||
params=args.params,
|
||||
metrics=args.metrics,
|
||||
**_selection(args),
|
||||
)
|
||||
except (SyncError, ApiError) as exc:
|
||||
return _fail(str(exc))
|
||||
except httpx.HTTPError as exc:
|
||||
return _unreachable(exc)
|
||||
return _write_rows(rows, args.format, args.out)
|
||||
return _export(
|
||||
args,
|
||||
lambda client: client.export_runs(
|
||||
flow=args.flow,
|
||||
ids=args.run,
|
||||
params=args.params,
|
||||
metrics=args.metrics,
|
||||
**_selection(args),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1148,6 +1216,12 @@ def add_parsers(subparsers: Any) -> None:
|
||||
default=1,
|
||||
help="keep every Nth point of each curve",
|
||||
)
|
||||
sub.add_argument(
|
||||
"--list",
|
||||
dest="list_names",
|
||||
action="store_true",
|
||||
help="print the metric names these runs carry, and stop",
|
||||
)
|
||||
sub.set_defaults(func=cmd_export_metrics)
|
||||
|
||||
sub = exports.add_parser(
|
||||
@@ -1158,9 +1232,15 @@ def add_parsers(subparsers: Any) -> None:
|
||||
"--params",
|
||||
default="",
|
||||
metavar="A,B",
|
||||
help="the inputs to put in columns (default: the ones that vary)",
|
||||
help=(
|
||||
"the inputs to put in columns, dotted into a record "
|
||||
"(default: the ones that vary)"
|
||||
),
|
||||
)
|
||||
sub.add_argument(
|
||||
"--metrics", default="", metavar="A,B", help="the final numbers to keep"
|
||||
"--metrics",
|
||||
default="",
|
||||
metavar="A,B",
|
||||
help="the final numbers to keep, e.g. final_metrics.train_loss",
|
||||
)
|
||||
sub.set_defaults(func=cmd_export_runs)
|
||||
|
||||
Reference in New Issue
Block a user