Export runs and their curves as tables an analysis reads

`fluksio export metrics` is the long table — a row per run, metric and step —
and `fluksio export runs` the wide one, a row per run with the inputs that
*vary* across the selection as columns beside its final numbers, status,
duration and the commit and digest of the code it ran. Both carry the run id
on every row, which is the join back to the run page and what makes an
exported file auditable. `Client.export_metrics`/`export_runs` answer the same
rows to a notebook.

The engine streams csv or jsonl from two routes declared above `/{run_id}`;
parquet is a client-side conversion behind the new `fluksio[parquet]` extra,
so nobody pays for pyarrow who does not want dtypes kept. The long export
reads each run through `_series`, so a cached node's curve comes with it, and
`--stride` thins each series rather than the concatenation of all of them.

Two things they needed on the way: `GET /runs` takes `?since=` and `?before=`,
so a long history pages by the last row's own timestamp instead of an offset
that shifts under it; and a read that reaches no engine now says so in half a
second rather than seven, because `runs`, `flavors`, `export` and an unwatched
`status` pass `retries=0`. Everything that submits keeps them.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01A9Hdrmf2cwNABCnE5x9UJa

backend/fluksio/sdk/cli.py

@@ -8,6 +8,7 @@ does, imports it inside the branch that asked for it.
 from __future__ import annotations
 
 import argparse
+import csv
 import getpass
 import importlib
 import itertools
@@ -16,7 +17,7 @@ import pkgutil
 import sys
 import time
 from collections.abc import Iterator
-from contextlib import contextmanager
+from contextlib import contextmanager, nullcontext
 from pathlib import Path
 from typing import Any
 
@@ -25,6 +26,7 @@ import httpx
 from fluksio.sdk import FLOWS, Flow, SyncError
 from fluksio.sdk.client import (
     GLOBAL_DATA_DIR,
+    RETRIES,
     WAIT_TOLERANCE,
     ApiError,
     Client,
@@ -216,13 +218,18 @@ def _engine_client() -> Iterator[Client]:
 
 
 @contextmanager
-def _client_for(args: argparse.Namespace) -> Iterator[Client]:
-    """The engine this command talks to: one running somewhere, or this one."""
+def _client_for(args: argparse.Namespace, retries: int = RETRIES) -> Iterator[Client]:
+    """The engine this command talks to: one running somewhere, or this one.
+
+    ``retries=0`` is what a read passes: an engine that is not there otherwise
+    takes the backoff — seconds — to say so, and nothing in a read is worth
+    waiting out a restart for. A command that submits keeps them.
+    """
     if getattr(args, "local", False):
         with _engine_client() as client:
             yield client
     else:
-        yield Client(url=args.url, token=args.token)
+        yield Client(url=args.url, token=args.token, retries=retries)
 
 
 # ---------------------------------------------------------------------------
@@ -708,7 +715,9 @@ def cmd_status(args: argparse.Namespace) -> int:
 
     console = Console()
     try:
-        with _client_for(args) as client:
+        # A watch outlives a blip and should ride one out; a single screen is
+        # a read, and says "not answering" at once.
+        with _client_for(args, retries=RETRIES if args.watch else 0) as client:
             if not args.watch:
                 console.print(_status_screen(client))
                 return 0
@@ -751,7 +760,7 @@ def _stamp(row: dict[str, Any]) -> str:
 
 def cmd_runs(args: argparse.Namespace) -> int:
     try:
-        with _client_for(args) as client:
+        with _client_for(args, retries=0) as client:
             rows = client.runs(flow=args.flow, limit=args.limit)
     except (SyncError, ApiError) as exc:
         return _fail(str(exc))
@@ -771,7 +780,7 @@ def cmd_runs(args: argparse.Namespace) -> int:
 def cmd_flavors(args: argparse.Namespace) -> int:
     """The named sizes a node can ask for."""
     try:
-        with _client_for(args) as client:
+        with _client_for(args, retries=0) as client:
             rows = client.flavors()
     except (SyncError, ApiError) as exc:
         return _fail(str(exc))
@@ -850,6 +859,91 @@ def cmd_sweep(args: argparse.Namespace) -> int:
         return _unreachable(exc, "The runs are still on the engine; `fluksio runs`.")
 
 
+def _selection(args: argparse.Namespace) -> dict[str, Any]:
+    """The filters both exports share, minus the ones left empty."""
+    return {
+        key: value
+        for key, value in (
+            ("status", args.status),
+            ("group", args.group),
+            ("since", args.since),
+            ("until", args.until),
+        )
+        if value
+    }
+
+
+def _write_rows(rows: list[dict[str, Any]], fmt: str, out: str) -> int:
+    """The exported rows, in the format asked for, to a file or to stdout.
+
+    The engine settles the columns over the whole selection before it sends
+    anything, so every row carries the same keys in the same order and the
+    first row's are the header.
+    """
+    if not rows:
+        print("fluksio: nothing matched, so nothing was written", file=sys.stderr)
+        return 0
+    if fmt == "parquet":
+        try:
+            import pyarrow
+            import pyarrow.parquet
+        except ImportError:
+            return _fail(
+                "parquet needs pyarrow — `pip install 'fluksio[parquet]'`, or "
+                "export csv and convert it"
+            )
+        pyarrow.parquet.write_table(pyarrow.Table.from_pylist(rows), out)
+        return 0
+    with open(out, "w", newline="") if out else nullcontext(sys.stdout) as handle:
+        if fmt == "jsonl":
+            handle.writelines(json.dumps(row) + "\n" for row in rows)
+        else:
+            writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
+            writer.writeheader()
+            writer.writerows(rows)
+    return 0
+
+
+def cmd_export_metrics(args: argparse.Namespace) -> int:
+    """Every selected run's numbers as one long table."""
+    if args.format == "parquet" and not args.out:
+        return _fail("--format parquet writes a file; name it with -o FILE")
+    try:
+        with _client_for(args, retries=0) as client:
+            rows = client.export_metrics(
+                flow=args.flow,
+                ids=args.run,
+                name=args.name,
+                stride=args.stride,
+                **_selection(args),
+            )
+    except (SyncError, ApiError) as exc:
+        return _fail(str(exc))
+    except httpx.HTTPError as exc:
+        return _unreachable(exc)
+    return _write_rows(rows, args.format, args.out)
+
+
+def cmd_export_runs(args: argparse.Namespace) -> int:
+    """One row per run: its inputs, its final numbers, what it ran."""
+    if args.format == "parquet" and not args.out:
+        return _fail("--format parquet writes a file; name it with -o FILE")
+    try:
+        with _client_for(args, retries=0) as client:
+            rows = client.export_runs(
+                flow=args.flow,
+                ids=args.run,
+                params=args.params,
+                metrics=args.metrics,
+                **_selection(args),
+            )
+    except (SyncError, ApiError) as exc:
+        return _fail(str(exc))
+    except httpx.HTTPError as exc:
+        return _unreachable(exc)
+    return _write_rows(rows, args.format, args.out)
+
+
 # ---------------------------------------------------------------------------
 # Wiring
 # ---------------------------------------------------------------------------
@@ -1014,3 +1108,59 @@ def add_parsers(subparsers: Any) -> None:
     )
     with_engine(parser, local=True)
     parser.set_defaults(func=cmd_sweep)
+
+    parser = subparsers.add_parser(
+        "export", help="runs and their numbers as a table an analysis reads"
+    )
+    exports = parser.add_subparsers(dest="table", required=True)
+
+    def with_selection(sub: argparse.ArgumentParser) -> None:
+        sub.add_argument("--flow", default="", help="only runs of this flow")
+        sub.add_argument(
+            "--run",
+            action="append",
+            default=[],
+            metavar="ID",
+            help="only this run; repeat for several",
+        )
+        sub.add_argument("--group", default="", help="only the runs of this sweep")
+        sub.add_argument("--status", default="", help="only runs that ended this way")
+        sub.add_argument(
+            "--since", default="", metavar="TS", help="only runs created at or after"
+        )
+        sub.add_argument(
+            "--until", default="", metavar="TS", help="only runs created before"
+        )
+        sub.add_argument("--format", default="csv", choices=("csv", "jsonl", "parquet"))
+        sub.add_argument(
+            "-o", "--out", default="", metavar="FILE", help="write here, not to stdout"
+        )
+        with_engine(sub, local=True)
+
+    sub = exports.add_parser(
+        "metrics", help="one row per run, metric and step — the tidy shape"
+    )
+    with_selection(sub)
+    sub.add_argument("--name", default="", metavar="A,B", help="only these metrics")
+    sub.add_argument(
+        "--stride",
+        type=int,
+        default=1,
+        help="keep every Nth point of each curve",
+    )
+    sub.set_defaults(func=cmd_export_metrics)
+
+    sub = exports.add_parser(
+        "runs", help="one row per run: its inputs, its final numbers, its status"
+    )
+    with_selection(sub)
+    sub.add_argument(
+        "--params",
+        default="",
+        metavar="A,B",
+        help="the inputs to put in columns (default: the ones that vary)",
+    )
+    sub.add_argument(
+        "--metrics", default="", metavar="A,B", help="the final numbers to keep"
+    )
+    sub.set_defaults(func=cmd_export_runs)

backend/fluksio/sdk/client.py

@@ -397,6 +397,75 @@ class Client:
             params={"ids": ",".join(ids), "metric": metric},
         )
 
+    def export_metrics(
+        self,
+        flow: str = "",
+        ids: Iterable[str] = (),
+        name: str = "",
+        stride: int = 1,
+        **filters: Any,
+    ) -> list[dict[str, Any]]:
+        """Runs' series as long rows: run, name, step, ts, value.
+
+        ``pd.DataFrame(client.export_metrics(flow="train"))`` is every loss
+        curve of that flow with the run id on each row. ``name`` keeps the
+        metrics it lists, ``stride`` thins each curve, and the selection is
+        narrowed the way :meth:`runs` is — ``status=``, ``group=``,
+        ``since=``, ``until=``.
+        """
+        query: dict[str, Any] = {"stride": stride}
+        if name:
+            query["name"] = name
+        return self._export("/runs/export/metrics", flow, ids, query, filters)
+
+    def export_runs(
+        self,
+        flow: str = "",
+        ids: Iterable[str] = (),
+        params: str = "",
+        metrics: str = "",
+        **filters: Any,
+    ) -> list[dict[str, Any]]:
+        """One row per run: its inputs as columns, its final numbers, its code.
+
+        The arm-comparison table. The inputs kept are the ones that vary
+        across the selection unless ``params`` names them, which is the axis a
+        sweep is read along.
+        """
+        query: dict[str, Any] = {}
+        if params:
+            query["params"] = params
+        if metrics:
+            query["metrics"] = metrics
+        return self._export("/runs/export/runs", flow, ids, query, filters)
+
+    def _export(
+        self,
+        path: str,
+        flow: str,
+        ids: Iterable[str],
+        query: dict[str, Any],
+        filters: dict[str, Any],
+    ) -> list[dict[str, Any]]:
+        """An export as parsed rows.
+
+        jsonl on the wire rather than csv: it keeps the types the engine has,
+        and it is the format the CLI converts to parquet from. Read whole —
+        the streaming is the engine's side, and a notebook wants a list.
+        """
+        if flow:
+            query["flow"] = flow
+        named = ",".join(ids)
+        if named:
+            query["ids"] = named
+        for key, value in filters.items():
+            query[key] = value.isoformat() if hasattr(value, "isoformat") else value
+        query["format"] = "jsonl"
+        response = self._request("GET", path, idempotent=True, params=query)
+        if response.status_code >= 400:
+            raise ApiError(response.status_code, _detail(response))
+        return [json.loads(line) for line in response.text.splitlines() if line]
+
     def cancel(self, run_id: str) -> Any:
         # Cancelling a cancelled run is cancelled.
         return self._call("POST", f"/runs/{run_id}/cancel", idempotent=True)