Stage caching for batch runs, and an engine that lives in the command

A code node in a batch run is now fingerprinted by its source, its raw
settings and the values it reads — an artifact input counting as its digest,
which is what the content addressing was always for. A run that finds the key
restores what the earlier one returned and skips the node, recorded as
`cached`. The run history is the cache: `run_node.outputs` beside the
`cache_key` the schema already had, no second store. On for code nodes, never
for the built-in and connector types that have side effects; off per node with
`@node(cache=False)` and per run with `--no-cache`.

Emissions are not replayed on a hit, so a cached training node returns its
result without redrawing its curve. Recorded in NOTEPAD.md with the two other
deliberate limits.

`fluksio run --local` boots the real app in the command's own process and
drives it through its ASGI interface behind the ordinary client, so a run no
longer needs a `serve` terminal beside it — same data directory, same history,
and the cache carries between the two. It always waits, because the engine it
starts lives exactly as long as the command.

Also: `fluksio sweep --param lr=0.1,0.01` for the product of the lists,
`run --follow` for a run's numbers as they arrive, Ctrl-C cancelling a waited
run rather than abandoning it, coloured statuses on a terminal, and `name`
made optional on the metrics endpoint so a follower can ask for every series.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/fluksio/sdk/cli.py

@@ -1,7 +1,8 @@
-"""The `fluksio sync`, `run`, `runs` and `login` commands.
+"""The `fluksio sync`, `run`, `runs`, `sweep` and `login` commands.
 
 Kept beside the SDK rather than in `fluksio.cli`: these are the client half of
-the tool, and none of them needs the engine to be importable.
+the tool, and none of them needs the engine to be importable — `--local`, which
+does, imports it inside the branch that asked for it.
 """
 
 from __future__ import annotations
@@ -9,9 +10,13 @@ from __future__ import annotations
 import argparse
 import getpass
 import importlib
+import itertools
 import json
 import pkgutil
 import sys
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
 from pathlib import Path
 from typing import Any
 
@@ -20,6 +25,7 @@ from fluksio.sdk.client import (
     GLOBAL_DATA_DIR,
     ApiError,
     Client,
+    RunHandle,
     data_dir,
     login,
     origin_of,
@@ -29,6 +35,17 @@ from fluksio.sdk.client import (
 
 __all__ = ["add_parsers", "discover"]
 
+#: Statuses worth a colour, and the SGR code each gets.
+_COLORS = {
+    "ok": "32",
+    "error": "31",
+    "cached": "36",
+    "cancelled": "33",
+    "abandoned": "31",
+    "running": "36",
+    "queued": "33",
+}
+
 
 def _say(message: str = "") -> None:
     print(message)
@@ -39,6 +56,20 @@ def _fail(message: str) -> int:
     return 1
 
 
+def _status(text: str, width: int = 0) -> str:
+    """A status, coloured when a terminal is reading it.
+
+    Padded before it is coloured: the escape sequences are characters as far
+    as `str.format` is concerned, and a column that lines up in a pipe would
+    not line up on screen.
+    """
+    body = f"{text:<{width}}" if width else text
+    code = _COLORS.get(text)
+    if not code or not sys.stdout.isatty():
+        return body
+    return f"\033[{code}m{body}\033[0m"
+
+
 # ---------------------------------------------------------------------------
 # Discovery
 # ---------------------------------------------------------------------------
@@ -112,6 +143,72 @@ def discover(targets: list[str]) -> list[Flow]:
     return list(FLOWS.values())
 
 
+# ---------------------------------------------------------------------------
+# Which engine
+# ---------------------------------------------------------------------------
+
+
+@contextmanager
+def _engine_client() -> Iterator[Client]:
+    """The engine itself, in this process, behind the ordinary client.
+
+    Everything `fluksio serve` does apart from listening on a socket: the same
+    data directory, the same database, the same admin, the same lifespan. The
+    app is driven through its ASGI interface, so a run costs what it costs on
+    a served engine and lands in the same history — which is what makes the
+    stage cache carry across the two.
+
+    The engine only exists for the length of the command, so nothing here is
+    written back as a login: a stored token belongs to whichever engine
+    `fluksio login` was pointed at.
+    """
+    # Imported here, not at module scope: everything else in this file is the
+    # client half and must keep working with no engine installed.
+    import logging
+
+    from fluksio.cli import _data_dir, _prepare, _print_new_admin
+
+    directory = _data_dir(None)
+    # Settings are read when the app is imported, so this comes first.
+    _prepare(directory)
+    # Nothing here goes over a network, so httpx logging each call as a
+    # request to "testserver" is noise that also happens to be untrue.
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+
+    from datetime import timedelta
+
+    from fastapi.testclient import TestClient
+    from sqlmodel import Session
+
+    from fluksio.core import security
+    from fluksio.core.bootstrap import ensure_superuser
+    from fluksio.core.db import engine
+
+    with Session(engine) as session:
+        admin, generated = ensure_superuser(session)
+        admin_id = admin.id
+        if generated:
+            _print_new_admin(admin.email, generated)
+    token = security.create_access_token(admin_id, expires_delta=timedelta(hours=12))
+
+    from fluksio.main import app
+
+    # Entering the client is what runs the lifespan: the worker pool, the
+    # controller and the run service all start here and stop on the way out.
+    with TestClient(app) as http:
+        yield Client(http=http, token=token)
+
+
+@contextmanager
+def _client_for(args: argparse.Namespace) -> Iterator[Client]:
+    """The engine this command talks to: one running somewhere, or this one."""
+    if getattr(args, "local", False):
+        with _engine_client() as client:
+            yield client
+    else:
+        yield Client(url=args.url, token=args.token)
+
+
 # ---------------------------------------------------------------------------
 # Commands
 # ---------------------------------------------------------------------------
@@ -196,12 +293,17 @@ def _coerce(value: str, dtype: str) -> Any:
     return json.loads(value)
 
 
-def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
-    """Turn `--lr 0.05` into a typed parameter, using the flow's own inputs."""
-    types = {
+def _input_types(definition: dict[str, Any]) -> dict[str, str]:
+    """What each of a flow's inputs is declared to be."""
+    return {
         str(entry["spec"]["name"]): str(entry["spec"].get("dtype", "float"))
         for entry in definition.get("inputs") or []
     }
+
+
+def _params(definition: dict[str, Any], rest: list[str]) -> dict[str, Any]:
+    """Turn `--lr 0.05` into a typed parameter, using the flow's own inputs."""
+    types = _input_types(definition)
     params: dict[str, Any] = {}
     pending: str | None = None
     for token in rest:
@@ -258,42 +360,156 @@ def _sync_first(client: Client) -> None:
         _say(f"synced {', '.join(r.flow for r in changed)}")
 
 
-def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
+def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
+    """Print a run's numbers as they arrive, until it is over.
+
+    Polled rather than pushed: the engine writes a metric down when it is
+    reported, so asking once a second draws the same curve a socket would
+    have, without either side holding a connection open. The status is read
+    before the numbers, so the last batch is never the one that gets missed.
+    """
+    seen: set[tuple[str, int]] = set()
+    while True:
+        done = handle.refresh().done
+        for point in client.metrics(handle.id):
+            mark = (str(point.get("name", "")), int(point.get("step", -1)))
+            if mark in seen:
+                continue
+            seen.add(mark)
+            _say(f"  {mark[0]}[{mark[1]}] = {point['value']:g}")
+        if done:
+            return
+        time.sleep(poll)
+
+
+def _cancel(client: Client, handle: RunHandle) -> int:
+    """Ctrl-C means stop the run, not just stop watching it."""
     try:
-        client = Client(url=args.url, token=args.token)
-        if not args.no_sync:
-            _sync_first(client)
-        stored = client.get_flow(args.flow)
-        if stored is None:
-            return _fail(f"no flow '{args.flow}' on that engine")
-        params = _params(stored.get("definition") or {}, rest)
-        handle = client.submit(args.flow, params, seed=args.seed)
+        client.cancel(handle.id)
+    except (SyncError, ApiError) as exc:
+        return _fail(f"could not cancel {handle.id}: {exc}")
+    _say(f"{handle.id}  {_status('cancelled')}")
+    return 130
+
+
+def _cached_note(client: Client, handle: RunHandle) -> str:
+    """How much of the run earlier ones had already answered."""
+    try:
+        nodes = client.run(handle.id).get("nodes") or []
+    except (SyncError, ApiError):
+        return ""
+    cached = sum(1 for node in nodes if node.get("status") == "cached")
+    return f"  ({cached}/{len(nodes)} {_status('cached')})" if cached else ""
+
+
+def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
+    # An in-process engine lives exactly as long as this command, so a run
+    # nobody waits for would be thrown away with the queue holding it.
+    wait = args.wait or args.follow or args.local
+    try:
+        with _client_for(args) as client:
+            if not args.no_sync:
+                _sync_first(client)
+            stored = client.get_flow(args.flow)
+            if stored is None:
+                return _fail(f"no flow '{args.flow}' on that engine")
+            params = _params(stored.get("definition") or {}, rest)
+            handle = client.submit(
+                args.flow, params, seed=args.seed, no_cache=args.no_cache
+            )
+            _say(f"{handle.id}  queued  {json.dumps(params)}")
+            if not wait:
+                return 0
+            try:
+                if args.follow:
+                    _follow(client, handle)
+                else:
+                    handle.wait(timeout=args.timeout)
+            except KeyboardInterrupt:
+                return _cancel(client, handle)
+            _say(
+                f"{handle.id}  {_status(handle.status)}  "
+                f"{json.dumps(handle.result)}{_cached_note(client, handle)}"
+            )
+            return 0 if handle.status == "ok" else 1
     except (SyncError, ApiError) as exc:
         return _fail(str(exc))
-    _say(f"{handle.id}  queued  {json.dumps(params)}")
-    if not args.wait:
-        return 0
-    handle.wait(timeout=args.timeout)
-    _say(f"{handle.id}  {handle.status}  {json.dumps(handle.result)}")
-    return 0 if handle.status == "ok" else 1
 
 
 def cmd_runs(args: argparse.Namespace) -> int:
     try:
-        rows = Client(url=args.url, token=args.token).runs(
-            flow=args.flow, limit=args.limit
-        )
+        with _client_for(args) as client:
+            rows = client.runs(flow=args.flow, limit=args.limit)
     except (SyncError, ApiError) as exc:
         return _fail(str(exc))
     for row in rows:
         commit = (row.get("origin_commit") or "")[:7]
         _say(
-            f"{row['id']}  {row['status']:<9} {row['flow']:<16} "
+            f"{row['id']}  {_status(row['status'], 9)} {row['flow']:<16} "
             f"{row['duration_ms'] / 1000:7.1f}s  {commit:<8} {json.dumps(row['params'])}"
         )
     return 0
 
 
+def _grid(
+    definition: dict[str, Any], values: list[str], seed: int | None
+) -> list[dict[str, Any]]:
+    """`--param lr=0.1,0.01 --param epochs=10,50` — every combination of them."""
+    types = _input_types(definition)
+    names: list[str] = []
+    columns: list[list[Any]] = []
+    for raw in values:
+        name, sep, listed = raw.partition("=")
+        name = name.replace("-", "_")
+        if not sep or not listed:
+            raise SyncError(f"--param takes name=value,value — got '{raw}'")
+        if name not in types:
+            raise SyncError(
+                f"'{name}' is not an input of this flow (it takes "
+                f"{', '.join(sorted(types)) or 'none'})"
+            )
+        names.append(name)
+        columns.append([_coerce(item, types[name]) for item in listed.split(",")])
+    return [
+        {"params": dict(zip(names, combination, strict=True)), "seed": seed}
+        for combination in itertools.product(*columns)
+    ]
+
+
+def cmd_sweep(args: argparse.Namespace) -> int:
+    wait = args.wait or args.local
+    try:
+        with _client_for(args) as client:
+            if not args.no_sync:
+                _sync_first(client)
+            stored = client.get_flow(args.flow)
+            if stored is None:
+                return _fail(f"no flow '{args.flow}' on that engine")
+            entries = _grid(stored.get("definition") or {}, args.param, args.seed)
+            handles = client.sweep(args.flow, entries, no_cache=args.no_cache)
+            for handle, entry in zip(handles, entries, strict=True):
+                _say(f"{handle.id}  queued  {json.dumps(entry['params'])}")
+            if not wait:
+                return 0
+            failed = 0
+            try:
+                for handle in handles:
+                    handle.wait(timeout=args.timeout)
+                    _say(
+                        f"{handle.id}  {_status(handle.status)}  "
+                        f"{json.dumps(handle.result)}"
+                    )
+                    failed += handle.status != "ok"
+            except KeyboardInterrupt:
+                for handle in handles:
+                    if not handle.refresh().done:
+                        _cancel(client, handle)
+                return 130
+            return 1 if failed else 0
+    except (SyncError, ApiError) as exc:
+        return _fail(str(exc))
+
+
 # ---------------------------------------------------------------------------
 # Wiring
 # ---------------------------------------------------------------------------
@@ -302,11 +518,17 @@ def cmd_runs(args: argparse.Namespace) -> int:
 def add_parsers(subparsers: Any) -> None:
     """Register the client commands on `fluksio`'s parser."""
 
-    def with_engine(sub: argparse.ArgumentParser) -> None:
+    def with_engine(sub: argparse.ArgumentParser, local: bool = False) -> None:
         sub.add_argument(
             "--url", default="", help="the engine (default: the last login)"
         )
         sub.add_argument("--token", default="", help="override the stored token")
+        if local:
+            sub.add_argument(
+                "--local",
+                action="store_true",
+                help="boot the engine in this process instead of talking to one",
+            )
 
     parser = subparsers.add_parser(
         "login", help="store a token for an engine elsewhere"
@@ -348,17 +570,46 @@ def add_parsers(subparsers: Any) -> None:
     parser.add_argument("flow")
     parser.add_argument("--seed", type=int, default=None)
     parser.add_argument("--wait", action="store_true", help="block until it finishes")
+    parser.add_argument(
+        "--follow",
+        action="store_true",
+        help="wait, printing the numbers it reports as they arrive",
+    )
     parser.add_argument("--timeout", type=float, default=0.0)
     parser.add_argument(
         "--no-sync",
         action="store_true",
         help="run what is already on the engine, without uploading first",
     )
-    with_engine(parser)
+    parser.add_argument(
+        "--no-cache",
+        action="store_true",
+        help="execute every node, even one an earlier run already answered",
+    )
+    with_engine(parser, local=True)
     parser.set_defaults(func=cmd_run)
 
     parser = subparsers.add_parser("runs", help="the runs an engine has recorded")
     parser.add_argument("--flow", default="")
     parser.add_argument("--limit", type=int, default=20)
-    with_engine(parser)
+    with_engine(parser, local=True)
     parser.set_defaults(func=cmd_runs)
+
+    parser = subparsers.add_parser(
+        "sweep", help="one flow, once per combination of the parameters given"
+    )
+    parser.add_argument("flow")
+    parser.add_argument(
+        "--param",
+        action="append",
+        default=[],
+        metavar="NAME=V1,V2",
+        help="an input and the values to try; repeat for a grid",
+    )
+    parser.add_argument("--seed", type=int, default=None)
+    parser.add_argument("--wait", action="store_true", help="block until all finish")
+    parser.add_argument("--timeout", type=float, default=0.0)
+    parser.add_argument("--no-sync", action="store_true")
+    parser.add_argument("--no-cache", action="store_true")
+    with_engine(parser, local=True)
+    parser.set_defaults(func=cmd_sweep)