Survive a busy engine: retry, idempotent submit, resilient waiting

A driver script died of one slow answer: httpx.ReadTimeout out of
RunHandle.refresh() with a 30 s read timeout and no retry anywhere, which
cost a sweep 78 of its 84 runs.

- Split the timeout (5 s connect, 120 s read): a wrong URL fails at once,
  and a busy engine gets longer than the slowest thing it does on purpose
  (a 60 s compile, a 15 s rebuild wait).
- Retry idempotent calls three times on a transport error or 502/503/504.
  503 is the engine's own "ask again" — it is what RebuildBusy answers.
- Submit carries a key the engine stores with the run, so a retry after a
  timeout returns that run instead of starting a second. A sweep keys every
  entry, so a half-created one recreates only what is missing.
- wait() and --follow tolerate five failed polls in a row; a 404 still stops
  at once, because that is an answer rather than a gap.
- CLI says "engine not answering" and names the run still on the engine,
  instead of printing a traceback.
- runs: clamp the params column to 80 characters; events() takes the
  flow/since/until the endpoint already had; RunHandle.failures answers
  "what killed this run" from the run's own node rows.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/fluksio/sdk/cli.py

@@ -20,9 +20,12 @@ from contextlib import contextmanager
 from pathlib import Path
 from typing import Any
 
+import httpx
+
 from fluksio.sdk import FLOWS, Flow, SyncError
 from fluksio.sdk.client import (
     GLOBAL_DATA_DIR,
+    WAIT_TOLERANCE,
     ApiError,
     Client,
     RunHandle,
@@ -56,6 +59,14 @@ def _fail(message: str) -> int:
     return 1
 
 
+def _unreachable(exc: Exception, note: str = "") -> int:
+    """The engine did not answer. Say so as a sentence, not a traceback."""
+    return _fail(
+        f"engine not answering ({type(exc).__name__}: {exc})"
+        + (f". {note}" if note else "")
+    )
+
+
 def _status(text: str, width: int = 0) -> str:
     """A status, coloured when a terminal is reading it.
 
@@ -267,6 +278,8 @@ def cmd_sync(args: argparse.Namespace) -> int:
         )
     except (SyncError, ApiError) as exc:
         return _fail(str(exc))
+    except httpx.HTTPError as exc:
+        return _unreachable(exc, "Nothing was published; sync again when it is back.")
 
     for report in reports:
         if report.unchanged:
@@ -402,9 +415,23 @@ def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
     before the numbers, so the last batch is never the one that gets missed.
     """
     seen: set[tuple[str, int]] = set()
+    failures = 0
     while True:
-        done = handle.refresh().done
-        for point in client.metrics(handle.id):
+        try:
+            done = handle.refresh().done
+            points = client.metrics(handle.id)
+        except (httpx.HTTPError, ApiError) as exc:
+            # Following an eight-hour run must not end because one poll of it
+            # did. The run is still going; only this side lost sight of it.
+            if isinstance(exc, ApiError) and exc.status < 500:
+                raise
+            failures += 1
+            if failures >= WAIT_TOLERANCE:
+                raise
+            time.sleep(poll)
+            continue
+        failures = 0
+        for point in points:
             mark = (str(point.get("name", "")), int(point.get("step", -1)))
             if mark in seen:
                 continue
@@ -419,7 +446,7 @@ def _cancel(client: Client, handle: RunHandle) -> int:
     """Ctrl-C means stop the run, not just stop watching it."""
     try:
         client.cancel(handle.id)
-    except (SyncError, ApiError) as exc:
+    except (SyncError, ApiError, httpx.HTTPError) as exc:
         return _fail(f"could not cancel {handle.id}: {exc}")
     _say(f"{handle.id}  {_status('cancelled')}")
     return 130
@@ -429,7 +456,7 @@ def _cached_note(client: Client, handle: RunHandle) -> str:
     """How much of the run earlier ones had already answered."""
     try:
         nodes = client.run(handle.id).get("nodes") or []
-    except (SyncError, ApiError):
+    except (SyncError, ApiError, httpx.HTTPError):
         return ""
     cached = sum(1 for node in nodes if node.get("status") == "cached")
     return f"  ({cached}/{len(nodes)} {_status('cached')})" if cached else ""
@@ -439,6 +466,7 @@ def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
     # An in-process engine lives exactly as long as this command, so a run
     # nobody waits for would be thrown away with the queue holding it.
     wait = args.wait or args.follow or args.local
+    handle: RunHandle | None = None
     try:
         with _client_for(args) as client:
             if not args.no_sync:
@@ -482,6 +510,12 @@ def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
             return 0 if handle.status == "ok" else 1
     except (SyncError, ApiError) as exc:
         return _fail(str(exc))
+    except httpx.HTTPError as exc:
+        # The run is the engine's, not this command's: it carries on, and its
+        # id is how to find it again.
+        return _unreachable(
+            exc, f"Run {handle.id} is still on the engine." if handle else ""
+        )
 
 
 # ---------------------------------------------------------------------------
@@ -651,6 +685,14 @@ def cmd_status(args: argparse.Namespace) -> int:
         return 130
     except (SyncError, ApiError) as exc:
         return _fail(str(exc))
+    except httpx.HTTPError as exc:
+        return _unreachable(exc)
+
+
+#: How much of a run's inputs the list shows. A flow taking a few kB of JSON
+#: would otherwise make the table unreadable; `client.runs()` is where the
+#: whole value is read.
+PARAMS_WIDTH = 80
 
 
 def cmd_runs(args: argparse.Namespace) -> int:
@@ -659,11 +701,16 @@ def cmd_runs(args: argparse.Namespace) -> int:
             rows = client.runs(flow=args.flow, limit=args.limit)
     except (SyncError, ApiError) as exc:
         return _fail(str(exc))
+    except httpx.HTTPError as exc:
+        return _unreachable(exc)
     for row in rows:
         commit = (row.get("origin_commit") or "")[:7]
+        params = json.dumps(row["params"])
+        if len(params) > PARAMS_WIDTH:
+            params = params[: PARAMS_WIDTH - 3] + "..."
         _say(
             f"{row['id']}  {_status(row['status'], 9)} {row['flow']:<16} "
-            f"{row['duration_ms'] / 1000:7.1f}s  {commit:<8} {json.dumps(row['params'])}"
+            f"{row['duration_ms'] / 1000:7.1f}s  {commit:<8} {params}"
         )
     return 0
 
@@ -725,6 +772,8 @@ def cmd_sweep(args: argparse.Namespace) -> int:
             return 1 if failed else 0
     except (SyncError, ApiError) as exc:
         return _fail(str(exc))
+    except httpx.HTTPError as exc:
+        return _unreachable(exc, "The runs are still on the engine; `fluksio runs`.")
 
 
 # ---------------------------------------------------------------------------