Survive a busy engine: retry, idempotent submit, resilient waiting

A driver script died of one slow answer: httpx.ReadTimeout out of
RunHandle.refresh() with a 30 s read timeout and no retry anywhere, which
cost a sweep 78 of its 84 runs.

- Split the timeout (5 s connect, 120 s read): a wrong URL fails at once,
  and a busy engine gets longer than the slowest thing it does on purpose
  (a 60 s compile, a 15 s rebuild wait).
- Retry idempotent calls three times on a transport error or 502/503/504.
  503 is the engine's own "ask again" — it is what RebuildBusy answers.
- Submit carries a key the engine stores with the run, so a retry after a
  timeout returns that run instead of starting a second. A sweep keys every
  entry, so a half-created one recreates only what is missing.
- wait() and --follow tolerate five failed polls in a row; a 404 still stops
  at once, because that is an answer rather than a gap.
- CLI says "engine not answering" and names the run still on the engine,
  instead of printing a traceback.
- runs: clamp the params column to 80 characters; events() takes the
  flow/since/until the endpoint already had; RunHandle.failures answers
  "what killed this run" from the run's own node rows.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/tests/api/routes/test_runs.py

@@ -19,6 +19,7 @@ from fluksio.flow.runs import (
     OUTPUT_CAP,
     RunCache,
     RunRejected,
+    RunService,
     _cacheable,
     new_run_id,
     resolve_references,
@@ -349,6 +350,59 @@ def test_a_run_records_which_caller_asked_for_it(
     assert refused.status_code == 422
 
 
+class _Unusable:
+    """Anything reaching this is something a deduplicated submit should not do."""
+
+    def __getattr__(self, name):
+        raise AssertionError(f"a repeated submit must not reach {name}")
+
+
+def test_a_repeated_submit_returns_the_run_it_already_made():
+    """The key is the answer to "did my first attempt land?".
+
+    Answered before the flow is even read: a caller retrying a submit it never
+    got a reply to is owed that run, whatever has been published since.
+    """
+    service = RunService(controller=_Unusable(), queue=_Unusable())
+    with Session(db_engine) as session:
+        session.add(
+            Run(
+                id="dedup-1",
+                flow="study",
+                status="running",
+                idempotency_key="key-abc",
+                created_at=datetime.now(UTC),
+            )
+        )
+        session.commit()
+
+    again = service.submit("study", {"lr": 0.1}, idempotency_key="key-abc")
+
+    assert again.id == "dedup-1"
+
+
+def test_a_key_nobody_used_submits_normally(
+    client, superuser_token_headers, monkeypatch
+):
+    """The route carries the key through; without one nothing changes."""
+    seen: dict[str, object] = {}
+
+    class Recorder:
+        def submit(self, name, **kwargs):
+            seen.update(kwargs)
+            return Run(id="keyed-1", flow=name, created_at=datetime.now(UTC))
+
+    monkeypatch.setattr(client.app.state, "run_service", Recorder())
+    answer = client.post(
+        f"{settings.API_V1_STR}/runs/flows/demo",
+        headers=superuser_token_headers,
+        json={"idempotency_key": "key-xyz"},
+    )
+
+    assert answer.status_code == 202
+    assert seen["idempotency_key"] == "key-xyz"
+
+
 def test_overview_is_not_read_as_a_run_id(client, superuser_token_headers):
     """`/overview` is declared before `/{run_id}`, which would swallow it."""
     answer = client.get(

backend/tests/sdk/test_client.py

@@ -0,0 +1,132 @@
+"""What the client does when the engine is slow, busy, or briefly gone."""
+
+import httpx
+import pytest
+
+from fluksio.sdk.client import WAIT_TOLERANCE, ApiError, Client, RunHandle
+
+
+def a_client(handler, monkeypatch, **kwargs):
+    """A client whose transport is a function, and whose backoff costs nothing."""
+    monkeypatch.setattr("fluksio.sdk.client.time.sleep", lambda _seconds: None)
+    transport = httpx.MockTransport(handler)
+    http = httpx.Client(transport=transport, base_url="http://engine")
+    return Client(url="http://engine", token="t", http=http, **kwargs)
+
+
+def test_idempotent_get_is_tried_again(monkeypatch):
+    calls = []
+
+    def handler(request):
+        calls.append(request)
+        if len(calls) < 3:
+            raise httpx.ReadTimeout("too slow", request=request)
+        return httpx.Response(200, json={"id": "r1", "status": "ok"})
+
+    client = a_client(handler, monkeypatch)
+    assert client.run("r1")["status"] == "ok"
+    assert len(calls) == 3
+
+
+def test_a_busy_engine_is_asked_again(monkeypatch):
+    codes = iter([503, 503, 200])
+
+    def handler(request):
+        code = next(codes)
+        return httpx.Response(code, json={"id": "r1"} if code == 200 else {})
+
+    client = a_client(handler, monkeypatch)
+    assert client.run("r1")["id"] == "r1"
+
+
+def test_giving_up_raises_what_it_last_saw(monkeypatch):
+    def handler(request):
+        raise httpx.ReadTimeout("too slow", request=request)
+
+    client = a_client(handler, monkeypatch, retries=2)
+    with pytest.raises(httpx.ReadTimeout):
+        client.run("r1")
+
+
+def test_a_write_is_not_repeated(monkeypatch):
+    calls = []
+
+    def handler(request):
+        calls.append(request)
+        raise httpx.ReadTimeout("too slow", request=request)
+
+    client = a_client(handler, monkeypatch)
+    with pytest.raises(httpx.ReadTimeout):
+        client.put_source("train", "fit", "code")
+    assert len(calls) == 1, "a source write means something different twice"
+
+
+def test_submit_carries_one_key_across_its_retries(monkeypatch):
+    bodies = []
+
+    def handler(request):
+        bodies.append(httpx.Response(200, content=request.content).json())
+        if len(bodies) < 3:
+            raise httpx.ConnectError("no route", request=request)
+        return httpx.Response(202, json={"id": "r1", "status": "queued"})
+
+    client = a_client(handler, monkeypatch)
+    assert client.submit("train", {"lr": 0.1}).id == "r1"
+    keys = {body["idempotency_key"] for body in bodies}
+    assert len(keys) == 1, "a retry must not read as a second run"
+    assert len(next(iter(keys))) == 32
+
+
+def test_a_sweep_keys_every_entry(monkeypatch):
+    seen = {}
+
+    def handler(request):
+        body = httpx.Response(200, content=request.content).json()
+        seen["runs"] = body["runs"]
+        return httpx.Response(202, json=[{"id": "r1"}, {"id": "r2"}])
+
+    client = a_client(handler, monkeypatch)
+    client.sweep("train", [{"params": {"lr": 0.1}}, {"params": {"lr": 0.2}}])
+    keys = [entry["idempotency_key"] for entry in seen["runs"]]
+    assert len(set(keys)) == 2
+
+
+def test_waiting_survives_a_few_bad_answers(monkeypatch):
+    answers = iter(
+        [503] * (WAIT_TOLERANCE - 1) + [200]  # then the run is finished
+    )
+
+    def handler(request):
+        code = next(answers)
+        if code != 200:
+            return httpx.Response(code, json={})
+        return httpx.Response(200, json={"id": "r1", "status": "ok"})
+
+    client = a_client(handler, monkeypatch, retries=0)
+    handle = RunHandle(client, "r1", {"status": "running"})
+    assert handle.wait(poll=0).status == "ok"
+
+
+def test_waiting_gives_up_eventually(monkeypatch):
+    def handler(request):
+        return httpx.Response(503, json={})
+
+    client = a_client(handler, monkeypatch, retries=0)
+    handle = RunHandle(client, "r1", {"status": "running"})
+    with pytest.raises(ApiError):
+        handle.wait(poll=0)
+
+
+def test_a_run_that_is_gone_stops_the_wait_at_once(monkeypatch):
+    calls = []
+
+    def handler(request):
+        calls.append(request)
+        return httpx.Response(404, json={"detail": "no such run"})
+
+    client = a_client(handler, monkeypatch, retries=0)
+    handle = RunHandle(client, "r1", {"status": "running"})
+    with pytest.raises(ApiError) as caught:
+        handle.wait(poll=0)
+    assert caught.value.status == 404
+    assert len(calls) == 1, "a 404 is an answer, not a blip"