Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Survive a busy engine: retry, idempotent submit, resilient waiting

Browse Source 
A driver script died of one slow answer: httpx.ReadTimeout out of
RunHandle.refresh() with a 30 s read timeout and no retry anywhere, which
cost a sweep 78 of its 84 runs.

- Split the timeout (5 s connect, 120 s read): a wrong URL fails at once,
  and a busy engine gets longer than the slowest thing it does on purpose
  (a 60 s compile, a 15 s rebuild wait).
- Retry idempotent calls three times on a transport error or 502/503/504.
  503 is the engine's own "ask again" — it is what RebuildBusy answers.
- Submit carries a key the engine stores with the run, so a retry after a
  timeout returns that run instead of starting a second. A sweep keys every
  entry, so a half-created one recreates only what is missing.
- wait() and --follow tolerate five failed polls in a row; a 404 still stops
  at once, because that is an answer rather than a gap.
- CLI says "engine not answering" and names the run still on the engine,
  instead of printing a traceback.
- runs: clamp the params column to 80 characters; events() takes the
  flow/since/until the endpoint already had; RunHandle.failures answers
  "what killed this run" from the run's own node rows.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
stroblme
2026-08-26 21:19:09 +02:00
co-authored by Claude Opus 5
parent 2cf45e4f39
commit 1f7c6646f1
12 changed files with 503 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/fluksio/sdk/cli.py
+54 -5
View File
@@ -20,9 +20,12 @@ from contextlib import contextmanager
from pathlib import Path
from typing import Any
import httpx
from fluksio.sdk import FLOWS, Flow, SyncError
from fluksio.sdk.client import (
GLOBAL_DATA_DIR,
WAIT_TOLERANCE,
ApiError,
Client,
RunHandle,
@@ -56,6 +59,14 @@ def _fail(message: str) -> int:
return 1
def _unreachable(exc: Exception, note: str = "") -> int:
"""The engine did not answer. Say so as a sentence, not a traceback."""
return _fail(
f"engine not answering ({type(exc).__name__}: {exc})"
+ (f". {note}" if note else "")
)
def _status(text: str, width: int = 0) -> str:
"""A status, coloured when a terminal is reading it.
@@ -267,6 +278,8 @@ def cmd_sync(args: argparse.Namespace) -> int:
)
except (SyncError, ApiError) as exc:
return _fail(str(exc))
except httpx.HTTPError as exc:
return _unreachable(exc, "Nothing was published; sync again when it is back.")
for report in reports:
if report.unchanged:
@@ -402,9 +415,23 @@ def _follow(client: Client, handle: RunHandle, poll: float = 1.0) -> None:
before the numbers, so the last batch is never the one that gets missed.
"""
seen: set[tuple[str, int]] = set()
failures = 0
while True:
done = handle.refresh().done
for point in client.metrics(handle.id):
try:
done = handle.refresh().done
points = client.metrics(handle.id)
except (httpx.HTTPError, ApiError) as exc:
# Following an eight-hour run must not end because one poll of it
# did. The run is still going; only this side lost sight of it.
if isinstance(exc, ApiError) and exc.status < 500:
raise
failures += 1
if failures >= WAIT_TOLERANCE:
raise
time.sleep(poll)
continue
failures = 0
for point in points:
mark = (str(point.get("name", "")), int(point.get("step", -1)))
if mark in seen:
continue
@@ -419,7 +446,7 @@ def _cancel(client: Client, handle: RunHandle) -> int:
"""Ctrl-C means stop the run, not just stop watching it."""
try:
client.cancel(handle.id)
except (SyncError, ApiError) as exc:
except (SyncError, ApiError, httpx.HTTPError) as exc:
return _fail(f"could not cancel {handle.id}: {exc}")
_say(f"{handle.id} {_status('cancelled')}")
return 130
@@ -429,7 +456,7 @@ def _cached_note(client: Client, handle: RunHandle) -> str:
"""How much of the run earlier ones had already answered."""
try:
nodes = client.run(handle.id).get("nodes") or []
except (SyncError, ApiError):
except (SyncError, ApiError, httpx.HTTPError):
return ""
cached = sum(1 for node in nodes if node.get("status") == "cached")
return f" ({cached}/{len(nodes)} {_status('cached')})" if cached else ""
@@ -439,6 +466,7 @@ def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
# An in-process engine lives exactly as long as this command, so a run
# nobody waits for would be thrown away with the queue holding it.
wait = args.wait or args.follow or args.local
handle: RunHandle | None = None
try:
with _client_for(args) as client:
if not args.no_sync:
@@ -482,6 +510,12 @@ def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
return 0 if handle.status == "ok" else 1
except (SyncError, ApiError) as exc:
return _fail(str(exc))
except httpx.HTTPError as exc:
# The run is the engine's, not this command's: it carries on, and its
# id is how to find it again.
return _unreachable(
exc, f"Run {handle.id} is still on the engine." if handle else ""
)
# ---------------------------------------------------------------------------
@@ -651,6 +685,14 @@ def cmd_status(args: argparse.Namespace) -> int:
return 130
except (SyncError, ApiError) as exc:
return _fail(str(exc))
except httpx.HTTPError as exc:
return _unreachable(exc)
#: How much of a run's inputs the list shows. A flow taking a few kB of JSON
#: would otherwise make the table unreadable; `client.runs()` is where the
#: whole value is read.
PARAMS_WIDTH = 80
def cmd_runs(args: argparse.Namespace) -> int:
@@ -659,11 +701,16 @@ def cmd_runs(args: argparse.Namespace) -> int:
rows = client.runs(flow=args.flow, limit=args.limit)
except (SyncError, ApiError) as exc:
return _fail(str(exc))
except httpx.HTTPError as exc:
return _unreachable(exc)
for row in rows:
commit = (row.get("origin_commit") or "")[:7]
params = json.dumps(row["params"])
if len(params) > PARAMS_WIDTH:
params = params[: PARAMS_WIDTH - 3] + "..."
_say(
f"{row['id']} {_status(row['status'], 9)} {row['flow']:<16} "
f"{row['duration_ms'] / 1000:7.1f}s {commit:<8} {json.dumps(row['params'])}"
f"{row['duration_ms'] / 1000:7.1f}s {commit:<8} {params}"
)
return 0
@@ -725,6 +772,8 @@ def cmd_sweep(args: argparse.Namespace) -> int:
return 1 if failed else 0
except (SyncError, ApiError) as exc:
return _fail(str(exc))
except httpx.HTTPError as exc:
return _unreachable(exc, "The runs are still on the engine; `fluksio runs`.")
# ---------------------------------------------------------------------------
backend/fluksio/sdk/client.py
+148 -20
View File
@@ -11,6 +11,7 @@ import json
import os
import subprocess
import time
import uuid
from collections.abc import Iterable
from pathlib import Path
from typing import Any
@@ -34,6 +35,27 @@ API = "/api/v1"
#: A run is over when it reaches one of these.
DONE = frozenset({"ok", "error", "cancelled", "abandoned"})
#: How long to wait for an answer. Split, because the two halves mean
#: different things: an engine that is not there refuses the connection at
#: once, so waiting 30 s for it only ever delays a typo in the URL, while an
#: engine that *is* there can be busy — a rebuild waits 15 s before it answers
#: 503, and compiling a node is given 60 s — so the read has to outlast the
#: slowest thing the engine does on purpose.
CONNECT_TIMEOUT = 5.0
READ_TIMEOUT = 120.0
#: How many times an idempotent call is tried again before it gives up, and
#: what a busy engine answers with. A read timeout used to kill a driver
#: script outright, which is a poor trade for a request that costs nothing to
#: repeat. 503 is the engine's own "ask again": it is what a rebuild answers.
RETRIES = 3
RETRY_STATUS = frozenset({502, 503, 504})
#: How many refreshes in a row may fail before waiting gives up. Each one is
#: already several attempts, so this is minutes of a missing engine, not a
#: blip.
WAIT_TOLERANCE = 5
#: What a project-local installation is called, beside `.venv` and `.git`.
DATA_DIR_NAME = ".fluksio"
@@ -124,13 +146,19 @@ class Client:
"""An authenticated engine, addressed over its HTTP API."""
def __init__(
self, url: str = "", token: str = "", http: Any = None, timeout: float = 30.0
self,
url: str = "",
token: str = "",
http: Any = None,
timeout: Any = None,
retries: int = RETRIES,
) -> None:
stored = _stored()
self.url = url or os.environ.get("FLUKSIO_URL") or stored.get("url") or ""
self.token = (
token or os.environ.get("FLUKSIO_TOKEN") or stored.get("token") or ""
)
self.retries = retries
if http is None:
if not self.url:
raise SyncError(
@@ -139,6 +167,10 @@ class Client:
)
import httpx
if timeout is None:
timeout = httpx.Timeout(
30.0, connect=CONNECT_TIMEOUT, read=READ_TIMEOUT
)
http = httpx.Client(base_url=self.url, timeout=timeout)
self.http = http
if self.token:
@@ -146,8 +178,38 @@ class Client:
# -- plumbing ----------------------------------------------------------
def _call(self, method: str, path: str, **kwargs: Any) -> Any:
response = self.http.request(method, f"{API}{path}", **kwargs)
def _request(
self, method: str, path: str, idempotent: bool = False, **kwargs: Any
) -> Any:
"""One request, tried again while it is safe and worth it.
Only a call that may be repeated without meaning something different
is retried — every GET, and the writes that carry a key or are already
a no-op the second time. The last attempt is made outside the loop, so
whatever it raises is what the caller sees.
"""
import httpx
attempts = self.retries + 1 if idempotent else 1
for attempt in range(attempts - 1):
try:
response = self.http.request(method, f"{API}{path}", **kwargs)
if response.status_code not in RETRY_STATUS:
return response
except httpx.TransportError:
pass
time.sleep(2**attempt)
return self.http.request(method, f"{API}{path}", **kwargs)
def _call(
self, method: str, path: str, idempotent: bool | None = None, **kwargs: Any
) -> Any:
response = self._request(
method,
path,
idempotent=(method == "GET") if idempotent is None else idempotent,
**kwargs,
)
if response.status_code == 409:
raise Conflict(_detail(response))
if response.status_code >= 400:
@@ -192,7 +254,9 @@ class Client:
def refresh_modules(self) -> None:
"""Retire the engine's workers, so the next run imports the code as it is."""
self._call("POST", "/modules/refresh")
# Retiring workers twice is retiring workers, and this is the call most
# likely to meet a rebuild's 503.
self._call("POST", "/modules/refresh", idempotent=True)
def flows(self) -> list[dict[str, Any]]:
"""Every flow with its node and error counts, as the home screen lists."""
@@ -211,11 +275,29 @@ class Client:
result: dict[str, Any] = self._call("GET", "/cloud/status")
return result
def events(self, kind: str = "failure", limit: int = 10) -> list[dict[str, Any]]:
"""What went wrong, or who changed what. Newest first."""
result = self._call(
"GET", "/observability/events", params={"kind": kind, "limit": limit}
)
def events(
self,
kind: str = "failure",
limit: int = 10,
flow: str = "",
since: Any = None,
until: Any = None,
) -> list[dict[str, Any]]:
"""What went wrong, or who changed what. Newest first.
Engine-wide, and narrowed by flow or by time. What *one run* did is a
question about that run: :attr:`RunHandle.failures` answers it from the
run's own node rows, which carry the traceback anyway.
"""
query: dict[str, Any] = {"kind": kind, "limit": limit}
if flow:
query["flow"] = flow
for name, value in (("since", since), ("until", until)):
if value is not None:
query[name] = (
value.isoformat() if hasattr(value, "isoformat") else value
)
result = self._call("GET", "/observability/events", params=query)
return list(result or [])
# -- runs --------------------------------------------------------------
@@ -229,15 +311,22 @@ class Client:
cause: str = "sdk",
) -> RunHandle:
"""Queue a run. ``cause`` is what the history records it as coming
from — a script is the default, `fluksio run` says so itself."""
from — a script is the default, `fluksio run` says so itself.
Safe to retry: the key is minted here, once per call, so an attempt
that timed out on the way back is answered with the run it made rather
than starting a second one.
"""
row = self._call(
"POST",
f"/runs/flows/{flow}",
idempotent=True,
json={
"params": params or {},
"seed": seed,
"no_cache": no_cache,
"cause": cause,
"idempotency_key": uuid.uuid4().hex,
},
)
return RunHandle(self, row["id"], row)
@@ -248,11 +337,17 @@ class Client:
entries: list[dict[str, Any]],
no_cache: bool = False,
) -> list[RunHandle]:
"""Many runs of one flow at once. The caller decides what varies."""
"""Many runs of one flow at once. The caller decides what varies.
A key per entry rather than one for the sweep: a retry then recreates
only the runs whose rows never landed.
"""
keyed = [{**entry, "idempotency_key": uuid.uuid4().hex} for entry in entries]
rows = self._call(
"POST",
f"/runs/flows/{flow}/sweep",
json={"runs": entries, "no_cache": no_cache},
idempotent=True,
json={"runs": keyed, "no_cache": no_cache},
)
return [RunHandle(self, row["id"], row) for row in rows]
@@ -280,10 +375,11 @@ class Client:
)
def cancel(self, run_id: str) -> Any:
return self._call("POST", f"/runs/{run_id}/cancel")
# Cancelling a cancelled run is cancelled.
return self._call("POST", f"/runs/{run_id}/cancel", idempotent=True)
def download(self, digest: str) -> bytes:
response = self.http.request("GET", f"{API}/artifacts/{digest}")
response = self._request("GET", f"/artifacts/{digest}", idempotent=True)
if response.status_code >= 400:
raise ApiError(response.status_code, _detail(response))
return bytes(response.content)
@@ -328,13 +424,45 @@ class RunHandle:
rows = self._row.get("artifacts")
return rows if isinstance(rows, list) else []
def wait(self, timeout: float = 0.0, poll: float = 1.0) -> RunHandle:
"""Block until the run is over, or ``timeout`` seconds have passed."""
deadline = time.monotonic() + timeout if timeout else 0.0
while True:
@property
def failures(self) -> list[dict[str, Any]]:
"""The nodes that failed, each with its error and its logs."""
if "nodes" not in self._row:
self.refresh()
if self.done:
return self
rows = self._row.get("nodes")
rows = rows if isinstance(rows, list) else []
return [row for row in rows if row.get("status") == "error"]
def wait(self, timeout: float = 0.0, poll: float = 1.0) -> RunHandle:
"""Block until the run is over, or ``timeout`` seconds have passed.
A run outlives the engine being briefly unreachable, so waiting for one
does too: a few failed refreshes in a row are a busy engine, not a
finished run. What is not tolerated is the engine answering — a 404
means the run is gone, and repeating the question will not bring it
back.
"""
import httpx
deadline = time.monotonic() + timeout if timeout else 0.0
failures = 0
while True:
try:
self.refresh()
except ApiError as exc:
if exc.status < 500:
raise
failures += 1
if failures >= WAIT_TOLERANCE:
raise
except httpx.HTTPError:
failures += 1
if failures >= WAIT_TOLERANCE:
raise
else:
failures = 0
if self.done:
return self
if deadline and time.monotonic() > deadline:
raise TimeoutError(f"run {self.id} is still {self.status}")
time.sleep(poll)