Keep a failed node's traceback on the run
The worker already sent it and the log panel already got it; the failure outcome kept the one-line error and the node's stdout and dropped the rest, so reading a failure back meant reproducing it under `run --local`. It rides in the node's logs now — no schema change, and the API row, the run detail page and `RunHandle.failures` carry it as they are. `_record_node` keeps the tail of the log cap rather than the head, so a chatty node cannot push the traceback past it, and `fluksio run` prints what each node said when a run does not end ok. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TXQv6KNyyvY7Z1etYTUUAd
This commit is contained in:
@@ -1114,7 +1114,10 @@ class Pipeline:
|
||||
ok=False,
|
||||
duration_ms=round((time.perf_counter() - started) * 1000, 2),
|
||||
error=error,
|
||||
logs=collected.text,
|
||||
# The run record is the only place a failure is read
|
||||
# back from after the fact, so it carries the traceback
|
||||
# the log panel got rather than the one line alone.
|
||||
logs=collected.text + logs.node_traceback(),
|
||||
)
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -1248,7 +1248,9 @@ class RunService:
|
||||
started_at=datetime.now(UTC),
|
||||
duration_ms=outcome.duration_ms,
|
||||
error=outcome.error[:ERROR_CAP],
|
||||
logs=outcome.logs[:LOG_CAP],
|
||||
# The tail, not the head: a failure appends its traceback, and a
|
||||
# chatty node would otherwise push it past the cap.
|
||||
logs=outcome.logs[-LOG_CAP:],
|
||||
cached_from=outcome.cached_from,
|
||||
# Together or not at all: a row carrying a key must be one a
|
||||
# lookup can actually restore from.
|
||||
|
||||
@@ -574,6 +574,20 @@ def _cancel(client: Client, handle: RunHandle) -> int:
|
||||
return 130
|
||||
|
||||
|
||||
def _report_failures(handle: RunHandle) -> None:
|
||||
"""What each failed node said, traceback included.
|
||||
|
||||
The alternative is reproducing the run under ``--local`` to see it, which
|
||||
is the whole reason the traceback is stored.
|
||||
"""
|
||||
for failed in handle.failures:
|
||||
_say(
|
||||
f" {failed.get('node', '')} {_status('error')} {failed.get('error', '')}"
|
||||
)
|
||||
for line in str(failed.get("logs") or "").splitlines():
|
||||
_say(f" {line}")
|
||||
|
||||
|
||||
def _cached_note(client: Client, handle: RunHandle) -> str:
|
||||
"""How much of the run earlier ones had already answered."""
|
||||
try:
|
||||
@@ -629,7 +643,10 @@ def cmd_run(args: argparse.Namespace, rest: list[str]) -> int:
|
||||
f"{handle.id} {_status(handle.status)} "
|
||||
f"{json.dumps(handle.result)}{_cached_note(client, handle)}"
|
||||
)
|
||||
return 0 if handle.status == "ok" else 1
|
||||
if handle.status == "ok":
|
||||
return 0
|
||||
_report_failures(handle)
|
||||
return 1
|
||||
except (SyncError, ApiError) as exc:
|
||||
return _fail(str(exc))
|
||||
except httpx.HTTPError as exc:
|
||||
|
||||
Reference in New Issue
Block a user