Keep a failed node's traceback on the run
The worker already sent it and the log panel already got it; the failure outcome kept the one-line error and the node's stdout and dropped the rest, so reading a failure back meant reproducing it under `run --local`. It rides in the node's logs now — no schema change, and the API row, the run detail page and `RunHandle.failures` carry it as they are. `_record_node` keeps the tail of the log cap rather than the head, so a chatty node cannot push the traceback past it, and `fluksio run` prints what each node said when a run does not end ok. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TXQv6KNyyvY7Z1etYTUUAd
This commit is contained in:
@@ -120,6 +120,10 @@ def test_observer_reports_a_failing_node_with_its_error():
|
||||
assert len(seen) == 1
|
||||
assert not seen[0].ok
|
||||
assert "no convergence" in seen[0].error
|
||||
# The traceback rides along in the logs, so a failure can be read back off
|
||||
# the run rather than reproduced under `run --local`.
|
||||
assert "Traceback" in seen[0].logs
|
||||
assert "ValueError: no convergence" in seen[0].logs
|
||||
|
||||
|
||||
def test_a_failing_observer_does_not_take_the_node_down():
|
||||
|
||||
@@ -282,6 +282,53 @@ def test_ctrl_c_while_waiting_cancels_the_run(monkeypatch) -> None:
|
||||
assert cancelled == ["run-1"]
|
||||
|
||||
|
||||
def test_a_failed_run_prints_what_each_node_said(monkeypatch, capsys) -> None:
|
||||
"""Otherwise the traceback is stored and nothing at a terminal shows it."""
|
||||
from contextlib import contextmanager
|
||||
|
||||
from fluksio.cli import _parser
|
||||
from fluksio.sdk import cli
|
||||
|
||||
class FakeHandle:
|
||||
id = "run-1"
|
||||
status = "error"
|
||||
result: dict[str, object] = {}
|
||||
failures = [
|
||||
{
|
||||
"node": "study.train",
|
||||
"error": "ValueError: no convergence",
|
||||
"logs": 'Traceback (most recent call last):\n File "<node>"\n',
|
||||
}
|
||||
]
|
||||
|
||||
def wait(self, timeout: float = 0.0) -> "FakeHandle":
|
||||
return self
|
||||
|
||||
class FakeClient:
|
||||
def get_flow(self, name: str) -> dict[str, object]:
|
||||
return {"definition": {"inputs": []}}
|
||||
|
||||
def submit(self, flow, params, seed=None, no_cache=False, cause="sdk"):
|
||||
return FakeHandle()
|
||||
|
||||
def run(self, run_id: str) -> dict[str, object]:
|
||||
return {"nodes": []}
|
||||
|
||||
@contextmanager
|
||||
def fake_engine():
|
||||
yield FakeClient()
|
||||
|
||||
monkeypatch.setattr(cli, "_engine_client", fake_engine)
|
||||
|
||||
args = _parser().parse_args(["run", "train", "--local", "--no-sync"])
|
||||
assert cli.cmd_run(args, []) == 1
|
||||
|
||||
printed = capsys.readouterr().out
|
||||
assert "study.train" in printed
|
||||
assert "ValueError: no convergence" in printed
|
||||
assert "Traceback (most recent call last):" in printed
|
||||
|
||||
|
||||
def test_an_artifact_input_may_be_named_rather_than_pasted() -> None:
|
||||
"""The engine resolves either spelling; the CLI just stops mangling them."""
|
||||
import json
|
||||
|
||||
Reference in New Issue
Block a user