stroblme
79c74116d1
The summary read degraded for 15 minutes after any single node_error, even once the node had run well again and including one bad input from a visitor. The controller now counts a node's failures since it last ran well and notes when it recovered; both carry across rebuilds and an acknowledgement resets the count. Two in a row inside the 15 minute window is "failing" (a problem), one is listed under warnings, and nodes.failing keeps meaning "failing now". Claude-Session: https://claude.ai/code/session_01SSi5AvZ7pFmySK8zy3NPMh
59 lines
1.8 KiB
Python
59 lines
1.8 KiB
Python
"""A node's failures in a row, which the health summary reads."""
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
|
|
from fluksio.flow.controller import FlowController, LoadedNode
|
|
from fluksio.flow.events import EventBus
|
|
from fluksio.flow.store import FlowStore
|
|
|
|
NODE = "house.calc"
|
|
|
|
|
|
def _failure(ts: float) -> dict:
|
|
return {"type": "node_error", "node": NODE, "error": "boom", "ts": ts}
|
|
|
|
|
|
def test_failures_count_until_the_node_runs_well(tmp_path: Path) -> None:
|
|
async def scenario() -> LoadedNode:
|
|
events = EventBus()
|
|
events.bind(asyncio.get_running_loop())
|
|
controller = FlowController(FlowStore(tmp_path / "flows"), events=events)
|
|
controller.loaded[NODE] = LoadedNode(id=NODE, flow="house")
|
|
watcher = asyncio.create_task(controller._watch_failures())
|
|
await asyncio.sleep(0)
|
|
|
|
events.publish(_failure(1.0))
|
|
events.publish(_failure(2.0))
|
|
await asyncio.sleep(0.05)
|
|
assert controller.loaded[NODE].failures_in_a_row == 2
|
|
|
|
events.publish({"type": "node_executed", "node": NODE, "ts": 3.0})
|
|
await asyncio.sleep(0.05)
|
|
watcher.cancel()
|
|
return controller.loaded[NODE]
|
|
|
|
entry = asyncio.run(scenario())
|
|
|
|
assert entry.failures_in_a_row == 0
|
|
assert entry.last_ok_ts == 3.0
|
|
# What it failed with stays for the canvas until someone acknowledges it.
|
|
assert entry.last_error == "boom"
|
|
|
|
|
|
def test_a_rebuilt_node_keeps_its_failures() -> None:
|
|
old = LoadedNode(
|
|
id=NODE,
|
|
flow="house",
|
|
last_error="boom",
|
|
last_error_ts=5.0,
|
|
failures_in_a_row=3,
|
|
last_ok_ts=1.0,
|
|
)
|
|
new = LoadedNode(id=NODE, flow="house")
|
|
|
|
new.inherit_failure(old)
|
|
|
|
assert (new.last_error, new.last_error_ts) == ("boom", 5.0)
|
|
assert (new.failures_in_a_row, new.last_ok_ts) == (3, 1.0)
|