stroblme
00842fa175
The summary read degraded for 15 minutes after any single node_error, even once the node had run well again and including one bad input from a visitor. The controller now counts a node's failures since it last ran well and notes when it recovered; both carry across rebuilds and an acknowledgement resets the count. Two in a row inside the 15 minute window is "failing" (a problem), one is listed under warnings, and nodes.failing keeps meaning "failing now".
59 lines
1.8 KiB
Python
59 lines
1.8 KiB
Python
"""A node's failures in a row, which the health summary reads."""
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
|
|
from fluksio.flow.controller import FlowController, LoadedNode
|
|
from fluksio.flow.events import EventBus
|
|
from fluksio.flow.store import FlowStore
|
|
|
|
NODE = "house.calc"
|
|
|
|
|
|
def _failure(ts: float) -> dict:
|
|
return {"type": "node_error", "node": NODE, "error": "boom", "ts": ts}
|
|
|
|
|
|
def test_failures_count_until_the_node_runs_well(tmp_path: Path) -> None:
|
|
async def scenario() -> LoadedNode:
|
|
events = EventBus()
|
|
events.bind(asyncio.get_running_loop())
|
|
controller = FlowController(FlowStore(tmp_path / "flows"), events=events)
|
|
controller.loaded[NODE] = LoadedNode(id=NODE, flow="house")
|
|
watcher = asyncio.create_task(controller._watch_failures())
|
|
await asyncio.sleep(0)
|
|
|
|
events.publish(_failure(1.0))
|
|
events.publish(_failure(2.0))
|
|
await asyncio.sleep(0.05)
|
|
assert controller.loaded[NODE].failures_in_a_row == 2
|
|
|
|
events.publish({"type": "node_executed", "node": NODE, "ts": 3.0})
|
|
await asyncio.sleep(0.05)
|
|
watcher.cancel()
|
|
return controller.loaded[NODE]
|
|
|
|
entry = asyncio.run(scenario())
|
|
|
|
assert entry.failures_in_a_row == 0
|
|
assert entry.last_ok_ts == 3.0
|
|
# What it failed with stays for the canvas until someone acknowledges it.
|
|
assert entry.last_error == "boom"
|
|
|
|
|
|
def test_a_rebuilt_node_keeps_its_failures() -> None:
|
|
old = LoadedNode(
|
|
id=NODE,
|
|
flow="house",
|
|
last_error="boom",
|
|
last_error_ts=5.0,
|
|
failures_in_a_row=3,
|
|
last_ok_ts=1.0,
|
|
)
|
|
new = LoadedNode(id=NODE, flow="house")
|
|
|
|
new.inherit_failure(old)
|
|
|
|
assert (new.last_error, new.last_error_ts) == ("boom", 5.0)
|
|
assert (new.failures_in_a_row, new.last_ok_ts) == (3, 1.0)
|