Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Health summary: a node is failing after two failures in a row; one is a warning, a good run ends both

Browse Source 
The summary read degraded for 15 minutes after any single node_error, even
once the node had run well again and including one bad input from a visitor.
The controller now counts a node's failures since it last ran well and notes
when it recovered; both carry across rebuilds and an acknowledgement resets
the count. Two in a row inside the 15 minute window is "failing" (a problem),
one is listed under warnings, and nodes.failing keeps meaning "failing now".
This commit is contained in:
stroblme
2026-10-28 12:42:35 +01:00
parent 2dc20911a3
commit 00842fa175
5 changed files with 161 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/fluksio/flow/controller.py
+29 -6
View File
@@ -177,12 +177,26 @@ class LoadedNode:
#: so only an acknowledgement clears it, not a good run and not a rebuild.
last_error: str = ""
last_error_ts: float | None = None
#: Failures since it last ran well, so one bad input reads differently from
#: a node that fails every time. The summary goes by this, not by the age
#: of `last_error_ts`.
failures_in_a_row: int = 0
#: When it last ran well after failing: the recovery the counter above
#: was reset by.
last_ok_ts: float | None = None
#: Published, and with no body of its own — so what runs is the new-node
#: template, which returns nothing at all.
missing_source: bool = False
#: The worker this node names although its type never runs on one.
ignored_worker: str = ""
def inherit_failure(self, previous: LoadedNode) -> None:
"""Keep what the node this one replaces last failed with."""
self.last_error = previous.last_error
self.last_error_ts = previous.last_error_ts
self.failures_in_a_row = previous.failures_in_a_row
self.last_ok_ts = previous.last_ok_ts
@dataclass
class RunContext:
@@ -597,13 +611,23 @@ class FlowController:
async with self.events.subscribe() as queue:
while True:
event = await queue.get()
if event.get("type") != "node_error":
kind = event.get("type")
if kind not in ("node_error", "node_executed"):
continue
entry = self.loaded.get(str(event.get("node") or ""))
if entry is None:
continue
ts = float(event.get("ts") or time.time())
if kind == "node_executed":
# Almost every event here, so only a node with a failure
# standing is touched: running well is how it recovers.
if entry.failures_in_a_row:
entry.failures_in_a_row = 0
entry.last_ok_ts = ts
continue
entry.last_error = str(event.get("error") or "")
entry.last_error_ts = float(event.get("ts") or time.time())
entry.last_error_ts = ts
entry.failures_in_a_row += 1
def acknowledge_error(self, node_id: str) -> None:
"""Forget what a node's last failure was. The only thing that clears it.
@@ -616,6 +640,7 @@ class FlowController:
raise KeyError(node_id)
entry.last_error = ""
entry.last_error_ts = None
entry.failures_in_a_row = 0
async def set_enabled(self, flow: str, enabled: bool) -> None:
"""Stop or start one flow, without building anything.
@@ -721,8 +746,7 @@ class FlowController:
for node_id, entry in loaded.items():
previous = self.loaded.get(node_id)
if previous is not None and previous.last_error:
entry.last_error = previous.last_error
entry.last_error_ts = previous.last_error_ts
entry.inherit_failure(previous)
self.loaded = loaded
self.pipeline = Pipeline(
@@ -820,8 +844,7 @@ class FlowController:
for node_id, entry in loaded.items():
previous = self.loaded.get(node_id)
if previous is not None and previous.last_error:
entry.last_error = previous.last_error
entry.last_error_ts = previous.last_error_ts
entry.inherit_failure(previous)
# Rebound rather than mutated: request threads and the failure
# watcher read these without holding anything.