Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Health summary: a node is failing after two failures in a row; one is a warning, a good run ends both

Browse Source 
The summary read degraded for 15 minutes after any single node_error, even
once the node had run well again and including one bad input from a visitor.
The controller now counts a node's failures since it last ran well and notes
when it recovered; both carry across rebuilds and an acknowledgement resets
the count. Two in a row inside the 15 minute window is "failing" (a problem),
one is listed under warnings, and nodes.failing keeps meaning "failing now".
This commit is contained in:
stroblme
2026-10-28 12:42:35 +01:00
parent 2dc20911a3
commit 00842fa175
5 changed files with 161 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/fluksio/api/routes/observability.py
+20 -4
View File
@@ -180,6 +180,14 @@ def _aware(when: datetime) -> datetime:
return when if when.tzinfo else when.replace(tzinfo=UTC)
#: How long after its last failure a node still counts as failing, and as
#: having failed. A node that stopped being called is not kept in the summary
#: forever, and a failure someone read is cleared with `acknowledge_error`.
FAILING_WINDOW_S = 900.0
#: Failures in a row, with no good run between, that make a node "failing".
FAILING_AFTER = 2
@router.get("/summary", response_model=HealthSummary)
async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
"""How the engine is doing right now. Always 200, degraded or not."""
@@ -215,14 +223,17 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
f"{', '.join(sorted(e.id for e in unhealthy))}"
)
# A node that loads and then fails on every call is not in `errored`, and
# until this it read as healthy. ponytail: one failure reads degraded for
# 15 minutes; a counter with decay if that proves noisy.
# would read as healthy. It reads degraded once it has failed twice running
# with no good run between, and only while the last failure is recent: one
# bad input, such as a visitor's, is a warning, and the next good run ends it.
now = time.time()
failing = [
recent = [
e
for e in entries
if e.last_error_ts is not None and now - e.last_error_ts < 900
if e.last_error_ts is not None and now - e.last_error_ts < FAILING_WINDOW_S
]
failing = [e for e in recent if e.failures_in_a_row >= FAILING_AFTER]
failed_once = [e for e in recent if 0 < e.failures_in_a_row < FAILING_AFTER]
if failing:
problems.append(
f"{len(failing)} node(s) failing: "
@@ -256,6 +267,11 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
if degraded
else []
)
if failed_once:
warnings.append(
f"{len(failed_once)} node(s) failed: "
f"{', '.join(sorted(e.id for e in failed_once))}"
)
return HealthSummary(
status="degraded" if problems else "ok",
backend/fluksio/flow/controller.py
+29 -6
View File
@@ -177,12 +177,26 @@ class LoadedNode:
#: so only an acknowledgement clears it, not a good run and not a rebuild.
last_error: str = ""
last_error_ts: float | None = None
#: Failures since it last ran well, so one bad input reads differently from
#: a node that fails every time. The summary goes by this, not by the age
#: of `last_error_ts`.
failures_in_a_row: int = 0
#: When it last ran well after failing: the recovery the counter above
#: was reset by.
last_ok_ts: float | None = None
#: Published, and with no body of its own — so what runs is the new-node
#: template, which returns nothing at all.
missing_source: bool = False
#: The worker this node names although its type never runs on one.
ignored_worker: str = ""
def inherit_failure(self, previous: LoadedNode) -> None:
"""Keep what the node this one replaces last failed with."""
self.last_error = previous.last_error
self.last_error_ts = previous.last_error_ts
self.failures_in_a_row = previous.failures_in_a_row
self.last_ok_ts = previous.last_ok_ts
@dataclass
class RunContext:
@@ -597,13 +611,23 @@ class FlowController:
async with self.events.subscribe() as queue:
while True:
event = await queue.get()
if event.get("type") != "node_error":
kind = event.get("type")
if kind not in ("node_error", "node_executed"):
continue
entry = self.loaded.get(str(event.get("node") or ""))
if entry is None:
continue
ts = float(event.get("ts") or time.time())
if kind == "node_executed":
# Almost every event here, so only a node with a failure
# standing is touched: running well is how it recovers.
if entry.failures_in_a_row:
entry.failures_in_a_row = 0
entry.last_ok_ts = ts
continue
entry.last_error = str(event.get("error") or "")
entry.last_error_ts = float(event.get("ts") or time.time())
entry.last_error_ts = ts
entry.failures_in_a_row += 1
def acknowledge_error(self, node_id: str) -> None:
"""Forget what a node's last failure was. The only thing that clears it.
@@ -616,6 +640,7 @@ class FlowController:
raise KeyError(node_id)
entry.last_error = ""
entry.last_error_ts = None
entry.failures_in_a_row = 0
async def set_enabled(self, flow: str, enabled: bool) -> None:
"""Stop or start one flow, without building anything.
@@ -721,8 +746,7 @@ class FlowController:
for node_id, entry in loaded.items():
previous = self.loaded.get(node_id)
if previous is not None and previous.last_error:
entry.last_error = previous.last_error
entry.last_error_ts = previous.last_error_ts
entry.inherit_failure(previous)
self.loaded = loaded
self.pipeline = Pipeline(
@@ -820,8 +844,7 @@ class FlowController:
for node_id, entry in loaded.items():
previous = self.loaded.get(node_id)
if previous is not None and previous.last_error:
entry.last_error = previous.last_error
entry.last_error_ts = previous.last_error_ts
entry.inherit_failure(previous)
# Rebound rather than mutated: request threads and the failure
# watcher read these without holding anything.