Health summary: a node is failing after two failures in a row; one is a warning, a good run ends both

The summary read degraded for 15 minutes after any single node_error, even
once the node had run well again and including one bad input from a visitor.
The controller now counts a node's failures since it last ran well and notes
when it recovered; both carry across rebuilds and an acknowledgement resets
the count. Two in a row inside the 15 minute window is "failing" (a problem),
one is listed under warnings, and nodes.failing keeps meaning "failing now".

backend/fluksio/api/routes/observability.py

@@ -180,6 +180,14 @@ def _aware(when: datetime) -> datetime:
     return when if when.tzinfo else when.replace(tzinfo=UTC)
 
 
+#: How long after its last failure a node still counts as failing, and as
+#: having failed. A node that stopped being called is not kept in the summary
+#: forever, and a failure someone read is cleared with `acknowledge_error`.
+FAILING_WINDOW_S = 900.0
+#: Failures in a row, with no good run between, that make a node "failing".
+FAILING_AFTER = 2
+
+
 @router.get("/summary", response_model=HealthSummary)
 async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
     """How the engine is doing right now. Always 200, degraded or not."""
@@ -215,14 +223,17 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
             f"{', '.join(sorted(e.id for e in unhealthy))}"
         )
     # A node that loads and then fails on every call is not in `errored`, and
-    # until this it read as healthy. ponytail: one failure reads degraded for
-    # 15 minutes; a counter with decay if that proves noisy.
+    # would read as healthy. It reads degraded once it has failed twice running
+    # with no good run between, and only while the last failure is recent: one
+    # bad input, such as a visitor's, is a warning, and the next good run ends it.
     now = time.time()
-    failing = [
+    recent = [
         e
         for e in entries
-        if e.last_error_ts is not None and now - e.last_error_ts < 900
+        if e.last_error_ts is not None and now - e.last_error_ts < FAILING_WINDOW_S
     ]
+    failing = [e for e in recent if e.failures_in_a_row >= FAILING_AFTER]
+    failed_once = [e for e in recent if 0 < e.failures_in_a_row < FAILING_AFTER]
     if failing:
         problems.append(
             f"{len(failing)} node(s) failing: "
@@ -256,6 +267,11 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
         if degraded
         else []
     )
+    if failed_once:
+        warnings.append(
+            f"{len(failed_once)} node(s) failed: "
+            f"{', '.join(sorted(e.id for e in failed_once))}"
+        )
 
     return HealthSummary(
         status="degraded" if problems else "ok",