Health summary: a node is failing after two failures in a row; one is a warning, a good run ends both

The summary read degraded for 15 minutes after any single node_error, even
once the node had run well again and including one bad input from a visitor.
The controller now counts a node's failures since it last ran well and notes
when it recovered; both carry across rebuilds and an acknowledgement resets
the count. Two in a row inside the 15 minute window is "failing" (a problem),
one is listed under warnings, and nodes.failing keeps meaning "failing now".

backend/tests/api/routes/test_observability.py

@@ -456,7 +456,8 @@ def test_a_node_failing_on_every_call_makes_the_summary_degraded(
 ) -> None:
     """A node that loads and then raises each time it runs is not in `error`.
 
-    It is named while its last failure is recent, and drops out once it is not.
+    Failing twice running makes the instance degraded, once is a warning, and
+    a failure that is no longer recent counts for neither.
     """
     import time
 
@@ -464,15 +465,28 @@ def test_a_node_failing_on_every_call_makes_the_summary_degraded(
 
     controller = client.app.state.flow_controller
     before = controller.loaded
+    now = time.time()
     controller.loaded = {
         "house.calc": LoadedNode(
-            id="house.calc", flow="house", last_error="boom", last_error_ts=time.time()
+            id="house.calc",
+            flow="house",
+            last_error="boom",
+            last_error_ts=now,
+            failures_in_a_row=2,
+        ),
+        "house.once": LoadedNode(
+            id="house.once",
+            flow="house",
+            last_error="boom",
+            last_error_ts=now,
+            failures_in_a_row=1,
         ),
         "house.old": LoadedNode(
             id="house.old",
             flow="house",
             last_error="boom",
-            last_error_ts=time.time() - 3600,
+            last_error_ts=now - 3600,
+            failures_in_a_row=5,
         ),
     }
     try:
@@ -483,7 +497,38 @@ def test_a_node_failing_on_every_call_makes_the_summary_degraded(
     assert body["status"] == "degraded"
     assert body["nodes"]["failing"] == 1
     (problem,) = [p for p in body["problems"] if "failing" in p]
-    assert "house.calc" in problem and "house.old" not in problem
+    assert "house.calc" in problem
+    assert "house.once" not in problem and "house.old" not in problem
+    (warning,) = [w for w in body["warnings"] if "failed" in w]
+    assert "house.once" in warning and "house.calc" not in warning
+
+
+def test_one_failure_is_a_warning_and_leaves_the_status_alone(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    import time
+
+    from fluksio.flow.controller import LoadedNode
+
+    controller = client.app.state.flow_controller
+    before = controller.loaded
+    controller.loaded = {
+        "house.once": LoadedNode(
+            id="house.once",
+            flow="house",
+            last_error="boom",
+            last_error_ts=time.time(),
+            failures_in_a_row=1,
+        )
+    }
+    try:
+        body = client.get(f"{PREFIX}/summary", headers=superuser_token_headers).json()
+    finally:
+        controller.loaded = before
+
+    assert body["status"] == "ok"
+    assert body["nodes"]["failing"] == 0
+    assert body["warnings"] == ["1 node(s) failed: house.once"]
 
 
 def test_a_superuser_can_flush_the_collector_now(

backend/tests/flow/test_node_failures.py

@@ -0,0 +1,58 @@
+"""A node's failures in a row, which the health summary reads."""
+
+import asyncio
+from pathlib import Path
+
+from fluksio.flow.controller import FlowController, LoadedNode
+from fluksio.flow.events import EventBus
+from fluksio.flow.store import FlowStore
+
+NODE = "house.calc"
+
+
+def _failure(ts: float) -> dict:
+    return {"type": "node_error", "node": NODE, "error": "boom", "ts": ts}
+
+
+def test_failures_count_until_the_node_runs_well(tmp_path: Path) -> None:
+    async def scenario() -> LoadedNode:
+        events = EventBus()
+        events.bind(asyncio.get_running_loop())
+        controller = FlowController(FlowStore(tmp_path / "flows"), events=events)
+        controller.loaded[NODE] = LoadedNode(id=NODE, flow="house")
+        watcher = asyncio.create_task(controller._watch_failures())
+        await asyncio.sleep(0)
+
+        events.publish(_failure(1.0))
+        events.publish(_failure(2.0))
+        await asyncio.sleep(0.05)
+        assert controller.loaded[NODE].failures_in_a_row == 2
+
+        events.publish({"type": "node_executed", "node": NODE, "ts": 3.0})
+        await asyncio.sleep(0.05)
+        watcher.cancel()
+        return controller.loaded[NODE]
+
+    entry = asyncio.run(scenario())
+
+    assert entry.failures_in_a_row == 0
+    assert entry.last_ok_ts == 3.0
+    # What it failed with stays for the canvas until someone acknowledges it.
+    assert entry.last_error == "boom"
+
+
+def test_a_rebuilt_node_keeps_its_failures() -> None:
+    old = LoadedNode(
+        id=NODE,
+        flow="house",
+        last_error="boom",
+        last_error_ts=5.0,
+        failures_in_a_row=3,
+        last_ok_ts=1.0,
+    )
+    new = LoadedNode(id=NODE, flow="house")
+
+    new.inherit_failure(old)
+
+    assert (new.last_error, new.last_error_ts) == ("boom", 5.0)
+    assert (new.failures_in_a_row, new.last_ok_ts) == (3, 1.0)