Health summary: a node is failing after two failures in a row; one is a warning, a good run ends both

The summary read degraded for 15 minutes after any single node_error, even
once the node had run well again and including one bad input from a visitor.
The controller now counts a node's failures since it last ran well and notes
when it recovered; both carry across rebuilds and an acknowledgement resets
the count. Two in a row inside the 15 minute window is "failing" (a problem),
one is listed under warnings, and nodes.failing keeps meaning "failing now".

backend/tests/api/routes/test_observability.py

@@ -456,7 +456,8 @@ def test_a_node_failing_on_every_call_makes_the_summary_degraded(
 ) -> None:
     """A node that loads and then raises each time it runs is not in `error`.
 
-    It is named while its last failure is recent, and drops out once it is not.
+    Failing twice running makes the instance degraded, once is a warning, and
+    a failure that is no longer recent counts for neither.
     """
     import time
 
@@ -464,15 +465,28 @@ def test_a_node_failing_on_every_call_makes_the_summary_degraded(
 
     controller = client.app.state.flow_controller
     before = controller.loaded
+    now = time.time()
     controller.loaded = {
         "house.calc": LoadedNode(
-            id="house.calc", flow="house", last_error="boom", last_error_ts=time.time()
+            id="house.calc",
+            flow="house",
+            last_error="boom",
+            last_error_ts=now,
+            failures_in_a_row=2,
+        ),
+        "house.once": LoadedNode(
+            id="house.once",
+            flow="house",
+            last_error="boom",
+            last_error_ts=now,
+            failures_in_a_row=1,
         ),
         "house.old": LoadedNode(
             id="house.old",
             flow="house",
             last_error="boom",
-            last_error_ts=time.time() - 3600,
+            last_error_ts=now - 3600,
+            failures_in_a_row=5,
         ),
     }
     try:
@@ -483,7 +497,38 @@ def test_a_node_failing_on_every_call_makes_the_summary_degraded(
     assert body["status"] == "degraded"
     assert body["nodes"]["failing"] == 1
     (problem,) = [p for p in body["problems"] if "failing" in p]
-    assert "house.calc" in problem and "house.old" not in problem
+    assert "house.calc" in problem
+    assert "house.once" not in problem and "house.old" not in problem
+    (warning,) = [w for w in body["warnings"] if "failed" in w]
+    assert "house.once" in warning and "house.calc" not in warning
+
+
+def test_one_failure_is_a_warning_and_leaves_the_status_alone(
+    client: TestClient, superuser_token_headers: dict[str, str]
+) -> None:
+    import time
+
+    from fluksio.flow.controller import LoadedNode
+
+    controller = client.app.state.flow_controller
+    before = controller.loaded
+    controller.loaded = {
+        "house.once": LoadedNode(
+            id="house.once",
+            flow="house",
+            last_error="boom",
+            last_error_ts=time.time(),
+            failures_in_a_row=1,
+        )
+    }
+    try:
+        body = client.get(f"{PREFIX}/summary", headers=superuser_token_headers).json()
+    finally:
+        controller.loaded = before
+
+    assert body["status"] == "ok"
+    assert body["nodes"]["failing"] == 0
+    assert body["warnings"] == ["1 node(s) failed: house.once"]
 
 
 def test_a_superuser_can_flush_the_collector_now(