Alerts: cooldowns survive a restart, and a connection must stay down a minute before it alerts

The alert manager kept its cooldowns and its ceiling per hour in memory, so a node that stayed down was announced again by every restart. Each delivered alert is now a row in the new alert_sent table (additive migration), loaded for the last seven days when the manager starts and pruned as new ones come in. A ledger that fails never stops an alert. The manager's clock is wall time for the same reason. A node_health "down" alert also waits 60 s (HEALTH_GRACE_S) in a pending task per node; a later event that is not "down" cancels it, as does stopping the manager, so a broker restarting with the stack alerts nobody. The cooldown used to be stamped before the ceiling per hour was checked, so an alert the ceiling held back still spent its cooldown. It is stamped only once the alert is sent now.
2026-10-28 12:46:42 +01:00
parent 00842fa175
commit bfaaa7cd03
8 changed files with 425 additions and 20 deletions
@@ -9,6 +9,7 @@ import pytest
 from fluksio.flow.alerts import (
    ALERTING_EVENTS,
    FLAP_THRESHOLD,
+    LEDGER_RETAIN_S,
    RATE_LIMIT,
    Alert,
    AlertManager,
@@ -20,6 +21,20 @@ from fluksio.flow.alerts import (
 from fluksio.flow.events import EventBus


+class Ledger:
+    """What the database holds, without one."""
+
+    def __init__(self) -> None:
+        self.rows: list[tuple[str, float]] = []
+
+    def load(self, since: float) -> list[tuple[str, float]]:
+        return [row for row in self.rows if row[1] >= since]
+
+    def add(self, key: str, ts: float, keep_since: float) -> None:
+        self.rows = [row for row in self.rows if row[1] >= keep_since]
+        self.rows.append((key, ts))
+
+
 class Clock:
    """A hand-wound clock, so cooldowns take no real time."""

@@ -39,7 +54,9 @@ def manager(clock: Clock) -> tuple[AlertManager, list[Alert]]:
        channels=[Channel(name="phone", kind="ntfy", config={"topic": "t"})],
        rules=[Rule(events=[], channels=["phone"], cooldown_s=900)],
    )
-    alerts = AlertManager(EventBus(), config=config, now=clock)
+    # No grace: these tests are about what is sent, not how long a connection
+    # has to stay down first.
+    alerts = AlertManager(EventBus(), config=config, now=clock, grace_s=0)

    async def capture(channel, alert):
        sent.append(alert)
@@ -153,6 +170,174 @@ def test_a_storm_is_capped():
    assert len(sent) == RATE_LIMIT


+def test_a_cooldown_survives_a_restart():
+    """A node that stays down was announced again by every restart."""
+    clock, ledger = Clock(), Ledger()
+
+    async def scenario() -> list[Alert]:
+        first, sent_before = manager(clock)
+        first._ledger = ledger
+        await first.handle(error_event())
+        clock.advance(120)
+
+        second, sent_after = manager(clock)
+        second._ledger = ledger
+        await second.restore()
+        await second.handle(error_event())
+        assert len(sent_before) == 1
+        clock.advance(1000)
+        await second.handle(error_event())
+        return sent_after
+
+    sent_after = asyncio.run(scenario())
+
+    # Silent inside the cooldown, and speaking again once it has passed.
+    assert len(sent_after) == 1
+
+
+def test_the_ceiling_per_hour_survives_a_restart():
+    clock, ledger = Clock(), Ledger()
+    clock.now = 10_000.0
+    ledger.rows = [
+        (f"node_error:heating.n{i}", clock.now - 60) for i in range(RATE_LIMIT)
+    ]
+    alerts, sent = manager(clock)
+    alerts._ledger = ledger
+
+    async def scenario():
+        await alerts.restore()
+        await alerts.handle(error_event("heating.new"))
+
+    asyncio.run(scenario())
+
+    assert sent == []
+
+
+def test_a_clock_set_back_does_not_hold_a_key_silent():
+    clock, ledger = Clock(), Ledger()
+    clock.now = 100.0
+    ledger.rows = [("node_error:heating.pump", 50_000.0)]
+    alerts, sent = manager(clock)
+    alerts._ledger = ledger
+
+    async def scenario():
+        await alerts.restore()
+        clock.advance(1000)
+        await alerts.handle(error_event())
+
+    asyncio.run(scenario())
+
+    assert len(sent) == 1
+
+
+def test_sent_alerts_are_written_to_the_ledger_and_old_ones_dropped():
+    clock, ledger = Clock(), Ledger()
+    clock.now = LEDGER_RETAIN_S * 2
+    ledger.rows = [("node_error:heating.old", 1.0)]
+    alerts, _sent = manager(clock)
+    alerts._ledger = ledger
+
+    asyncio.run(alerts.handle(error_event()))
+
+    assert ledger.rows == [("node_error:heating.pump", clock.now)]
+
+
+def test_a_ledger_that_fails_does_not_stop_the_alert():
+    clock = Clock()
+    alerts, sent = manager(clock)
+
+    class Broken:
+        def load(self, since):
+            raise OSError("disk")
+
+        def add(self, key, ts, keep_since):
+            raise OSError("disk")
+
+    alerts._ledger = Broken()  # type: ignore[assignment]
+
+    async def scenario():
+        await alerts.restore()
+        await alerts.handle(error_event())
+
+    asyncio.run(scenario())
+
+    assert len(sent) == 1
+
+
+def test_an_alert_held_back_by_the_ceiling_does_not_spend_its_cooldown():
+    clock = Clock()
+    alerts, sent = manager(clock)
+
+    async def scenario():
+        for i in range(RATE_LIMIT):
+            await alerts.handle(error_event(f"heating.n{i}"))
+        await alerts.handle(error_event("heating.late"))
+        # An hour on, the ceiling has cleared and the same fault is still due.
+        clock.advance(3601)
+        await alerts.handle(error_event("heating.late"))
+
+    asyncio.run(scenario())
+
+    assert [a.node for a in sent][-1] == "heating.late"
+    assert "1 more since the last alert" in sent[-1].body
+
+
+def down(node: str = "heating.mqtt") -> dict:
+    return {"type": "node_health", "node": node, "health": "down", "detail": "refused"}
+
+
+def up(node: str = "heating.mqtt") -> dict:
+    return {"type": "node_health", "node": node, "health": "ok"}
+
+
+def with_grace(grace_s: float) -> tuple[AlertManager, list[Alert]]:
+    alerts, sent = manager(Clock())
+    alerts.grace_s = grace_s
+    return alerts, sent
+
+
+def test_a_connection_that_comes_back_within_the_grace_alerts_nobody():
+    alerts, sent = with_grace(0.05)
+
+    async def scenario():
+        await alerts.handle(down())
+        await alerts.handle(up())
+        await asyncio.sleep(0.1)
+
+    asyncio.run(scenario())
+
+    assert sent == []
+
+
+def test_a_connection_that_stays_down_is_told_after_the_grace():
+    alerts, sent = with_grace(0.05)
+
+    async def scenario():
+        await alerts.handle(down())
+        assert sent == []
+        await asyncio.sleep(0.1)
+
+    asyncio.run(scenario())
+
+    assert [a.title for a in sent] == ["heating.mqtt lost its connection"]
+
+
+def test_a_pending_alert_is_dropped_when_the_manager_stops():
+    alerts, sent = with_grace(0.05)
+
+    async def scenario():
+        await alerts.handle(down())
+        runner = asyncio.create_task(alerts.run())
+        await asyncio.sleep(0.01)
+        runner.cancel()
+        await asyncio.gather(runner, return_exceptions=True)
+        await asyncio.sleep(0.1)
+
+    asyncio.run(scenario())
+
+    assert sent == []
+
+
 def test_nothing_is_sent_without_a_rule():
    clock = Clock()
    alerts, sent = manager(clock)
@@ -0,0 +1,26 @@
+"""The alert ledger: what the manager remembers across a restart."""
+
+import pytest
+from sqlalchemy import delete
+from sqlmodel import Session
+
+from fluksio.core.db import engine
+from fluksio.flow.alert_ledger import AlertLedger
+from fluksio.models import AlertSent
+
+
+@pytest.fixture(autouse=True)
+def _empty(db: Session):  # noqa: ARG001 - the schema has to exist first
+    with Session(engine) as session:
+        session.exec(delete(AlertSent))
+        session.commit()
+    yield
+
+
+def test_a_sent_alert_comes_back_and_old_ones_are_pruned():
+    ledger = AlertLedger()
+    ledger.add("node_error:house.calc", 100.0, keep_since=0.0)
+    ledger.add("node_health:house.mqtt", 5000.0, keep_since=1000.0)
+
+    assert ledger.load(since=0.0) == [("node_health:house.mqtt", 5000.0)]
+    assert ledger.load(since=6000.0) == []