Alerts: a delivery no channel took leaves the cooldown unspent

backend/fluksio/flow/alerts.py

@@ -306,8 +306,8 @@ class AlertManager:
             # Held back, not spent: the cooldown only starts once it is sent.
             self._suppressed[key] = self._suppressed.get(key, 0) + 1
             return
+        previous = self._last_sent.get(key)
         now = self._stamp(key)
-        await self._remember(key, now)
 
         held = self._suppressed.pop(key, 0)
         if held:
@@ -316,9 +316,19 @@ class AlertManager:
             )
 
         names = {name for rule in rules for name in rule.channels}
+        delivered = False
         for channel in self.config.channels:
             if channel.enabled and channel.name in names:
-                await self.send(channel, alert)
+                delivered = await self.send(channel, alert) or delivered
+        if delivered:
+            await self._remember(key, now)
+        else:
+            # Nobody heard it, so the next one tries again rather than wait
+            # out a cooldown nothing started.
+            if previous is None:
+                self._last_sent.pop(key, None)
+            else:
+                self._last_sent[key] = previous
 
     # -------------------------------------------------------------------------
     # Deciding whether to speak
@@ -391,8 +401,8 @@ class AlertManager:
 
     async def send(
         self, channel: Channel, alert: Alert, raise_on_error: bool = False
-    ) -> None:
-        """Deliver one alert.
+    ) -> bool:
+        """Deliver one alert, and say whether it went.
 
         A failure is logged and swallowed, because one dead channel must not
         stop the others hearing about the same fault. The test button passes
@@ -405,7 +415,7 @@ class AlertManager:
             if raise_on_error:
                 raise
             logger.error("Channel '%s' has unusable settings: %s", channel.name, exc)
-            return
+            return False
 
         try:
             if channel.kind == "ntfy":
@@ -424,6 +434,8 @@ class AlertManager:
             if raise_on_error:
                 raise
             logger.error("Could not alert through '%s': %s", channel.name, exc)
+            return False
+        return True
 
     async def _send_ntfy(self, config: dict[str, Any], alert: Alert) -> None:
         server = str(config.get("server") or "https://ntfy.sh").rstrip("/")

backend/tests/flow/test_alerts.py

@@ -60,6 +60,7 @@ def manager(clock: Clock) -> tuple[AlertManager, list[Alert]]:
 
     async def capture(channel, alert):
         sent.append(alert)
+        return True
 
     alerts.send = capture  # type: ignore[method-assign]
     return alerts, sent
@@ -264,6 +265,27 @@ def test_a_ledger_that_fails_does_not_stop_the_alert():
     assert len(sent) == 1
 
 
+def test_an_alert_no_channel_took_does_not_spend_its_cooldown():
+    clock = Clock()
+    alerts, sent = manager(clock)
+    delivered = iter([False, True])
+
+    async def flaky(channel, alert):
+        sent.append(alert)
+        return next(delivered)
+
+    alerts.send = flaky  # type: ignore[method-assign]
+
+    async def scenario():
+        await alerts.handle(error_event())
+        clock.advance(10)
+        await alerts.handle(error_event())
+
+    asyncio.run(scenario())
+
+    assert len(sent) == 2
+
+
 def test_an_alert_held_back_by_the_ceiling_does_not_spend_its_cooldown():
     clock = Clock()
     alerts, sent = manager(clock)

backend/tests/flow/test_device_node.py

@@ -141,8 +141,9 @@ def test_a_device_away_after_a_restart_alerts_nobody(tmp_path: Path) -> None:
         ),
     )
 
-    async def capture(channel: Channel, alert: Alert) -> None:
+    async def capture(channel: Channel, alert: Alert) -> bool:
         sent.append(alert)
+        return True
 
     alerts.send = capture  # type: ignore[method-assign]