From 7a7315043d9fd5e92809afc77a7ae9c33ad35b11 Mon Sep 17 00:00:00 2001 From: stroblme Date: Sun, 1 Nov 2026 10:59:28 +0100 Subject: [PATCH] Alerts: a delivery no channel took leaves the cooldown unspent --- backend/fluksio/flow/alerts.py | 22 +++++++++++++++++----- backend/tests/flow/test_alerts.py | 22 ++++++++++++++++++++++ backend/tests/flow/test_device_node.py | 3 ++- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/backend/fluksio/flow/alerts.py b/backend/fluksio/flow/alerts.py index 4450cba..da1b8fc 100644 --- a/backend/fluksio/flow/alerts.py +++ b/backend/fluksio/flow/alerts.py @@ -306,8 +306,8 @@ class AlertManager: # Held back, not spent: the cooldown only starts once it is sent. self._suppressed[key] = self._suppressed.get(key, 0) + 1 return + previous = self._last_sent.get(key) now = self._stamp(key) - await self._remember(key, now) held = self._suppressed.pop(key, 0) if held: @@ -316,9 +316,19 @@ class AlertManager: ) names = {name for rule in rules for name in rule.channels} + delivered = False for channel in self.config.channels: if channel.enabled and channel.name in names: - await self.send(channel, alert) + delivered = await self.send(channel, alert) or delivered + if delivered: + await self._remember(key, now) + else: + # Nobody heard it, so the next one tries again rather than wait + # out a cooldown nothing started. + if previous is None: + self._last_sent.pop(key, None) + else: + self._last_sent[key] = previous # ------------------------------------------------------------------------- # Deciding whether to speak @@ -391,8 +401,8 @@ class AlertManager: async def send( self, channel: Channel, alert: Alert, raise_on_error: bool = False - ) -> None: - """Deliver one alert. + ) -> bool: + """Deliver one alert, and say whether it went. A failure is logged and swallowed, because one dead channel must not stop the others hearing about the same fault. The test button passes @@ -405,7 +415,7 @@ class AlertManager: if raise_on_error: raise logger.error("Channel '%s' has unusable settings: %s", channel.name, exc) - return + return False try: if channel.kind == "ntfy": @@ -424,6 +434,8 @@ class AlertManager: if raise_on_error: raise logger.error("Could not alert through '%s': %s", channel.name, exc) + return False + return True async def _send_ntfy(self, config: dict[str, Any], alert: Alert) -> None: server = str(config.get("server") or "https://ntfy.sh").rstrip("/") diff --git a/backend/tests/flow/test_alerts.py b/backend/tests/flow/test_alerts.py index 9c58eb7..41a0231 100644 --- a/backend/tests/flow/test_alerts.py +++ b/backend/tests/flow/test_alerts.py @@ -60,6 +60,7 @@ def manager(clock: Clock) -> tuple[AlertManager, list[Alert]]: async def capture(channel, alert): sent.append(alert) + return True alerts.send = capture # type: ignore[method-assign] return alerts, sent @@ -264,6 +265,27 @@ def test_a_ledger_that_fails_does_not_stop_the_alert(): assert len(sent) == 1 +def test_an_alert_no_channel_took_does_not_spend_its_cooldown(): + clock = Clock() + alerts, sent = manager(clock) + delivered = iter([False, True]) + + async def flaky(channel, alert): + sent.append(alert) + return next(delivered) + + alerts.send = flaky # type: ignore[method-assign] + + async def scenario(): + await alerts.handle(error_event()) + clock.advance(10) + await alerts.handle(error_event()) + + asyncio.run(scenario()) + + assert len(sent) == 2 + + def test_an_alert_held_back_by_the_ceiling_does_not_spend_its_cooldown(): clock = Clock() alerts, sent = manager(clock) diff --git a/backend/tests/flow/test_device_node.py b/backend/tests/flow/test_device_node.py index 3598e1d..d2bdcb7 100644 --- a/backend/tests/flow/test_device_node.py +++ b/backend/tests/flow/test_device_node.py @@ -141,8 +141,9 @@ def test_a_device_away_after_a_restart_alerts_nobody(tmp_path: Path) -> None: ), ) - async def capture(channel: Channel, alert: Alert) -> None: + async def capture(channel: Channel, alert: Alert) -> bool: sent.append(alert) + return True alerts.send = capture # type: ignore[method-assign]