"""Alerting: failing loudly once, not thirty-six thousand times.""" import asyncio import re from pathlib import Path import pytest from fluksio.flow.alerts import ( ALERTING_EVENTS, FLAP_THRESHOLD, RATE_LIMIT, Alert, AlertManager, AlertsConfig, Channel, Rule, describe, ) from fluksio.flow.events import EventBus class Clock: """A hand-wound clock, so cooldowns take no real time.""" def __init__(self) -> None: self.now = 0.0 def __call__(self) -> float: return self.now def advance(self, seconds: float) -> None: self.now += seconds def manager(clock: Clock) -> tuple[AlertManager, list[Alert]]: sent: list[Alert] = [] config = AlertsConfig( channels=[Channel(name="phone", kind="ntfy", config={"topic": "t"})], rules=[Rule(events=[], channels=["phone"], cooldown_s=900)], ) alerts = AlertManager(EventBus(), config=config, now=clock) async def capture(channel, alert): sent.append(alert) alerts.send = capture # type: ignore[method-assign] return alerts, sent def error_event(node: str = "heating.pump") -> dict: return {"type": "node_error", "flow": "heating", "node": node, "error": "boom"} def test_a_failure_reaches_the_channel(): clock = Clock() alerts, sent = manager(clock) asyncio.run(alerts.handle(error_event())) assert [a.title for a in sent] == ["heating.pump failed"] assert sent[0].body == "boom" def test_ordinary_traffic_is_not_an_alert(): clock = Clock() alerts, sent = manager(clock) asyncio.run(alerts.handle({"type": "message_value", "name": "heating.temp"})) asyncio.run(alerts.handle({"type": "node_executed", "node": "heating.pump"})) assert sent == [] def test_the_same_failure_repeating_is_one_alert(): clock = Clock() alerts, sent = manager(clock) async def scenario(): for _ in range(50): await alerts.handle(error_event()) clock.advance(1) asyncio.run(scenario()) assert len(sent) == 1 def test_the_cooldown_ends_and_says_what_was_missed(): clock = Clock() alerts, sent = manager(clock) async def scenario(): await alerts.handle(error_event()) for _ in range(4): clock.advance(10) await alerts.handle(error_event()) clock.advance(1000) await alerts.handle(error_event()) asyncio.run(scenario()) assert len(sent) == 2 assert "4 more since the last alert" in sent[1].body def test_different_nodes_alert_separately(): clock = Clock() alerts, sent = manager(clock) async def scenario(): await alerts.handle(error_event("heating.pump")) await alerts.handle(error_event("heating.valve")) asyncio.run(scenario()) assert len(sent) == 2 def test_a_flapping_connection_goes_quiet(): """A device dropping every few seconds is one story, not one alert each.""" clock = Clock() alerts, sent = manager(clock) async def scenario(): for _ in range(FLAP_THRESHOLD * 2 + 6): await alerts.handle( {"type": "node_health", "node": "heating.pump", "health": "down"} ) clock.advance(5) await alerts.handle( {"type": "node_health", "node": "heating.pump", "health": "ok"} ) clock.advance(5) asyncio.run(scenario()) # The first drop is worth knowing about; the rest is noise. assert len(sent) == 1 def test_a_storm_is_capped(): clock = Clock() alerts, sent = manager(clock) async def scenario(): for i in range(RATE_LIMIT + 20): await alerts.handle(error_event(f"heating.n{i}")) clock.advance(1) asyncio.run(scenario()) assert len(sent) == RATE_LIMIT def test_nothing_is_sent_without_a_rule(): clock = Clock() alerts, sent = manager(clock) alerts.config = AlertsConfig( channels=[Channel(name="phone", kind="ntfy", config={"topic": "t"})], rules=[Rule(events=["queue_unavailable"], channels=["phone"])], ) asyncio.run(alerts.handle(error_event())) assert sent == [] def test_alerting_can_be_switched_off(): clock = Clock() alerts, sent = manager(clock) alerts.config = alerts.config.model_copy(update={"enabled": False}) asyncio.run(alerts.handle(error_event())) assert sent == [] @pytest.mark.parametrize( ("event", "expected"), [ ( {"type": "flow_quarantined", "flow": "heating"}, "Flow 'heating' was quarantined", ), ( {"type": "queue_unavailable", "error": "gone"}, "The work queue is unreachable", ), ({"type": "engine_degraded", "reason": "lag"}, "The engine is struggling"), ({"type": "node_health", "health": "ok"}, None), # The engine publishes `health`, not `status`: reading the wrong key # meant a device dropping never alerted anyone. ( {"type": "node_health", "node": "heating.pump", "health": "down"}, "heating.pump lost its connection", ), ], ) def test_every_alerting_event_reads_as_a_sentence(event, expected): alert = describe(event) assert (alert.title if alert else None) == expected def test_a_dashboard_channel_publishes_the_alert_as_a_record(): published: list[tuple[str, dict]] = [] config = AlertsConfig( channels=[ Channel(name="panel", kind="dashboard", config={"message": "house.notice"}) ], rules=[Rule(events=[], channels=["panel"])], ) alerts = AlertManager(EventBus(), config=config, now=Clock()) alerts.publish = lambda name, value: published.append((name, value)) asyncio.run(alerts.handle(error_event())) assert len(published) == 1 name, record = published[0] assert name == "house.notice" # Flat named scalars, which is what a notification widget binds. assert record["title"] == "heating.pump failed" assert record["severity"] == "error" assert all(isinstance(v, str) for v in record.values()) def test_a_dashboard_channel_without_a_message_says_so(): alerts = AlertManager(EventBus(), now=Clock()) alerts.publish = lambda name, value: None channel = Channel(name="panel", kind="dashboard") with pytest.raises(ValueError): asyncio.run( alerts.send(channel, Alert(title="t", body="b"), raise_on_error=True) ) ALERTS_ROUTE = Path(__file__).parents[3] / "frontend/src/routes/_layout/alerts.tsx" @pytest.mark.skipif(not ALERTS_ROUTE.exists(), reason="no frontend in this checkout") def test_the_alerts_screen_offers_every_alerting_event(): """The chooser lists the events by hand, so it can drift out of this set. Only the ids have to agree — the labels beside them are UI copy. """ block = re.search(r"const EVENTS[^=]*= \[(.*?)\n\]", ALERTS_ROUTE.read_text(), re.S) assert block, "the EVENTS list moved — this test needs following" assert set(re.findall(r'\["(\w+)"', block.group(1))) == ALERTING_EVENTS