"""Supervision: a background task that dies gets restarted, until it is hopeless.""" import asyncio import pytest from app.flow import supervision from app.flow.events import EventBus from app.flow.supervision import FAILURE_BUDGET, Supervisor @pytest.fixture(autouse=True) def no_backoff(monkeypatch: pytest.MonkeyPatch): """Real backoff would make these tests a minute long.""" monkeypatch.setattr(supervision, "BACKOFF", (0.0,)) async def _settle() -> None: """Let the supervisor work through its restarts.""" for _ in range(50): await asyncio.sleep(0) def test_a_crashing_loop_is_restarted(): attempts = 0 async def loop() -> None: nonlocal attempts attempts += 1 if attempts < 3: raise ConnectionError("broker went away") async def scenario() -> None: supervisor = Supervisor() supervisor.spawn("mqtt", "heating", loop) await _settle() assert attempts == 3 assert supervisor.quarantined == set() await supervisor.cancel_all() asyncio.run(scenario()) def test_a_loop_that_returns_is_left_alone(): """Returning is how a loop says it is finished, not that it failed.""" attempts = 0 async def loop() -> None: nonlocal attempts attempts += 1 async def scenario() -> None: supervisor = Supervisor() supervisor.spawn("cron", "heating", loop) await _settle() assert attempts == 1 await supervisor.cancel_all() asyncio.run(scenario()) def test_a_flow_that_keeps_crashing_is_quarantined(): attempts = 0 events: list[dict] = [] bus = EventBus() bus.publish = events.append # type: ignore[method-assign] async def loop() -> None: nonlocal attempts attempts += 1 raise RuntimeError("nope") async def scenario() -> None: supervisor = Supervisor(bus) supervisor.spawn("mqtt", "heating", loop) await _settle() # Restarting stops once the budget is gone, rather than spinning forever. assert attempts == FAILURE_BUDGET assert supervisor.quarantined == {"heating"} assert [e["type"] for e in events].count("task_crashed") == FAILURE_BUDGET assert [e["type"] for e in events][-1] == "flow_quarantined" await supervisor.cancel_all() asyncio.run(scenario()) def test_the_budget_is_per_flow(): ran = 0 async def failing() -> None: raise RuntimeError("nope") async def healthy() -> None: nonlocal ran ran += 1 await asyncio.sleep(3600) async def scenario() -> None: supervisor = Supervisor() supervisor.spawn("a", "broken", failing) supervisor.spawn("b", "fine", healthy) await _settle() assert supervisor.quarantined == {"broken"} assert ran == 1 await supervisor.cancel_all() asyncio.run(scenario()) def test_cancel_all_is_idempotent(): async def loop() -> None: await asyncio.sleep(3600) async def scenario() -> None: supervisor = Supervisor() supervisor.spawn("mqtt", "heating", loop) await _settle() await supervisor.cancel_all() await supervisor.cancel_all() asyncio.run(scenario())