app/backend/tests/flow/test_supervision.py

"""Supervision: a background task that dies gets restarted, until it is hopeless."""

import asyncio

import pytest

from fluksio.flow import supervision
from fluksio.flow.events import EventBus
from fluksio.flow.supervision import FAILURE_BUDGET, Supervisor


@pytest.fixture(autouse=True)
def no_backoff(monkeypatch: pytest.MonkeyPatch):
    """Real backoff would make these tests a minute long."""
    monkeypatch.setattr(supervision, "BACKOFF", (0.0,))
    # Long enough that a quarantine holds for the length of an assertion, short
    # enough that the test asking it to expire does not wait five minutes.
    monkeypatch.setattr(supervision, "QUARANTINE_BACKOFF", (0.05,))


async def _settle() -> None:
    """Let the supervisor work through its restarts."""
    for _ in range(50):
        await asyncio.sleep(0)


def test_a_crashing_loop_is_restarted():
    attempts = 0

    async def loop() -> None:
        nonlocal attempts
        attempts += 1
        if attempts < 3:
            raise ConnectionError("broker went away")

    async def scenario() -> None:
        supervisor = Supervisor()
        supervisor.spawn("mqtt", "heating", loop)
        await _settle()

        assert attempts == 3
        assert supervisor.quarantined == set()
        await supervisor.cancel_all()

    asyncio.run(scenario())


def test_a_loop_that_returns_is_left_alone():
    """Returning is how a loop says it is finished, not that it failed."""
    attempts = 0

    async def loop() -> None:
        nonlocal attempts
        attempts += 1

    async def scenario() -> None:
        supervisor = Supervisor()
        supervisor.spawn("cron", "heating", loop)
        await _settle()

        assert attempts == 1
        await supervisor.cancel_all()

    asyncio.run(scenario())


def test_a_flow_that_keeps_crashing_is_stood_down():
    attempts = 0
    events: list[dict] = []
    bus = EventBus()
    bus.publish = events.append  # type: ignore[method-assign]

    async def loop() -> None:
        nonlocal attempts
        attempts += 1
        raise RuntimeError("nope")

    async def scenario() -> None:
        supervisor = Supervisor(bus)
        supervisor.spawn("mqtt", "heating", loop)
        await _settle()

        # Restarting stops once the budget is gone, rather than spinning forever.
        assert attempts == FAILURE_BUDGET
        assert supervisor.quarantined == {"heating"}
        assert [e["type"] for e in events].count("task_crashed") == FAILURE_BUDGET
        quarantine = [e for e in events if e["type"] == "flow_quarantined"][-1]
        assert quarantine["retry_in_s"] == 0.05
        await supervisor.cancel_all()

    asyncio.run(scenario())


def test_one_flows_failure_does_not_stand_another_down():
    ran = 0

    async def failing() -> None:
        raise RuntimeError("nope")

    async def healthy() -> None:
        nonlocal ran
        ran += 1
        await asyncio.sleep(3600)

    async def scenario() -> None:
        supervisor = Supervisor()
        supervisor.spawn("a", "broken", failing)
        supervisor.spawn("b", "fine", healthy)
        await _settle()

        assert supervisor.quarantined == {"broken"}
        assert ran == 1
        await supervisor.cancel_all()

    asyncio.run(scenario())


def test_two_tasks_on_one_flow_do_not_spend_each_others_budget():
    """The house's power flow: an MQTT subscriber and its keepalive publisher.

    Both die with the same broker. Sharing one budget meant the flow was stood
    down after half the restarts either task was allowed on its own.
    """
    tries = {"sub": 0, "pub": 0}

    def crasher(which: str):
        async def loop() -> None:
            tries[which] += 1
            raise ConnectionError("broker went away")

        return loop

    async def scenario() -> None:
        supervisor = Supervisor()
        supervisor.spawn("power.cerbo:mqtt", "power", crasher("sub"))
        supervisor.spawn("power.keepalive:mqtt-out", "power", crasher("pub"))
        await _settle()

        assert tries == {"sub": FAILURE_BUDGET, "pub": FAILURE_BUDGET}
        await supervisor.cancel_all()

    asyncio.run(scenario())


def test_a_quarantine_expires_and_the_task_tries_again():
    """The whole point: a broker that comes back is picked up without a human."""
    attempts = 0

    async def loop() -> None:
        nonlocal attempts
        attempts += 1
        if attempts <= FAILURE_BUDGET:
            raise ConnectionError("broker went away")
        await asyncio.sleep(3600)

    async def scenario() -> None:
        supervisor = Supervisor()
        supervisor.spawn("mqtt", "power", loop)
        await _settle()

        assert attempts == FAILURE_BUDGET
        assert supervisor.quarantined == {"power"}

        # Sit out the rest, which the fixture has shortened to 50ms.
        await asyncio.sleep(0.1)
        await _settle()

        assert attempts == FAILURE_BUDGET + 1
        assert supervisor.quarantined == set()
        await supervisor.cancel_all()

    asyncio.run(scenario())


def test_cancel_all_is_idempotent():
    async def loop() -> None:
        await asyncio.sleep(3600)

    async def scenario() -> None:
        supervisor = Supervisor()
        supervisor.spawn("mqtt", "heating", loop)
        await _settle()

        await supervisor.cancel_all()
        await supervisor.cancel_all()

    asyncio.run(scenario())


def test_cancelling_one_flow_leaves_another_flows_tasks_running():
    """Rebuilding one flow must not take every other flow's loops with it."""

    async def loop() -> None:
        await asyncio.sleep(3600)

    async def scenario() -> None:
        supervisor = Supervisor()
        supervisor.spawn("gone.sub", "gone", loop)
        supervisor.spawn("fine.sub", "fine", loop)
        await _settle()

        survivor = supervisor._tasks["fine.sub"]
        await supervisor.cancel_flow("gone")

        assert list(supervisor._tasks) == ["fine.sub"]
        assert not survivor.done()
        await supervisor.cancel_all()

    asyncio.run(scenario())


def test_cancelling_one_flow_lifts_only_its_quarantine():
    async def failing() -> None:
        raise RuntimeError("nope")

    async def scenario() -> None:
        supervisor = Supervisor()
        supervisor.spawn("a", "broken", failing)
        supervisor.spawn("b", "also-broken", failing)
        await _settle()

        assert supervisor.quarantined == {"broken", "also-broken"}

        await supervisor.cancel_flow("broken")

        assert supervisor.quarantined == {"also-broken"}
        # And its budget, or the flow would be quarantined again on the first
        # crash after the rebuild that was meant to fix it.
        assert "a" not in supervisor._failures
        assert "b" in supervisor._failures
        await supervisor.cancel_all()

    asyncio.run(scenario())