app/backend/tests/flow/test_queue.py

"""The work queue, and what the execution service does with it."""

import threading
import time

from fluksio.flow import executor
from fluksio.flow import queue as queue_module
from fluksio.flow.executor import ExecutionService
from fluksio.flow.messages import DType, MessageSpec
from fluksio.flow.nodes import Node
from fluksio.flow.pipeline import Pipeline
from fluksio.flow.queue import MemoryWorkQueue, WorkItem
from fluksio.flow.state import MemoryState


def test_items_come_back_in_the_order_they_went_in():
    queue = MemoryWorkQueue()
    for i in range(3):
        queue.add(WorkItem(kind="cascade", node=f"f.n{i}", flow="f"))

    claimed = queue.claim(10, 10)

    assert [item.node for item in claimed] == ["f.n0", "f.n1", "f.n2"]
    # Every item gets an id, which is what idempotency markers hang off.
    assert all(item.entry_id for item in claimed)


def test_claiming_an_empty_queue_waits_and_gives_up():
    queue = MemoryWorkQueue()
    started = time.monotonic()

    assert queue.claim(1, 50) == []
    assert time.monotonic() - started >= 0.04


def test_a_delayed_item_stays_put_until_it_is_due():
    queue = MemoryWorkQueue()
    queue.add_delayed(WorkItem(kind="cascade", node="f.n", flow="f"), time.time() + 60)

    assert queue.claim(1, 10) == []
    assert queue.move_due(time.time()) == 0

    assert queue.move_due(time.time() + 61) == 1
    assert [i.node for i in queue.claim(1, 10)] == ["f.n"]


def test_the_queue_says_when_the_soonest_delayed_item_is_due():
    """What lets the engine sleep to a deadline rather than poll for it."""
    queue = MemoryWorkQueue()
    assert queue.next_due() is None

    queue.add_delayed(WorkItem(kind="cascade", node="f.late", flow="f"), 500.0)
    queue.add_delayed(WorkItem(kind="cascade", node="f.soon", flow="f"), 100.0)
    assert queue.next_due() == 100.0

    queue.move_due(200.0)
    assert queue.next_due() == 500.0


def test_scheduling_says_a_deadline_moved():
    """A delay scheduled for 200ms must cut short a sleep to the next second."""
    queue = MemoryWorkQueue()
    woken = threading.Event()
    queue.on_delayed = woken.set

    queue.add_delayed(WorkItem(kind="cascade", node="f.n", flow="f"), time.time() + 0.2)

    assert woken.is_set()


def test_a_due_timer_is_claimed_before_the_backlog():
    """Lateness is what a timer is measured by; queued work waits on no clock."""
    queue = MemoryWorkQueue()
    for i in range(3):
        queue.add(WorkItem(kind="cascade", node=f"f.queued{i}", flow="f"))
    queue.add_delayed(WorkItem(kind="cascade", node="f.timer", flow="f"), 100.0)

    queue.move_due(200.0)

    assert [i.node for i in queue.claim(10, 10)][0] == "f.timer"


def test_a_delay_fires_without_waiting_out_the_poll():
    """The whole point: a 200ms delay is 200ms late, not up to a second."""
    queue = MemoryWorkQueue()
    service = ExecutionService(queue)
    dispatched: list[float] = []
    service._dispatch = lambda item: dispatched.append(time.monotonic())  # type: ignore[method-assign]
    service.start()
    try:
        started = time.monotonic()
        queue.add_delayed(
            WorkItem(kind="cascade", node="f.n", flow="f"), time.time() + 0.2
        )
        # Claimed by the consumer thread, which is what calls _dispatch.
        deadline = time.monotonic() + 2.0
        while not dispatched and time.monotonic() < deadline:
            time.sleep(0.01)
    finally:
        service.stop()

    assert dispatched, "the deferred item never ran"
    late = dispatched[0] - started - 0.2
    # A fixed one-second poll made this up to 1.0s; the budget is generous
    # because CI is not a real-time machine.
    assert late < 0.3, f"fired {late:.3f}s late"


def test_parked_work_comes_back_oldest_first():
    queue = MemoryWorkQueue()
    for i in range(3):
        queue.park("heating", WorkItem(kind="cascade", node=f"f.n{i}", flow="heating"))

    assert [i.node for i in queue.unpark("heating")] == ["f.n0", "f.n1", "f.n2"]
    # Unparking empties it, so a second resume does not replay the same work.
    assert queue.unpark("heating") == []


def test_a_deleted_flow_leaves_nothing_parked():
    queue = MemoryWorkQueue()
    queue.park("gone", WorkItem(kind="cascade", node="gone.n", flow="gone"))

    queue.clear_flow("gone")

    assert queue.unpark("gone") == []


def test_claimed_work_counts_as_in_flight_until_it_is_acknowledged():
    """The health tile's "in flight" reads zero without this."""
    queue = MemoryWorkQueue()
    queue.add(WorkItem(kind="cascade", node="f.n", flow="f"))

    assert queue.stats()["pending"] == 0
    # The stream length was never a backlog, so the key is gone from both queues.
    assert "depth" not in queue.stats()

    (item,) = queue.claim(1, 10)
    assert queue.stats()["pending"] == 1

    queue.ack(item)
    assert queue.stats()["pending"] == 0


def test_work_waiting_to_be_claimed_is_the_backlog():
    """`pending` is what is running; an engine hours behind reports it as idle."""
    queue = MemoryWorkQueue()
    queue.add(WorkItem(kind="cascade", node="f.n", flow="f"))

    assert queue.stats()["backlog"] == 1
    assert queue.stats()["pending"] == 0

    (item,) = queue.claim(1, 10)
    assert queue.stats()["backlog"] == 0
    assert queue.stats()["pending"] == 1

    queue.ack(item)
    assert queue.stats()["backlog"] == 0


def test_a_sustained_backlog_says_the_engine_is_behind():
    """A flow enqueuing faster than the pool drains produced no signal at all."""
    events: list[dict] = []
    queue = MemoryWorkQueue()
    service = ExecutionService(queue)
    service._publish = events.append  # type: ignore[method-assign]
    for _ in range(executor.BACKLOG_DEGRADED):
        queue.add(WorkItem(kind="cascade", node="f.n", flow="f"))

    for _ in range(executor.BACKLOG_STRIKES - 1):
        service._check_backlog()
    assert events == []

    service._check_backlog()
    assert [e["type"] for e in events] == ["engine_degraded"]
    assert service.stats()["behind"] is True

    # Said once, not once every five seconds for as long as it lasts.
    service._check_backlog()
    assert len(events) == 1

    # And a drained queue clears it, so the next backlog is announced again.
    queue.claim(executor.BACKLOG_DEGRADED, 10)
    service._check_backlog()
    assert service.stats()["behind"] is False


def _pipeline_with_a_consumer() -> tuple[Pipeline, Node, MemoryState, list]:
    """A source whose message a consumer records."""
    seen: list[float] = []

    def consume(reading, params):
        seen.append(reading)
        return {"doubled": reading * 2}

    source = Node(
        f=lambda params: None,
        provides=[MessageSpec(name="reading", port="reading", dtype=DType.FLOAT)],
        name="source",
    )
    consumer = Node(
        f=consume,
        requires=[MessageSpec(name="reading", port="reading", dtype=DType.FLOAT)],
        provides=[MessageSpec(name="doubled", port="doubled", dtype=DType.FLOAT)],
        name="consumer",
    )
    source.assign_flow("f", "source")
    consumer.assign_flow("f", "consumer")

    state = MemoryState()
    queue = MemoryWorkQueue()
    pipeline = Pipeline(nodes=[source, consumer], state=state, work_queue=queue)
    return pipeline, source, state, seen


def test_a_trigger_is_journaled_rather_than_run_on_the_spot():
    pipeline, source, state, seen = _pipeline_with_a_consumer()

    source.inject({"reading": 3.0})

    # Nothing ran yet: the value is in the queue, not in state.
    assert seen == []
    assert "f.reading" not in state

    service = ExecutionService(pipeline._queue)
    service.bind(pipeline)
    for item in pipeline._queue.claim(10, 10):
        service._run_item(item)

    assert seen == [3.0]
    assert state["f.doubled"] == 6.0


def test_work_for_a_paused_flow_is_held_and_released_on_resume():
    pipeline, source, state, seen = _pipeline_with_a_consumer()
    service = ExecutionService(pipeline._queue)
    service.bind(pipeline)

    pipeline.pause("f")
    source.inject({"reading": 1.0})
    for item in pipeline._queue.claim(10, 10):
        service._run_item(item)

    assert seen == []

    pipeline.resume("f")
    for item in pipeline._queue.unpark("f"):
        pipeline._queue.add(item)
    for item in pipeline._queue.claim(10, 10):
        service._run_item(item)

    assert seen == [1.0]


def test_a_step_runs_one_held_item_and_leaves_the_flow_paused():
    pipeline, source, _state, seen = _pipeline_with_a_consumer()
    service = ExecutionService(pipeline._queue)
    service.bind(pipeline)

    pipeline.pause("f")
    source.inject({"reading": 1.0})
    source.inject({"reading": 2.0})
    for item in pipeline._queue.claim(10, 10):
        service._run_item(item)
    assert seen == []

    assert service.step("f") == "f.source"

    assert seen == [1.0]
    # Still paused, and the second value is still waiting for the next step.
    assert pipeline.is_paused("f")
    assert [i.outputs for i in pipeline._queue.unpark("f")] == [{"f.reading": 2.0}]


def test_stepping_a_flow_with_nothing_held_says_so_rather_than_failing():
    pipeline, _source, _state, _seen = _pipeline_with_a_consumer()
    service = ExecutionService(pipeline._queue)
    service.bind(pipeline)

    pipeline.pause("f")

    assert service.step("f") is None


def test_work_for_a_stopped_flow_is_dropped():
    pipeline, source, state, seen = _pipeline_with_a_consumer()
    stopped = Pipeline(
        nodes=pipeline.nodes,
        state=pipeline.state,
        work_queue=pipeline._queue,
        disabled_flows={"f"},
    )
    service = ExecutionService(pipeline._queue)
    service.bind(stopped)

    # Reaching the queue at all takes a direct add: trigger drops it earlier.
    stopped._queue.add(
        WorkItem(kind="cascade", node="f.source", flow="f", outputs={"f.reading": 1.0})
    )
    for item in stopped._queue.claim(10, 10):
        service._run_item(item)

    assert seen == []


def test_an_item_that_keeps_coming_back_is_dead_lettered():
    pipeline, _source, _state, seen = _pipeline_with_a_consumer()
    service = ExecutionService(pipeline._queue)
    service.bind(pipeline)

    item = WorkItem(
        kind="cascade",
        node="f.source",
        flow="f",
        outputs={"f.reading": 1.0},
        deliveries=4,
    )
    service._run_item(item)

    # Given up on rather than run again, so a poison item cannot loop forever.
    assert seen == []


def test_an_item_for_a_node_that_no_longer_exists_is_dropped():
    pipeline, _source, _state, seen = _pipeline_with_a_consumer()
    service = ExecutionService(pipeline._queue)
    service.bind(pipeline)

    service._run_item(WorkItem(kind="cascade", node="f.removed", flow="f"))

    assert seen == []


def test_a_replayed_item_does_not_repeat_a_side_effect():
    """At-least-once delivery must not mean two of the same outgoing request."""
    calls: list[float] = []

    def send(reading, params):
        calls.append(reading)
        return None

    source = Node(
        f=lambda params: None,
        provides=[MessageSpec(name="reading", port="reading", dtype=DType.FLOAT)],
        name="source",
    )

    class SendingNode(Node):
        """Stands in for the built-ins that reach outside."""

        idempotent = False

    sender = SendingNode(
        f=send,
        requires=[MessageSpec(name="reading", port="reading", dtype=DType.FLOAT)],
        name="sender",
    )
    source.assign_flow("f", "source")
    sender.assign_flow("f", "sender")

    queue = MemoryWorkQueue()
    pipeline = Pipeline(nodes=[source, sender], state=MemoryState(), work_queue=queue)
    service = ExecutionService(queue)
    service.bind(pipeline)

    source.inject({"reading": 5.0})
    (item,) = queue.claim(10, 10)
    service._run_item(item)
    assert calls == [5.0]

    # The same item again, as a reaper would hand it back after a crash.
    item.deliveries = 2
    service._run_item(item)

    assert calls == [5.0]


# -----------------------------------------------------------------------------
# Telling the queue that a long node is working, not lost
#
# What marks an item abandoned is nobody touching it. A node with no timeout
# may run for hours, so the engine holding it says so on a timer — and an
# engine that died says nothing, which is the distinction the reaper needs.
# -----------------------------------------------------------------------------


class RecordingQueue(MemoryWorkQueue):
    """A memory queue that writes down what it was asked to hold on to."""

    def __init__(self) -> None:
        super().__init__()
        self.touched: list[list[str]] = []

    def touch(self, entry_ids: list[str]) -> None:
        self.touched.append(list(entry_ids))


def test_work_in_flight_is_touched_until_it_finishes(monkeypatch):
    monkeypatch.setattr(executor, "TOUCH_INTERVAL_S", 0.0)
    monkeypatch.setattr(executor, "DELAYED_INTERVAL_S", 0.05)
    running = threading.Event()
    release = threading.Event()

    def slow(reading, params):
        running.set()
        release.wait(5)
        return {"doubled": reading * 2}

    source = Node(
        f=lambda params: None,
        provides=[MessageSpec(name="reading", dtype=DType.FLOAT)],
        name="source",
    )
    consumer = Node(
        f=slow,
        requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
        provides=[MessageSpec(name="doubled", dtype=DType.FLOAT)],
        name="consumer",
    )
    source.assign_flow("f", "source")
    consumer.assign_flow("f", "consumer")

    queue = RecordingQueue()
    pipeline = Pipeline(nodes=[source, consumer], state=MemoryState(), work_queue=queue)
    service = ExecutionService(queue)
    service.bind(pipeline)
    service.start()
    try:
        source.inject({"reading": 3.0})
        assert running.wait(5)
        # Give the timer a couple of passes while the node is still in there.
        time.sleep(0.2)
        held = [ids for ids in queue.touched if ids]
        assert held, "a running item was never touched"

        release.set()
        time.sleep(0.3)
        # Once it is done it is acknowledged, so there is nothing to hold.
        assert queue.touched[-1] == []
    finally:
        release.set()
        service.stop()


def test_no_more_is_claimed_than_the_pool_can_run():
    """A backlog belongs in the queue, not inside the process.

    Claiming ahead of the pool used to leave every waiting item counted as a
    busy cascade and holding its journal entry open, so four cascade threads
    reported hundreds in flight on an engine that was merely behind.
    """
    release = threading.Event()

    def slow(reading, params):
        release.wait(5)
        return {"doubled": reading * 2}

    source = Node(
        f=lambda params: None,
        provides=[MessageSpec(name="reading", dtype=DType.FLOAT)],
        name="source",
    )
    consumer = Node(
        f=slow,
        requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
        provides=[MessageSpec(name="doubled", dtype=DType.FLOAT)],
        name="consumer",
    )
    source.assign_flow("f", "source")
    consumer.assign_flow("f", "consumer")

    queue = MemoryWorkQueue()
    pipeline = Pipeline(nodes=[source, consumer], state=MemoryState(), work_queue=queue)
    service = ExecutionService(queue)
    service.bind(pipeline)
    service.start()
    try:
        for i in range(40):
            queue.add(
                WorkItem(
                    kind="cascade",
                    node="f.source",
                    flow="f",
                    outputs={"f.reading": float(i)},
                )
            )
        time.sleep(0.5)

        stats = service.stats()
        assert stats["cascades_busy"] <= service.max_cascades
        # And the journal entries of what is only waiting are still free.
        assert stats["pending"] <= service.max_cascades
        # Waiting is not idle: the rest of the forty is the backlog.
        assert stats["backlog"] >= 30
    finally:
        release.set()
        service.stop()


# -----------------------------------------------------------------------------
# Emissions: values a node publishes while it is still running
# -----------------------------------------------------------------------------


def _emitting_pipeline() -> tuple[Pipeline, Node, MemoryState, list]:
    """A source with a streaming port, and a consumer that records every value."""
    seen: list[float] = []

    def consume(chunk, params):
        seen.append(chunk)
        return None

    source = Node(
        f=lambda params: None,
        provides=[
            MessageSpec(name="chunk", port="chunk", dtype=DType.FLOAT, stream=True)
        ],
        name="source",
    )
    consumer = Node(
        f=consume,
        requires=[MessageSpec(name="chunk", port="chunk", dtype=DType.FLOAT)],
        name="consumer",
    )
    source.assign_flow("f", "source")
    consumer.assign_flow("f", "consumer")

    state = MemoryState()
    queue = MemoryWorkQueue()
    pipeline = Pipeline(nodes=[source, consumer], state=state, work_queue=queue)
    return pipeline, source, state, seen


def test_every_emitted_chunk_reaches_the_consumer():
    """A consumer slower than its producer must not skip what it missed.

    Reading state instead would give whichever chunk is newest by the time the
    item runs — fine for a temperature, lossy for a second of speech.
    """
    pipeline, source, state, seen = _emitting_pipeline()
    service = ExecutionService(pipeline._queue)
    service.bind(pipeline)

    # Both emitted before either item is claimed, so state has moved on.
    pipeline.publish_emission(source, {"f.chunk": 1.0})
    pipeline.publish_emission(source, {"f.chunk": 2.0})
    assert state["f.chunk"] == 2.0

    for item in pipeline._queue.claim(10, 10):
        service._run_item(item)

    assert seen == [1.0, 2.0]


def test_a_delivered_emission_does_not_write_state_a_second_time():
    """The value in state stays the newest one, whenever an item is claimed."""
    pipeline, source, state, seen = _emitting_pipeline()
    service = ExecutionService(pipeline._queue)
    service.bind(pipeline)

    pipeline.publish_emission(source, {"f.chunk": 1.0})
    # What the node returned at the end, after the emission it made on the way.
    pipeline.apply_outputs(source, {"f.chunk": 9.0})

    for item in pipeline._queue.claim(10, 10):
        service._run_item(item)

    assert seen[0] == 1.0
    # Re-applying the carried chunk here is what would undo the final value.
    assert state["f.chunk"] == 9.0


def test_a_throttled_emission_wakes_nothing():
    pipeline, source, _state, seen = _emitting_pipeline()
    source.provides["f.chunk"] = MessageSpec(
        name="f.chunk", port="chunk", dtype=DType.FLOAT, stream=True, interval=60
    )
    service = ExecutionService(pipeline._queue)
    service.bind(pipeline)

    pipeline.publish_emission(source, {"f.chunk": 1.0})
    pipeline.publish_emission(source, {"f.chunk": 2.0})
    for item in pipeline._queue.claim(10, 10):
        service._run_item(item)

    # The first is let through; the second is held by the interval, and a value
    # nothing published is nothing to wake on.
    assert seen == [1.0]


def test_a_due_timer_is_claimable_while_the_backlog_is_not():
    """What a saturated engine uses to get a motor's stop out.

    Every cascade slot held by a long node used to mean the due lane was not
    merely behind the backlog but unread, because the capacity gate sits in
    front of the claim.
    """
    queue = MemoryWorkQueue()
    queue.add(WorkItem(kind="cascade", node="f.backlog", flow="f"))
    queue.add_delayed(
        WorkItem(kind="cascade", node="f.timer", flow="f"), time.time() - 1
    )
    assert queue.move_due(time.time()) == 1

    claimed = queue.claim(10, 10, due_only=True)
    assert [item.node for item in claimed] == ["f.timer"]

    # And the backlog is still there for whoever comes next.
    assert [item.node for item in queue.claim(10, 10)] == ["f.backlog"]


def test_a_due_only_claim_of_nothing_does_not_take_the_backlog():
    queue = MemoryWorkQueue()
    queue.add(WorkItem(kind="cascade", node="f.backlog", flow="f"))

    assert queue.claim(10, 10, due_only=True) == []
    assert len(queue.claim(10, 10)) == 1


def test_done_markers_do_not_grow_without_bound():
    """The Redis side expires them after an hour; this one is capped by count."""
    queue = MemoryWorkQueue()
    for i in range(queue_module.DONE_MARKERS + 500):
        queue.mark_done(f"entry-{i}", "f.node")

    assert len(queue._done) == queue_module.DONE_MARKERS
    # The newest are what redelivery would ask about.
    assert queue.was_done(f"entry-{queue_module.DONE_MARKERS + 499}", "f.node")
    assert not queue.was_done("entry-0", "f.node")