Journal work before running it, so a crash stops losing messages

Execution was fire-and-forget: an MQTT message or webhook ran a cascade on a ThreadPoolExecutor built for that one wave, and an engine that died halfway through simply lost whatever was in flight. Concurrent triggers each built their own pool, so load meant unbounded threads. Every external trigger is now journaled to a Redis Streams queue before anything runs, and acknowledged only once its cascade finishes. A consumer thread drives cascades on one long-lived pool while node bodies run on another, so a cascade cannot starve the nodes it is waiting for. A reaper reclaims what a dead consumer never acknowledged — verified end to end: work journaled while the engine was stopped runs on restart, and work abandoned mid-cascade comes back as a second delivery. At-least-once needs a guard, so nodes that reach outside are marked non-idempotent and skipped on a redelivery they already completed. Without Redis the queue degrades to an in-memory one that does not pretend to be durable, and interactive callers still run inline. Also fixes two things this turned up: a delay node was sleeping on a worker thread, where a handful of them could occupy the whole pool, and webhooks 404'd whenever MCP was enabled because the app mounted at / answered first for every path. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011LF61rxW1FG5YCD2J9YqjY
2026-08-16 07:57:07 +02:00
parent 2cc25970e3
commit 04329149b3
17 changed files with 1271 additions and 66 deletions
@@ -0,0 +1,220 @@
+"""The work queue, and what the execution service does with it."""
+
+import time
+
+from app.flow.executor import ExecutionService
+from app.flow.messages import DType, MessageSpec
+from app.flow.nodes import Node
+from app.flow.pipeline import Pipeline
+from app.flow.queue import MemoryWorkQueue, WorkItem
+from app.flow.state import MemoryState
+
+
+def test_items_come_back_in_the_order_they_went_in():
+    queue = MemoryWorkQueue()
+    for i in range(3):
+        queue.add(WorkItem(kind="cascade", node=f"f.n{i}", flow="f"))
+
+    claimed = queue.claim(10, 10)
+
+    assert [item.node for item in claimed] == ["f.n0", "f.n1", "f.n2"]
+    # Every item gets an id, which is what idempotency markers hang off.
+    assert all(item.entry_id for item in claimed)
+
+
+def test_claiming_an_empty_queue_waits_and_gives_up():
+    queue = MemoryWorkQueue()
+    started = time.monotonic()
+
+    assert queue.claim(1, 50) == []
+    assert time.monotonic() - started >= 0.04
+
+
+def test_a_delayed_item_stays_put_until_it_is_due():
+    queue = MemoryWorkQueue()
+    queue.add_delayed(WorkItem(kind="cascade", node="f.n", flow="f"), time.time() + 60)
+
+    assert queue.claim(1, 10) == []
+    assert queue.move_due(time.time()) == 0
+
+    assert queue.move_due(time.time() + 61) == 1
+    assert [i.node for i in queue.claim(1, 10)] == ["f.n"]
+
+
+def test_parked_work_comes_back_oldest_first():
+    queue = MemoryWorkQueue()
+    for i in range(3):
+        queue.park("heating", WorkItem(kind="cascade", node=f"f.n{i}", flow="heating"))
+
+    assert [i.node for i in queue.unpark("heating")] == ["f.n0", "f.n1", "f.n2"]
+    # Unparking empties it, so a second resume does not replay the same work.
+    assert queue.unpark("heating") == []
+
+
+def test_a_deleted_flow_leaves_nothing_parked():
+    queue = MemoryWorkQueue()
+    queue.park("gone", WorkItem(kind="cascade", node="gone.n", flow="gone"))
+
+    queue.clear_flow("gone")
+
+    assert queue.unpark("gone") == []
+
+
+def _pipeline_with_a_consumer() -> tuple[Pipeline, Node, MemoryState, list]:
+    """A source whose message a consumer records."""
+    seen: list[float] = []
+
+    def consume(reading, params):
+        seen.append(reading)
+        return {"doubled": reading * 2}
+
+    source = Node(
+        f=lambda params: None,
+        provides=[MessageSpec(name="reading", port="reading", dtype=DType.FLOAT)],
+        name="source",
+    )
+    consumer = Node(
+        f=consume,
+        requires=[MessageSpec(name="reading", port="reading", dtype=DType.FLOAT)],
+        provides=[MessageSpec(name="doubled", port="doubled", dtype=DType.FLOAT)],
+        name="consumer",
+    )
+    source.assign_flow("f", "source")
+    consumer.assign_flow("f", "consumer")
+
+    state = MemoryState()
+    queue = MemoryWorkQueue()
+    pipeline = Pipeline(nodes=[source, consumer], state=state, work_queue=queue)
+    return pipeline, source, state, seen
+
+
+def test_a_trigger_is_journaled_rather_than_run_on_the_spot():
+    pipeline, source, state, seen = _pipeline_with_a_consumer()
+
+    source.inject({"reading": 3.0})
+
+    # Nothing ran yet: the value is in the queue, not in state.
+    assert seen == []
+    assert "f.reading" not in state
+
+    service = ExecutionService(pipeline._queue)
+    service.bind(pipeline)
+    for item in pipeline._queue.claim(10, 10):
+        service._run_item(item)
+
+    assert seen == [3.0]
+    assert state["f.doubled"] == 6.0
+
+
+def test_work_for_a_paused_flow_is_held_and_released_on_resume():
+    pipeline, source, state, seen = _pipeline_with_a_consumer()
+    service = ExecutionService(pipeline._queue)
+    service.bind(pipeline)
+
+    pipeline.pause("f")
+    source.inject({"reading": 1.0})
+    for item in pipeline._queue.claim(10, 10):
+        service._run_item(item)
+
+    assert seen == []
+
+    pipeline.resume("f")
+    for item in pipeline._queue.unpark("f"):
+        pipeline._queue.add(item)
+    for item in pipeline._queue.claim(10, 10):
+        service._run_item(item)
+
+    assert seen == [1.0]
+
+
+def test_work_for_a_stopped_flow_is_dropped():
+    pipeline, source, state, seen = _pipeline_with_a_consumer()
+    stopped = Pipeline(
+        nodes=pipeline.nodes,
+        state=pipeline.state,
+        work_queue=pipeline._queue,
+        disabled_flows={"f"},
+    )
+    service = ExecutionService(pipeline._queue)
+    service.bind(stopped)
+
+    # Reaching the queue at all takes a direct add: trigger drops it earlier.
+    stopped._queue.add(
+        WorkItem(kind="cascade", node="f.source", flow="f", outputs={"f.reading": 1.0})
+    )
+    for item in stopped._queue.claim(10, 10):
+        service._run_item(item)
+
+    assert seen == []
+
+
+def test_an_item_that_keeps_coming_back_is_dead_lettered():
+    pipeline, _source, _state, seen = _pipeline_with_a_consumer()
+    service = ExecutionService(pipeline._queue)
+    service.bind(pipeline)
+
+    item = WorkItem(
+        kind="cascade",
+        node="f.source",
+        flow="f",
+        outputs={"f.reading": 1.0},
+        deliveries=4,
+    )
+    service._run_item(item)
+
+    # Given up on rather than run again, so a poison item cannot loop forever.
+    assert seen == []
+
+
+def test_an_item_for_a_node_that_no_longer_exists_is_dropped():
+    pipeline, _source, _state, seen = _pipeline_with_a_consumer()
+    service = ExecutionService(pipeline._queue)
+    service.bind(pipeline)
+
+    service._run_item(WorkItem(kind="cascade", node="f.removed", flow="f"))
+
+    assert seen == []
+
+
+def test_a_replayed_item_does_not_repeat_a_side_effect():
+    """At-least-once delivery must not mean two of the same outgoing request."""
+    calls: list[float] = []
+
+    def send(reading, params):
+        calls.append(reading)
+        return None
+
+    source = Node(
+        f=lambda params: None,
+        provides=[MessageSpec(name="reading", port="reading", dtype=DType.FLOAT)],
+        name="source",
+    )
+
+    class SendingNode(Node):
+        """Stands in for the built-ins that reach outside."""
+
+        idempotent = False
+
+    sender = SendingNode(
+        f=send,
+        requires=[MessageSpec(name="reading", port="reading", dtype=DType.FLOAT)],
+        name="sender",
+    )
+    source.assign_flow("f", "source")
+    sender.assign_flow("f", "sender")
+
+    queue = MemoryWorkQueue()
+    pipeline = Pipeline(nodes=[source, sender], state=MemoryState(), work_queue=queue)
+    service = ExecutionService(queue)
+    service.bind(pipeline)
+
+    source.inject({"reading": 5.0})
+    (item,) = queue.claim(10, 10)
+    service._run_item(item)
+    assert calls == [5.0]
+
+    # The same item again, as a reaper would hand it back after a crash.
+    item.deliveries = 2
+    service._run_item(item)
+
+    assert calls == [5.0]