Startup deletes the work queue's keys from before it moved from pipeline: to queue: (stream, timers, dead letters, parked lists, done markers), logged with what they held; an old stream still holding undelivered work stays, is logged as an error, and counts as queued work for the keys it names

backend/tests/flow/test_redis_queue.py

@@ -1,12 +1,15 @@
 """Redis delivery checks, enabled by FLUKSIO_TEST_REDIS_HOST (and optional PORT)."""
 
+import logging
 import os
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 
+import orjson
 import pytest
 import redis
 
+from fluksio.flow.namespace import touches
 from fluksio.flow.queue import RedisWorkQueue, WorkItem
 
 
@@ -93,3 +96,60 @@ def test_clearing_a_flow_drops_its_queued_claimed_and_held_work(queue):
     assert queue.reclaim_stale(0) == []
     queue.ack(claimed)
     assert [i.node for i in queue.claim(10, 10)] == ["kept.a", ""]
+
+
+def _old_layout(queue, root, *, acked):
+    """What the queue wrote under the state backend's root before it moved."""
+    r = queue._redis
+    item = WorkItem(kind="cascade", node="home.tick", flow="home").to_fields()
+    r.xgroup_create(f"{root}:__queue__", "engine", id="0", mkstream=True)
+    r.xadd(f"{root}:__queue__", item)
+    r.xadd(f"{root}:__queue__", item)
+    # Both delivered and acknowledged, or one claimed and one never handed out.
+    read = r.xreadgroup(
+        "engine", "old", {f"{root}:__queue__": ">"}, None if acked else 1
+    )
+    if acked:
+        r.xack(f"{root}:__queue__", "engine", *[i for i, _ in read[0][1]])
+    flush = WorkItem(kind="flush", node="home.house", flow="home").to_fields()
+    r.zadd(f"{root}:__delayed__", {orjson.dumps(flush): 1787297711.0})
+    r.xadd(f"{root}:__dead__", {**item, "reason": "failed"})
+    r.rpush(f"{root}:__parked__:home", orjson.dumps(item))
+    r.set(f"{root}:__done__:1-0:home.tick", "1")
+
+
+def test_the_old_layout_is_deleted_once_and_the_queue_kept(queue, caplog):
+    root = f"{queue._ns}-old"
+    queue.add(WorkItem(kind="cascade", node="f.a", flow="f"))
+    queue.add_delayed(WorkItem(kind="flush", node="f.b", flow="f"), 9e9)
+    queue.park("f", WorkItem(kind="cascade", node="f.c", flow="f"))
+    current = set(queue._redis.scan_iter(match=f"{queue._ns}:*"))
+    _old_layout(queue, root, acked=True)
+
+    with caplog.at_level(logging.WARNING, logger="fluksio.flow.queue"):
+        queue.drop_legacy(root)
+
+    assert list(queue._redis.scan_iter(match=f"{root}:*")) == []
+    assert set(queue._redis.scan_iter(match=f"{queue._ns}:*")) == current
+    assert len(queue.live_items()) == 3
+    logged = caplog.text
+    assert "1 flush home.house" in logged and "__parked__:home" in logged
+    assert "__dead__" in logged and "__done__:*' (1 keys)" in logged
+    caplog.clear()
+    queue.drop_legacy(root)
+    assert caplog.records == []
+
+
+def test_undelivered_old_work_is_kept_and_counted(queue, caplog):
+    root = f"{queue._ns}-old"
+    _old_layout(queue, root, acked=False)
+
+    with caplog.at_level(logging.WARNING, logger="fluksio.flow.queue"):
+        queue.drop_legacy(root)
+
+    # Claimed and never acknowledged, and never handed out at all.
+    assert set(queue._redis.scan_iter(match=f"{root}:*")) == {f"{root}:__queue__"}
+    assert "2 cascade home.tick" in caplog.text
+    assert [i.node for i in queue.live_items()] == ["home.tick", "home.tick"]
+    assert not touches("home", "__queue__")
+    queue._redis.delete(f"{root}:__queue__")