Startup deletes the work queue's keys from before it moved from pipeline: to queue: (stream, timers, dead letters, parked lists, done markers), logged with what they held; an old stream still holding undelivered work stays, is logged as an error, and counts as queued work for the keys it names

backend/fluksio/flow/queue.py

@@ -22,7 +22,7 @@ import secrets
 import threading
 import time
 from abc import ABC, abstractmethod
-from collections import OrderedDict, deque
+from collections import Counter, OrderedDict, deque
 from collections.abc import Callable, Iterable
 from dataclasses import dataclass, field
 from typing import Any, cast
@@ -72,6 +72,9 @@ STRUCTURES = frozenset(
         "__dead__",
     }
 )
+#: Where the Redis queue lived until it got a root of its own (2026-08): the
+#: state backend's. Nothing has read what it left there since (`drop_legacy`).
+LEGACY_ROOT = "pipeline"
 
 # A promotion is one Redis operation, including when the caller loses its
 # connection before hearing the result. Write first: Lua does not roll back a
@@ -200,6 +203,12 @@ def owned_by(item: WorkItem, flow: str) -> bool:
     )
 
 
+def _described(items: Iterable[WorkItem]) -> str:
+    """What *items* are, for a log line: ``2 flush home.house, 1 cascade ...``."""
+    counts = Counter((item.kind, item.node) for item in items)
+    return ", ".join(f"{n} {kind} {node}" for (kind, node), n in counts.most_common())
+
+
 def done_marker(key: str) -> tuple[str, str]:
     """The entry id and node a done marker is for: `RedisWorkQueue._done_key`
     read back. Split at the last colon, since an entry id from the due lane
@@ -468,6 +477,12 @@ class WorkQueue(ABC):
         another engine on the same server, which a rename cannot hold."""
         return []
 
+    def drop_legacy(self, root: str = LEGACY_ROOT) -> None:
+        """Delete what the queue kept under *root* before it moved. For
+        startup, before anything lists state keys. A queue in this process
+        has nothing to delete."""
+        return None
+
     @abstractmethod
     def rename(
         self, rename: Rename, progress: Callable[[str], None] | None = None
@@ -776,6 +791,8 @@ class RedisWorkQueue(WorkQueue):
         self._hold_lock = threading.Condition()
         self._holding = False
         self._adding = 0
+        # The old layout's stream, while it holds work nobody delivered.
+        self._legacy = ""
         self._ensure_group()
         self._drop_stale_consumers()
 
@@ -814,6 +831,64 @@ class RedisWorkQueue(WorkQueue):
                 except redis.ResponseError:
                     pass
 
+    def drop_legacy(self, root: str = LEGACY_ROOT) -> None:
+        """Delete the keys the queue wrote under *root* before it had its own.
+
+        They sit among the state backend's keys, as families `namespace` does
+        not know, so a key check, a delete or a rename would trip over them.
+        The stream goes once its group has delivered and acknowledged every
+        entry. One still holding work stays and is logged as an error on
+        every start. Nothing runs that work, but deleting it is the
+        operator's call, and until then `live_items` counts it, so a key it
+        names is not given to a new flow. Timers, parked work, dead letters
+        and done markers go regardless, each logged with what it held. Run
+        again, it finds nothing and says nothing.
+        """
+        stream = f"{root}:__queue__"
+        if live := self._live_entries(stream):
+            self._legacy = stream
+            logger.error(
+                "Kept the old queue stream '%s': %d entries were never delivered "
+                "or acknowledged (%s). Nothing runs them, and a new flow cannot "
+                "take a key they name. Read them with XRANGE, then delete the "
+                "key by hand (DEPLOY.md)",
+                stream,
+                len(live),
+                _described(WorkItem.from_fields(f, i) for i, f in live),
+            )
+        for key in (stream, f"{root}:__dead__"):
+            if key != self._legacy and self._redis.exists(key):
+                entries = cast(int, self._redis.xlen(key))
+                self._redis.delete(key)
+                logger.warning(
+                    "Deleted the old queue key '%s' (%d entries)", key, entries
+                )
+        timers = f"{root}:__delayed__"
+        for key in [timers, *self._redis.scan_iter(f"{root}:__parked__:*")]:
+            raw = cast(
+                list[str],
+                self._redis.zrange(key, 0, -1)
+                if key == timers
+                else self._redis.lrange(key, 0, -1),
+            )
+            if raw:
+                items = [WorkItem.from_fields(orjson.loads(r), "") for r in raw]
+                self._redis.delete(key)
+                logger.warning(
+                    "Deleted the old queue key '%s' (%d items: %s)",
+                    key,
+                    len(items),
+                    _described(items),
+                )
+        if markers := list(self._redis.scan_iter(f"{root}:{DONE}:*")):
+            self._redis.delete(*markers)
+            logger.warning(
+                "Deleted the old done markers '%s:%s:*' (%d keys)",
+                root,
+                DONE,
+                len(markers),
+            )
+
     def _parked_key(self, flow: str) -> str:
         return f"{self._ns}:__parked__:{flow}"
 
@@ -1069,6 +1144,13 @@ class RedisWorkQueue(WorkQueue):
             WorkItem.from_fields(orjson.loads(raw), "")
             for raw in cast(list[str], self._redis.lrange(self._held_key, 0, -1))
         ]
+        if self._legacy:
+            # Nothing runs it unless someone moves it back by hand, and then
+            # a new flow under a key it names would inherit it (`drop_legacy`).
+            items += [
+                WorkItem.from_fields(fields, "")
+                for _, fields in self._live_entries(self._legacy)
+            ]
         return items
 
     def _group_info(self, stream: str = "") -> dict[str, Any]: