Memory-queue ids carry a per-instance boot token, so a restart without Redis no longer overwrites earlier run rows

This commit is contained in:
2026-10-14 23:33:25 +02:00
parent 2ac997ce37
commit 245e68449f
2 changed files with 17 additions and 1 deletions
+5 -1
View File
@@ -18,6 +18,7 @@ from __future__ import annotations
import heapq import heapq
import logging import logging
import secrets
import threading import threading
import time import time
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
@@ -272,12 +273,15 @@ class MemoryWorkQueue(WorkQueue):
self._in_flight = 0 self._in_flight = 0
self._counter = 0 self._counter = 0
self._seq = 0 self._seq = 0
# The ids become `FlowRun.id`, a primary key. Counting from zero alone,
# a restart would overwrite the previous boot's rows.
self._boot = secrets.token_hex(4)
self._lock = threading.Lock() self._lock = threading.Lock()
self._wake = threading.Condition(self._lock) self._wake = threading.Condition(self._lock)
def _next_id(self) -> str: def _next_id(self) -> str:
self._seq += 1 self._seq += 1
return f"mem-{self._seq}" return f"mem-{self._boot}-{self._seq}"
def add(self, item: WorkItem) -> None: def add(self, item: WorkItem) -> None:
with self._wake: with self._wake:
+12
View File
@@ -25,6 +25,18 @@ def test_items_come_back_in_the_order_they_went_in():
assert all(item.entry_id for item in claimed) assert all(item.entry_id for item in claimed)
def test_two_queues_never_hand_out_the_same_id():
"""A restart without Redis must not reuse the last boot's run ids."""
first, second = MemoryWorkQueue(), MemoryWorkQueue()
for queue in (first, second):
queue.add(WorkItem(kind="cascade", node="f.n", flow="f"))
ids = [queue.claim(1, 10)[0].entry_id for queue in (first, second)]
assert ids[0] != ids[1]
assert all(len(entry_id) <= 64 for entry_id in ids)
def test_claiming_an_empty_queue_waits_and_gives_up(): def test_claiming_an_empty_queue_waits_and_gives_up():
queue = MemoryWorkQueue() queue = MemoryWorkQueue()
started = time.monotonic() started = time.monotonic()