Rename migrations for live state and the work queue: state keys move with RENAMENX in batches (type, TTL and history capacity kept), the queue rebuilds its streams with the same ids, group and pending entries, rewrites timers, parked and held items and done markers, and can hold new work in a list released in order; nothing calls them yet

This commit is contained in:
2026-11-02 19:50:54 +01:00
parent ebcc480a8c
commit 46b57de4ec
5 changed files with 981 additions and 83 deletions
+17 -68
View File
@@ -3,7 +3,7 @@ FLUKSIO_TEST_REDIS_HOST (and optional PORT).
They prove on a real server that a stream copied into a temp key with its ids,
group and pending entries and swapped in with RENAME looks like the original to
the engine (docs/private/id_refac_rename.md, 3.1).
the engine (`queue.rebuild_stream`, docs/private/id_refac_rename.md, 3.1).
"""
import os
@@ -13,6 +13,9 @@ import uuid
import pytest
import redis
from fluksio.flow.namespace import Rename
from fluksio.flow.queue import rebuild_stream
GROUP = "engine"
@@ -45,67 +48,7 @@ def _id(entry_id: str) -> tuple[int, int]:
return int(ms), int(seq)
def _rename_flow(fields: dict[str, str]) -> dict[str, str]:
return {**fields, "flow": "new"} if fields.get("flow") == "old" else fields
# ponytail: unpaged. S3 moves it into RedisWorkQueue and pages XRANGE and XPENDING.
def rebuild(r, stream: str, rewrite, *, swap: bool = True) -> None:
"""Copy *stream* into a temp key with the same ids, group and pending
entries, rewriting the entries still to run, and swap it in."""
tmp = f"{stream}:rename"
r.delete(tmp)
info = r.xinfo_stream(stream)
(group,) = [g for g in r.xinfo_groups(stream) if g["name"] == GROUP]
last = group["last-delivered-id"]
pel = (
r.xpending_range(stream, GROUP, "-", "+", count=group["pending"])
if group["pending"]
else []
)
pending = {p["message_id"] for p in pel}
pipe = r.pipeline(transaction=False)
# MKSTREAM so XSETID finds the key even when nothing is copied.
pipe.xgroup_create(tmp, GROUP, id=last, mkstream=True)
for consumer in r.xinfo_consumers(stream, GROUP):
pipe.xgroup_createconsumer(tmp, GROUP, consumer["name"])
copied = set()
for entry_id, fields in r.xrange(stream):
live = entry_id in pending or _id(entry_id) > _id(last)
pipe.xadd(tmp, rewrite(fields) if live else fields, id=entry_id)
copied.add(entry_id)
pipe.execute_command(
"XSETID",
tmp,
info["last-generated-id"],
"ENTRIESADDED",
info["entries-added"],
"MAXDELETEDID",
info["max-deleted-entry-id"],
)
# Redis caps ENTRIESREAD at the stream's entries-added counter, which is
# still the copied count before XSETID, so it is set only now.
if group["entries-read"] is not None:
pipe.execute_command(
"XGROUP", "SETID", tmp, GROUP, last, "ENTRIESREAD", group["entries-read"]
)
for p in pel:
if p["message_id"] in copied:
pipe.xclaim(
tmp,
GROUP,
p["consumer"],
0,
[p["message_id"]],
idle=p["time_since_delivered"],
retrycount=p["times_delivered"],
force=True,
justid=True,
)
if swap:
pipe.rename(tmp, stream)
pipe.execute()
OLD_NEW = Rename("old", "new")
def _pel(r, stream: str) -> list[dict]:
@@ -162,7 +105,7 @@ def test_rebuild_keeps_ids_group_and_pending_entries(r, ns):
idle_before = {p["message_id"]: p["time_since_delivered"] for p in _pel(r, stream)}
started = time.monotonic()
rebuild(r, stream, _rename_flow)
rebuild_stream(r, stream, OLD_NEW)
idle_after = {p["message_id"]: p["time_since_delivered"] for p in _pel(r, stream)}
elapsed_ms = (time.monotonic() - started) * 1000
@@ -202,26 +145,32 @@ def test_rebuild_of_an_emptied_stream_keeps_its_id_counter(r, ns):
r.xdel(stream, *ids)
before = _snapshot(r, stream)
rebuild(r, stream, _rename_flow)
rebuild_stream(r, stream, OLD_NEW)
assert _snapshot(r, stream) == before
assert _id(r.xadd(stream, {"flow": "new"})) > _id(ids[-1])
def test_a_rerun_after_a_crash_on_either_side_of_rename_converges(r, ns):
def test_a_rerun_after_a_crash_on_either_side_of_rename_converges(r, ns, monkeypatch):
stream = f"{ns}:queue"
_busy_stream(r, stream)
original = r.xrange(stream)
def crash(*args):
raise redis.ConnectionError
# Crash before RENAME: the stream is untouched and the temp key is left.
rebuild(r, stream, _rename_flow, swap=False)
with monkeypatch.context() as patch:
patch.setattr(r, "rename", crash)
with pytest.raises(redis.ConnectionError):
rebuild_stream(r, stream, OLD_NEW)
assert r.xrange(stream) == original
assert r.exists(f"{stream}:rename")
rebuild(r, stream, _rename_flow)
rebuild_stream(r, stream, OLD_NEW)
once = _snapshot(r, stream)
# Crash after RENAME, before the journal checkpoint: the rerun changes nothing.
rebuild(r, stream, _rename_flow)
rebuild_stream(r, stream, OLD_NEW)
assert _snapshot(r, stream) == once
assert not r.exists(f"{stream}:rename")