Rename migrations for live state and the work queue: state keys move with RENAMENX in batches (type, TTL and history capacity kept), the queue rebuilds its streams with the same ids, group and pending entries, rewrites timers, parked and held items and done markers, and can hold new work in a list released in order; nothing calls them yet

This commit is contained in:
2026-11-02 19:50:54 +01:00
parent ebcc480a8c
commit 46b57de4ec
5 changed files with 981 additions and 83 deletions
+379 -8
View File
@@ -23,14 +23,15 @@ import threading
import time
from abc import ABC, abstractmethod
from collections import OrderedDict, deque
from collections.abc import Callable
from collections.abc import Callable, Iterable
from dataclasses import dataclass, field
from typing import Any, cast
import orjson
import redis
from fluksio.flow.state import CONNECT_TIMEOUT_S, SOCKET_TIMEOUT_S
from fluksio.flow.namespace import Rename
from fluksio.flow.state import CONNECT_TIMEOUT_S, SOCKET_TIMEOUT_S, move_keys
logger = logging.getLogger(__name__)
@@ -54,6 +55,23 @@ STALE_CONSUMER_IDLE_MS = 3_600_000
DONE_MARKERS = 10_000
# The Redis queue keeps them as ``<namespace>:__done__:<entry id>:<node>``.
DONE = "__done__"
# How many entries a rename reads, or writes, per round trip.
RENAME_PAGE = 500
#: Every structure under the Redis queue's namespace. `rename` rewrites the
#: names in all of them but the dead letters, which are history. One added
#: here holds names too, or has to say why it does not.
STRUCTURES = frozenset(
{
"__queue__",
"__due__",
"__delayed__",
"__parked__",
"__parked_flows__",
"__held__",
DONE,
"__dead__",
}
)
# A promotion is one Redis operation, including when the caller loses its
# connection before hearing the result. Write first: Lua does not roll back a
@@ -72,6 +90,24 @@ end
return #due
"""
# Work held during a rename goes into the backlog oldest first, behind
# everything already there. As in `_MOVE_DUE`, an item leaves the held list
# only once its XADD went through. 100 per call, so Redis is never blocked
# for long.
_RELEASE = """
local held = redis.call('LRANGE', KEYS[1], 0, 99)
for _, raw in ipairs(held) do
local fields = {}
for key, value in pairs(cjson.decode(raw)) do
fields[#fields + 1] = key
fields[#fields + 1] = value
end
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[1], '*', unpack(fields))
redis.call('LPOP', KEYS[1])
end
return #held
"""
@dataclass
class WorkItem:
@@ -172,6 +208,94 @@ def done_marker(key: str) -> tuple[str, str]:
return entry_id, node
def _stream_id(entry_id: str) -> tuple[int, int]:
ms, _, seq = entry_id.partition("-")
return int(ms), int(seq)
def rebuild_stream(client: redis.Redis, stream: str, rename: Rename) -> None:
"""Rewrite the names in the entries of *stream* still to run, keeping
every entry id, the group and its consumers, each pending entry's owner,
delivery count and idle time, and the stream's id counters.
An entry cannot be changed in place, and one added again gets a new id,
which is what runs, events and done markers are filed under, and loses its
delivery count. So the stream is copied into a temporary key with its ids
and swapped in with one RENAME, the only commit point: a rerun after a
crash on either side of it converges. Acknowledged entries are history and
copied as they are. Nothing may write to the stream meanwhile. Needs Redis
7.0 (docs/private/id_refac_rename.md, 3.1).
"""
tmp = f"{stream}:rename"
client.delete(tmp)
info = cast(dict[str, Any], client.xinfo_stream(stream))
groups = cast(list[dict[str, Any]], client.xinfo_groups(stream))
group = next(g for g in groups if g["name"] == GROUP)
last = str(group["last-delivered-id"])
pel: list[dict[str, Any]] = []
start = "-"
while rows := cast(
list[dict[str, Any]],
client.xpending_range(stream, GROUP, min=start, max="+", count=RENAME_PAGE),
):
pel += rows
start = f"({rows[-1]['message_id']}"
pending = {row["message_id"] for row in pel}
pipe = client.pipeline(transaction=False)
# MKSTREAM, so XSETID finds the key when there is nothing to copy.
pipe.xgroup_create(tmp, GROUP, id=last, mkstream=True)
for consumer in cast(list[dict[str, Any]], client.xinfo_consumers(stream, GROUP)):
pipe.xgroup_createconsumer(tmp, GROUP, consumer["name"])
copied: set[str] = set()
start = "-"
while entries := cast(
list[tuple[str, dict[str, str]]],
client.xrange(stream, min=start, max="+", count=RENAME_PAGE),
):
for entry_id, fields in entries:
live = entry_id in pending or _stream_id(entry_id) > _stream_id(last)
renamed = rename.fields(fields) if live else None
pipe.xadd(tmp, cast(Any, renamed or fields), id=entry_id)
if entry_id in pending:
copied.add(entry_id)
pipe.execute()
start = f"({entries[-1][0]}"
# redis-py has no XSETID of its own, and leaves the generic call untyped.
cast(Any, pipe).execute_command(
"XSETID",
tmp,
info["last-generated-id"],
"ENTRIESADDED",
info["entries-added"],
"MAXDELETEDID",
info["max-deleted-entry-id"],
)
# Redis caps ENTRIESREAD at the stream's entries-added counter, which only
# XSETID has set. Unknown stays unknown.
if group["entries-read"] is not None:
pipe.xgroup_setid(tmp, GROUP, last, entries_read=group["entries-read"])
# A pending entry whose stream entry was trimmed is not recreated, as
# `_reclaim` drops it too. XCLAIM would still create its consumer.
for row in pel:
if row["message_id"] in copied:
pipe.xclaim(
tmp,
GROUP,
row["consumer"],
0,
[row["message_id"]],
idle=row["time_since_delivered"],
retrycount=row["times_delivered"],
force=True,
justid=True,
)
pipe.execute()
# On its own: a pipeline without a transaction runs every command even
# after one of them failed.
client.rename(tmp, stream)
class WorkQueue(ABC):
"""What the execution service pulls from."""
@@ -309,6 +433,52 @@ class WorkQueue(ABC):
def close(self) -> None:
"""Release whatever the queue holds."""
# -------------------------------------------------------------------------
# Renaming a flow key
# -------------------------------------------------------------------------
@abstractmethod
def hold(self) -> None:
"""Divert `add` into a held list until `release`, so a rename can
rewrite the queue while listeners keep journaling work."""
@abstractmethod
def release(self) -> None:
"""Move the held work into the backlog, oldest first and behind
everything already there, and stop diverting."""
@property
@abstractmethod
def held(self) -> bool:
"""Whether `add` is diverted."""
@abstractmethod
def release_held(self) -> int:
"""Release a held list left by a process that stopped mid-rename. For
startup, before anything claims. Returns how many items moved."""
@abstractmethod
def rename(
self, rename: Rename, progress: Callable[[str], None] | None = None
) -> None:
"""Move the names in the work still to run from ``rename.old`` to
``rename.new`` (`Rename.fields`).
Queued, claimed, delayed, parked and held items, and the done markers
that keep a replayed side effect from repeating. Entry ids, order,
deadlines, delivery counts, owners and guards stay as they are. Dead
letters and acknowledged entries are history and stay too. A rerun
after a crash completes the job. Nothing may claim, promote, reclaim,
park or unpark meanwhile.
:param progress: Called with the name of each structure once it is done.
"""
@abstractmethod
def count(self, rename: Rename) -> dict[str, int]:
"""How many entries `rename` would rewrite: ``queue_items`` (queued,
claimed or held), ``delayed``, ``parked`` and ``done_markers``."""
class MemoryWorkQueue(WorkQueue):
"""In-process queue. No durability: a crash loses whatever is in flight."""
@@ -328,6 +498,9 @@ class MemoryWorkQueue(WorkQueue):
self._done: OrderedDict[tuple[str, str], None] = OrderedDict()
# Claimed and not yet acknowledged, which is what Redis's `pending` is.
self._in_flight = 0
# What was added while a rename held the queue (`hold`).
self._held: deque[WorkItem] = deque()
self._holding = False
self._counter = 0
self._seq = 0
# The ids become `FlowRun.id`, a primary key. Counting from zero alone,
@@ -344,6 +517,9 @@ class MemoryWorkQueue(WorkQueue):
with self._wake:
if not item.entry_id:
item.entry_id = self._next_id()
if self._holding:
self._held.append(item)
return
self._items.append(item)
self._wake.notify()
@@ -439,6 +615,7 @@ class MemoryWorkQueue(WorkQueue):
*self._due,
*(entry[2] for entry in self._delayed),
*(item for held in self._parked.values() for item in held),
*self._held,
]
def backlog(self) -> int:
@@ -473,6 +650,66 @@ class MemoryWorkQueue(WorkQueue):
def close(self) -> None:
"""Nothing is held outside the process."""
def hold(self) -> None:
with self._lock:
self._holding = True
def release(self) -> None:
with self._wake:
self._holding = False
self._items.extend(self._held)
self._held.clear()
self._wake.notify()
@property
def held(self) -> bool:
return self._holding
def release_held(self) -> int:
"""Nothing held here outlives the process."""
return 0
def rename(
self, rename: Rename, progress: Callable[[str], None] | None = None
) -> None:
"""One pass under the lock. Timers keep their place in the heap,
which orders them by deadline and arrival only."""
def moved(item: WorkItem) -> WorkItem:
return rename.item(item) or item
with self._lock:
if rename.old in self._parked and rename.new in self._parked:
raise RuntimeError(f"Work is parked under '{rename.new}' already")
for lane in (self._items, self._due, self._held):
items = [moved(item) for item in lane]
lane.clear()
lane.extend(items)
self._delayed = [(at, n, moved(item)) for at, n, item in self._delayed]
self._parked = {
rename.flow(flow): [moved(item) for item in items]
for flow, items in self._parked.items()
}
self._done = OrderedDict(
((entry_id, rename.name(node)), None) for entry_id, node in self._done
)
if progress is not None:
progress("queue")
def count(self, rename: Rename) -> dict[str, int]:
"""What is claimed is gone from here, so it is not counted."""
def changed(items: Iterable[WorkItem]) -> int:
return sum(rename.item(item) is not None for item in items)
with self._lock:
return {
"queue_items": changed([*self._items, *self._due, *self._held]),
"delayed": changed(entry[2] for entry in self._delayed),
"parked": changed(i for held in self._parked.values() for i in held),
"done_markers": sum(rename.name(n) != n for _, n in self._done),
}
class RedisWorkQueue(WorkQueue):
"""Redis Streams queue: the one that survives the process.
@@ -515,7 +752,16 @@ class RedisWorkQueue(WorkQueue):
# finding them with a keyspace scan walked every state and idempotency
# key in the database, twice per health poll.
self._parked_flows_key = f"{namespace}:__parked_flows__"
# What was added while a rename held the queue (`hold`).
self._held_key = f"{namespace}:__held__"
self._move_due = self._redis.register_script(_MOVE_DUE)
self._release = self._redis.register_script(_RELEASE)
# `add` reads the flag and writes under this, and `release` empties the
# held list and clears the flag under it. So nothing added after `hold`
# lands in a stream a rename is rebuilding, and nothing added while
# held overtakes held work.
self._hold_lock = threading.Lock()
self._holding = False
self._ensure_group()
self._drop_stale_consumers()
@@ -561,12 +807,16 @@ class RedisWorkQueue(WorkQueue):
return f"{self._ns}:{DONE}:{entry_id}:{node}"
def add(self, item: WorkItem) -> None:
self._redis.xadd(
self._stream,
cast(Any, item.to_fields()),
maxlen=STREAM_MAXLEN,
approximate=True,
)
with self._hold_lock:
if self._holding:
self._redis.rpush(self._held_key, orjson.dumps(item.to_fields()))
return
self._redis.xadd(
self._stream,
cast(Any, item.to_fields()),
maxlen=STREAM_MAXLEN,
approximate=True,
)
def add_delayed(self, item: WorkItem, not_before: float) -> None:
item.not_before = not_before
@@ -795,6 +1045,10 @@ class RedisWorkQueue(WorkQueue):
list[str], self._redis.lrange(self._parked_key(flow), 0, -1)
)
]
items += [
WorkItem.from_fields(orjson.loads(raw), "")
for raw in cast(list[str], self._redis.lrange(self._held_key, 0, -1))
]
return items
def _group_info(self, stream: str = "") -> dict[str, Any]:
@@ -896,3 +1150,120 @@ class RedisWorkQueue(WorkQueue):
def close(self) -> None:
self._redis.close()
def hold(self) -> None:
with self._hold_lock:
self._holding = True
def release(self) -> None:
with self._hold_lock:
self.release_held()
self._holding = False
@property
def held(self) -> bool:
return self._holding
def release_held(self) -> int:
released = 0
while moved := cast(
int,
self._release(keys=[self._held_key, self._stream], args=[STREAM_MAXLEN]),
):
released += moved
return released
def rename(
self, rename: Rename, progress: Callable[[str], None] | None = None
) -> None:
"""Each structure as section 3 of docs/private/id_refac_rename.md has
it: the streams rebuilt, then timers, lists and done markers, each
rewritten one member, slot or key at a time."""
def done(structure: str) -> None:
if progress is not None:
progress(structure)
for stream, lane in ((self._due_stream, "due"), (self._stream, "backlog")):
rebuild_stream(self._redis, stream, rename)
done(lane)
# One transaction, so each timer moves with its deadline. Promotion is
# held, and a timer added meanwhile is not touched.
pipe = self._redis.pipeline()
for raw, at in cast(
list[tuple[str, float]],
self._redis.zrange(self._delayed_key, 0, -1, withscores=True),
):
fields = rename.fields(orjson.loads(raw))
if fields is not None:
pipe.zrem(self._delayed_key, raw)
pipe.zadd(self._delayed_key, {orjson.dumps(fields): at})
pipe.execute()
done("delayed")
# Slot by slot: nothing pops while held, and `add` only appends, so
# the indices read hold.
for key in [*self._parked_keys(rename), self._held_key]:
pipe = self._redis.pipeline(transaction=False)
for i, raw in enumerate(cast(list[str], self._redis.lrange(key, 0, -1))):
fields = rename.fields(orjson.loads(raw))
if fields is not None:
pipe.lset(key, i, orjson.dumps(fields).decode())
pipe.execute()
old, new = self._parked_key(rename.old), self._parked_key(rename.new)
if self._redis.exists(old):
# Checked first: inside the transaction a refused RENAMENX would
# not stop the index from moving.
if self._redis.exists(new):
raise RuntimeError(f"Work is parked under '{rename.new}' already")
pipe = self._redis.pipeline()
pipe.renamenx(old, new)
pipe.srem(self._parked_flows_key, rename.old)
pipe.sadd(self._parked_flows_key, rename.new)
pipe.execute()
done("lists")
move_keys(self._redis, self._marker_moves(rename))
done("done markers")
def _parked_keys(self, rename: Rename) -> list[str]:
"""Every parked list, the renamed flow's under either name included."""
flows = cast(set[str], self._redis.smembers(self._parked_flows_key))
return list(
dict.fromkeys(
map(self._parked_key, [*sorted(flows), rename.old, rename.new])
)
)
def _marker_moves(self, rename: Rename) -> list[tuple[str, str]]:
moves = []
for key in self._redis.scan_iter(
match=f"{self._ns}:{DONE}:*", count=RENAME_PAGE
):
entry_id, node = done_marker(key)
if rename.name(node) != node:
moves.append((key, self._done_key(entry_id, rename.name(node))))
return moves
def count(self, rename: Rename) -> dict[str, int]:
def changed(raws: Iterable[str]) -> int:
return sum(rename.fields(orjson.loads(raw)) is not None for raw in raws)
def listed(key: str) -> list[str]:
return cast(list[str], self._redis.lrange(key, 0, -1))
live = [
fields
for stream in (self._due_stream, self._stream)
for _, fields in self._live_entries(stream)
]
return {
"queue_items": sum(rename.fields(f) is not None for f in live)
+ changed(listed(self._held_key)),
"delayed": changed(
cast(list[str], self._redis.zrange(self._delayed_key, 0, -1))
),
"parked": sum(changed(listed(key)) for key in self._parked_keys(rename)),
"done_markers": len(self._marker_moves(rename)),
}