Rename migrations for live state and the work queue: state keys move with RENAMENX in batches (type, TTL and history capacity kept), the queue rebuilds its streams with the same ids, group and pending entries, rewrites timers, parked and held items and done markers, and can hold new work in a list released in order; nothing calls them yet
This commit is contained in:
@@ -23,14 +23,15 @@ import threading
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import OrderedDict, deque
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, cast
|
||||
|
||||
import orjson
|
||||
import redis
|
||||
|
||||
from fluksio.flow.state import CONNECT_TIMEOUT_S, SOCKET_TIMEOUT_S
|
||||
from fluksio.flow.namespace import Rename
|
||||
from fluksio.flow.state import CONNECT_TIMEOUT_S, SOCKET_TIMEOUT_S, move_keys
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -54,6 +55,23 @@ STALE_CONSUMER_IDLE_MS = 3_600_000
|
||||
DONE_MARKERS = 10_000
|
||||
# The Redis queue keeps them as ``<namespace>:__done__:<entry id>:<node>``.
|
||||
DONE = "__done__"
|
||||
# How many entries a rename reads, or writes, per round trip.
|
||||
RENAME_PAGE = 500
|
||||
#: Every structure under the Redis queue's namespace. `rename` rewrites the
|
||||
#: names in all of them but the dead letters, which are history. One added
|
||||
#: here holds names too, or has to say why it does not.
|
||||
STRUCTURES = frozenset(
|
||||
{
|
||||
"__queue__",
|
||||
"__due__",
|
||||
"__delayed__",
|
||||
"__parked__",
|
||||
"__parked_flows__",
|
||||
"__held__",
|
||||
DONE,
|
||||
"__dead__",
|
||||
}
|
||||
)
|
||||
|
||||
# A promotion is one Redis operation, including when the caller loses its
|
||||
# connection before hearing the result. Write first: Lua does not roll back a
|
||||
@@ -72,6 +90,24 @@ end
|
||||
return #due
|
||||
"""
|
||||
|
||||
# Work held during a rename goes into the backlog oldest first, behind
|
||||
# everything already there. As in `_MOVE_DUE`, an item leaves the held list
|
||||
# only once its XADD went through. 100 per call, so Redis is never blocked
|
||||
# for long.
|
||||
_RELEASE = """
|
||||
local held = redis.call('LRANGE', KEYS[1], 0, 99)
|
||||
for _, raw in ipairs(held) do
|
||||
local fields = {}
|
||||
for key, value in pairs(cjson.decode(raw)) do
|
||||
fields[#fields + 1] = key
|
||||
fields[#fields + 1] = value
|
||||
end
|
||||
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[1], '*', unpack(fields))
|
||||
redis.call('LPOP', KEYS[1])
|
||||
end
|
||||
return #held
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkItem:
|
||||
@@ -172,6 +208,94 @@ def done_marker(key: str) -> tuple[str, str]:
|
||||
return entry_id, node
|
||||
|
||||
|
||||
def _stream_id(entry_id: str) -> tuple[int, int]:
|
||||
ms, _, seq = entry_id.partition("-")
|
||||
return int(ms), int(seq)
|
||||
|
||||
|
||||
def rebuild_stream(client: redis.Redis, stream: str, rename: Rename) -> None:
|
||||
"""Rewrite the names in the entries of *stream* still to run, keeping
|
||||
every entry id, the group and its consumers, each pending entry's owner,
|
||||
delivery count and idle time, and the stream's id counters.
|
||||
|
||||
An entry cannot be changed in place, and one added again gets a new id,
|
||||
which is what runs, events and done markers are filed under, and loses its
|
||||
delivery count. So the stream is copied into a temporary key with its ids
|
||||
and swapped in with one RENAME, the only commit point: a rerun after a
|
||||
crash on either side of it converges. Acknowledged entries are history and
|
||||
copied as they are. Nothing may write to the stream meanwhile. Needs Redis
|
||||
7.0 (docs/private/id_refac_rename.md, 3.1).
|
||||
"""
|
||||
tmp = f"{stream}:rename"
|
||||
client.delete(tmp)
|
||||
info = cast(dict[str, Any], client.xinfo_stream(stream))
|
||||
groups = cast(list[dict[str, Any]], client.xinfo_groups(stream))
|
||||
group = next(g for g in groups if g["name"] == GROUP)
|
||||
last = str(group["last-delivered-id"])
|
||||
pel: list[dict[str, Any]] = []
|
||||
start = "-"
|
||||
while rows := cast(
|
||||
list[dict[str, Any]],
|
||||
client.xpending_range(stream, GROUP, min=start, max="+", count=RENAME_PAGE),
|
||||
):
|
||||
pel += rows
|
||||
start = f"({rows[-1]['message_id']}"
|
||||
pending = {row["message_id"] for row in pel}
|
||||
|
||||
pipe = client.pipeline(transaction=False)
|
||||
# MKSTREAM, so XSETID finds the key when there is nothing to copy.
|
||||
pipe.xgroup_create(tmp, GROUP, id=last, mkstream=True)
|
||||
for consumer in cast(list[dict[str, Any]], client.xinfo_consumers(stream, GROUP)):
|
||||
pipe.xgroup_createconsumer(tmp, GROUP, consumer["name"])
|
||||
copied: set[str] = set()
|
||||
start = "-"
|
||||
while entries := cast(
|
||||
list[tuple[str, dict[str, str]]],
|
||||
client.xrange(stream, min=start, max="+", count=RENAME_PAGE),
|
||||
):
|
||||
for entry_id, fields in entries:
|
||||
live = entry_id in pending or _stream_id(entry_id) > _stream_id(last)
|
||||
renamed = rename.fields(fields) if live else None
|
||||
pipe.xadd(tmp, cast(Any, renamed or fields), id=entry_id)
|
||||
if entry_id in pending:
|
||||
copied.add(entry_id)
|
||||
pipe.execute()
|
||||
start = f"({entries[-1][0]}"
|
||||
# redis-py has no XSETID of its own, and leaves the generic call untyped.
|
||||
cast(Any, pipe).execute_command(
|
||||
"XSETID",
|
||||
tmp,
|
||||
info["last-generated-id"],
|
||||
"ENTRIESADDED",
|
||||
info["entries-added"],
|
||||
"MAXDELETEDID",
|
||||
info["max-deleted-entry-id"],
|
||||
)
|
||||
# Redis caps ENTRIESREAD at the stream's entries-added counter, which only
|
||||
# XSETID has set. Unknown stays unknown.
|
||||
if group["entries-read"] is not None:
|
||||
pipe.xgroup_setid(tmp, GROUP, last, entries_read=group["entries-read"])
|
||||
# A pending entry whose stream entry was trimmed is not recreated, as
|
||||
# `_reclaim` drops it too. XCLAIM would still create its consumer.
|
||||
for row in pel:
|
||||
if row["message_id"] in copied:
|
||||
pipe.xclaim(
|
||||
tmp,
|
||||
GROUP,
|
||||
row["consumer"],
|
||||
0,
|
||||
[row["message_id"]],
|
||||
idle=row["time_since_delivered"],
|
||||
retrycount=row["times_delivered"],
|
||||
force=True,
|
||||
justid=True,
|
||||
)
|
||||
pipe.execute()
|
||||
# On its own: a pipeline without a transaction runs every command even
|
||||
# after one of them failed.
|
||||
client.rename(tmp, stream)
|
||||
|
||||
|
||||
class WorkQueue(ABC):
|
||||
"""What the execution service pulls from."""
|
||||
|
||||
@@ -309,6 +433,52 @@ class WorkQueue(ABC):
|
||||
def close(self) -> None:
|
||||
"""Release whatever the queue holds."""
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Renaming a flow key
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
def hold(self) -> None:
|
||||
"""Divert `add` into a held list until `release`, so a rename can
|
||||
rewrite the queue while listeners keep journaling work."""
|
||||
|
||||
@abstractmethod
|
||||
def release(self) -> None:
|
||||
"""Move the held work into the backlog, oldest first and behind
|
||||
everything already there, and stop diverting."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def held(self) -> bool:
|
||||
"""Whether `add` is diverted."""
|
||||
|
||||
@abstractmethod
|
||||
def release_held(self) -> int:
|
||||
"""Release a held list left by a process that stopped mid-rename. For
|
||||
startup, before anything claims. Returns how many items moved."""
|
||||
|
||||
@abstractmethod
|
||||
def rename(
|
||||
self, rename: Rename, progress: Callable[[str], None] | None = None
|
||||
) -> None:
|
||||
"""Move the names in the work still to run from ``rename.old`` to
|
||||
``rename.new`` (`Rename.fields`).
|
||||
|
||||
Queued, claimed, delayed, parked and held items, and the done markers
|
||||
that keep a replayed side effect from repeating. Entry ids, order,
|
||||
deadlines, delivery counts, owners and guards stay as they are. Dead
|
||||
letters and acknowledged entries are history and stay too. A rerun
|
||||
after a crash completes the job. Nothing may claim, promote, reclaim,
|
||||
park or unpark meanwhile.
|
||||
|
||||
:param progress: Called with the name of each structure once it is done.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def count(self, rename: Rename) -> dict[str, int]:
|
||||
"""How many entries `rename` would rewrite: ``queue_items`` (queued,
|
||||
claimed or held), ``delayed``, ``parked`` and ``done_markers``."""
|
||||
|
||||
|
||||
class MemoryWorkQueue(WorkQueue):
|
||||
"""In-process queue. No durability: a crash loses whatever is in flight."""
|
||||
@@ -328,6 +498,9 @@ class MemoryWorkQueue(WorkQueue):
|
||||
self._done: OrderedDict[tuple[str, str], None] = OrderedDict()
|
||||
# Claimed and not yet acknowledged, which is what Redis's `pending` is.
|
||||
self._in_flight = 0
|
||||
# What was added while a rename held the queue (`hold`).
|
||||
self._held: deque[WorkItem] = deque()
|
||||
self._holding = False
|
||||
self._counter = 0
|
||||
self._seq = 0
|
||||
# The ids become `FlowRun.id`, a primary key. Counting from zero alone,
|
||||
@@ -344,6 +517,9 @@ class MemoryWorkQueue(WorkQueue):
|
||||
with self._wake:
|
||||
if not item.entry_id:
|
||||
item.entry_id = self._next_id()
|
||||
if self._holding:
|
||||
self._held.append(item)
|
||||
return
|
||||
self._items.append(item)
|
||||
self._wake.notify()
|
||||
|
||||
@@ -439,6 +615,7 @@ class MemoryWorkQueue(WorkQueue):
|
||||
*self._due,
|
||||
*(entry[2] for entry in self._delayed),
|
||||
*(item for held in self._parked.values() for item in held),
|
||||
*self._held,
|
||||
]
|
||||
|
||||
def backlog(self) -> int:
|
||||
@@ -473,6 +650,66 @@ class MemoryWorkQueue(WorkQueue):
|
||||
def close(self) -> None:
|
||||
"""Nothing is held outside the process."""
|
||||
|
||||
def hold(self) -> None:
|
||||
with self._lock:
|
||||
self._holding = True
|
||||
|
||||
def release(self) -> None:
|
||||
with self._wake:
|
||||
self._holding = False
|
||||
self._items.extend(self._held)
|
||||
self._held.clear()
|
||||
self._wake.notify()
|
||||
|
||||
@property
|
||||
def held(self) -> bool:
|
||||
return self._holding
|
||||
|
||||
def release_held(self) -> int:
|
||||
"""Nothing held here outlives the process."""
|
||||
return 0
|
||||
|
||||
def rename(
|
||||
self, rename: Rename, progress: Callable[[str], None] | None = None
|
||||
) -> None:
|
||||
"""One pass under the lock. Timers keep their place in the heap,
|
||||
which orders them by deadline and arrival only."""
|
||||
|
||||
def moved(item: WorkItem) -> WorkItem:
|
||||
return rename.item(item) or item
|
||||
|
||||
with self._lock:
|
||||
if rename.old in self._parked and rename.new in self._parked:
|
||||
raise RuntimeError(f"Work is parked under '{rename.new}' already")
|
||||
for lane in (self._items, self._due, self._held):
|
||||
items = [moved(item) for item in lane]
|
||||
lane.clear()
|
||||
lane.extend(items)
|
||||
self._delayed = [(at, n, moved(item)) for at, n, item in self._delayed]
|
||||
self._parked = {
|
||||
rename.flow(flow): [moved(item) for item in items]
|
||||
for flow, items in self._parked.items()
|
||||
}
|
||||
self._done = OrderedDict(
|
||||
((entry_id, rename.name(node)), None) for entry_id, node in self._done
|
||||
)
|
||||
if progress is not None:
|
||||
progress("queue")
|
||||
|
||||
def count(self, rename: Rename) -> dict[str, int]:
|
||||
"""What is claimed is gone from here, so it is not counted."""
|
||||
|
||||
def changed(items: Iterable[WorkItem]) -> int:
|
||||
return sum(rename.item(item) is not None for item in items)
|
||||
|
||||
with self._lock:
|
||||
return {
|
||||
"queue_items": changed([*self._items, *self._due, *self._held]),
|
||||
"delayed": changed(entry[2] for entry in self._delayed),
|
||||
"parked": changed(i for held in self._parked.values() for i in held),
|
||||
"done_markers": sum(rename.name(n) != n for _, n in self._done),
|
||||
}
|
||||
|
||||
|
||||
class RedisWorkQueue(WorkQueue):
|
||||
"""Redis Streams queue: the one that survives the process.
|
||||
@@ -515,7 +752,16 @@ class RedisWorkQueue(WorkQueue):
|
||||
# finding them with a keyspace scan walked every state and idempotency
|
||||
# key in the database, twice per health poll.
|
||||
self._parked_flows_key = f"{namespace}:__parked_flows__"
|
||||
# What was added while a rename held the queue (`hold`).
|
||||
self._held_key = f"{namespace}:__held__"
|
||||
self._move_due = self._redis.register_script(_MOVE_DUE)
|
||||
self._release = self._redis.register_script(_RELEASE)
|
||||
# `add` reads the flag and writes under this, and `release` empties the
|
||||
# held list and clears the flag under it. So nothing added after `hold`
|
||||
# lands in a stream a rename is rebuilding, and nothing added while
|
||||
# held overtakes held work.
|
||||
self._hold_lock = threading.Lock()
|
||||
self._holding = False
|
||||
self._ensure_group()
|
||||
self._drop_stale_consumers()
|
||||
|
||||
@@ -561,12 +807,16 @@ class RedisWorkQueue(WorkQueue):
|
||||
return f"{self._ns}:{DONE}:{entry_id}:{node}"
|
||||
|
||||
def add(self, item: WorkItem) -> None:
|
||||
self._redis.xadd(
|
||||
self._stream,
|
||||
cast(Any, item.to_fields()),
|
||||
maxlen=STREAM_MAXLEN,
|
||||
approximate=True,
|
||||
)
|
||||
with self._hold_lock:
|
||||
if self._holding:
|
||||
self._redis.rpush(self._held_key, orjson.dumps(item.to_fields()))
|
||||
return
|
||||
self._redis.xadd(
|
||||
self._stream,
|
||||
cast(Any, item.to_fields()),
|
||||
maxlen=STREAM_MAXLEN,
|
||||
approximate=True,
|
||||
)
|
||||
|
||||
def add_delayed(self, item: WorkItem, not_before: float) -> None:
|
||||
item.not_before = not_before
|
||||
@@ -795,6 +1045,10 @@ class RedisWorkQueue(WorkQueue):
|
||||
list[str], self._redis.lrange(self._parked_key(flow), 0, -1)
|
||||
)
|
||||
]
|
||||
items += [
|
||||
WorkItem.from_fields(orjson.loads(raw), "")
|
||||
for raw in cast(list[str], self._redis.lrange(self._held_key, 0, -1))
|
||||
]
|
||||
return items
|
||||
|
||||
def _group_info(self, stream: str = "") -> dict[str, Any]:
|
||||
@@ -896,3 +1150,120 @@ class RedisWorkQueue(WorkQueue):
|
||||
|
||||
def close(self) -> None:
|
||||
self._redis.close()
|
||||
|
||||
def hold(self) -> None:
|
||||
with self._hold_lock:
|
||||
self._holding = True
|
||||
|
||||
def release(self) -> None:
|
||||
with self._hold_lock:
|
||||
self.release_held()
|
||||
self._holding = False
|
||||
|
||||
@property
|
||||
def held(self) -> bool:
|
||||
return self._holding
|
||||
|
||||
def release_held(self) -> int:
|
||||
released = 0
|
||||
while moved := cast(
|
||||
int,
|
||||
self._release(keys=[self._held_key, self._stream], args=[STREAM_MAXLEN]),
|
||||
):
|
||||
released += moved
|
||||
return released
|
||||
|
||||
def rename(
|
||||
self, rename: Rename, progress: Callable[[str], None] | None = None
|
||||
) -> None:
|
||||
"""Each structure as section 3 of docs/private/id_refac_rename.md has
|
||||
it: the streams rebuilt, then timers, lists and done markers, each
|
||||
rewritten one member, slot or key at a time."""
|
||||
|
||||
def done(structure: str) -> None:
|
||||
if progress is not None:
|
||||
progress(structure)
|
||||
|
||||
for stream, lane in ((self._due_stream, "due"), (self._stream, "backlog")):
|
||||
rebuild_stream(self._redis, stream, rename)
|
||||
done(lane)
|
||||
|
||||
# One transaction, so each timer moves with its deadline. Promotion is
|
||||
# held, and a timer added meanwhile is not touched.
|
||||
pipe = self._redis.pipeline()
|
||||
for raw, at in cast(
|
||||
list[tuple[str, float]],
|
||||
self._redis.zrange(self._delayed_key, 0, -1, withscores=True),
|
||||
):
|
||||
fields = rename.fields(orjson.loads(raw))
|
||||
if fields is not None:
|
||||
pipe.zrem(self._delayed_key, raw)
|
||||
pipe.zadd(self._delayed_key, {orjson.dumps(fields): at})
|
||||
pipe.execute()
|
||||
done("delayed")
|
||||
|
||||
# Slot by slot: nothing pops while held, and `add` only appends, so
|
||||
# the indices read hold.
|
||||
for key in [*self._parked_keys(rename), self._held_key]:
|
||||
pipe = self._redis.pipeline(transaction=False)
|
||||
for i, raw in enumerate(cast(list[str], self._redis.lrange(key, 0, -1))):
|
||||
fields = rename.fields(orjson.loads(raw))
|
||||
if fields is not None:
|
||||
pipe.lset(key, i, orjson.dumps(fields).decode())
|
||||
pipe.execute()
|
||||
old, new = self._parked_key(rename.old), self._parked_key(rename.new)
|
||||
if self._redis.exists(old):
|
||||
# Checked first: inside the transaction a refused RENAMENX would
|
||||
# not stop the index from moving.
|
||||
if self._redis.exists(new):
|
||||
raise RuntimeError(f"Work is parked under '{rename.new}' already")
|
||||
pipe = self._redis.pipeline()
|
||||
pipe.renamenx(old, new)
|
||||
pipe.srem(self._parked_flows_key, rename.old)
|
||||
pipe.sadd(self._parked_flows_key, rename.new)
|
||||
pipe.execute()
|
||||
done("lists")
|
||||
|
||||
move_keys(self._redis, self._marker_moves(rename))
|
||||
done("done markers")
|
||||
|
||||
def _parked_keys(self, rename: Rename) -> list[str]:
|
||||
"""Every parked list, the renamed flow's under either name included."""
|
||||
flows = cast(set[str], self._redis.smembers(self._parked_flows_key))
|
||||
return list(
|
||||
dict.fromkeys(
|
||||
map(self._parked_key, [*sorted(flows), rename.old, rename.new])
|
||||
)
|
||||
)
|
||||
|
||||
def _marker_moves(self, rename: Rename) -> list[tuple[str, str]]:
|
||||
moves = []
|
||||
for key in self._redis.scan_iter(
|
||||
match=f"{self._ns}:{DONE}:*", count=RENAME_PAGE
|
||||
):
|
||||
entry_id, node = done_marker(key)
|
||||
if rename.name(node) != node:
|
||||
moves.append((key, self._done_key(entry_id, rename.name(node))))
|
||||
return moves
|
||||
|
||||
def count(self, rename: Rename) -> dict[str, int]:
|
||||
def changed(raws: Iterable[str]) -> int:
|
||||
return sum(rename.fields(orjson.loads(raw)) is not None for raw in raws)
|
||||
|
||||
def listed(key: str) -> list[str]:
|
||||
return cast(list[str], self._redis.lrange(key, 0, -1))
|
||||
|
||||
live = [
|
||||
fields
|
||||
for stream in (self._due_stream, self._stream)
|
||||
for _, fields in self._live_entries(stream)
|
||||
]
|
||||
return {
|
||||
"queue_items": sum(rename.fields(f) is not None for f in live)
|
||||
+ changed(listed(self._held_key)),
|
||||
"delayed": changed(
|
||||
cast(list[str], self._redis.zrange(self._delayed_key, 0, -1))
|
||||
),
|
||||
"parked": sum(changed(listed(key)) for key in self._parked_keys(rename)),
|
||||
"done_markers": len(self._marker_moves(rename)),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user