Rename migrations for live state and the work queue: state keys move with RENAMENX in batches (type, TTL and history capacity kept), the queue rebuilds its streams with the same ids, group and pending entries, rewrites timers, parked and held items and done markers, and can hold new work in a list released in order; nothing calls them yet

This commit is contained in:
2026-11-02 19:50:54 +01:00
parent ebcc480a8c
commit 46b57de4ec
5 changed files with 981 additions and 83 deletions
+113 -2
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from collections import deque
from collections.abc import Iterable, Iterator
from collections.abc import Callable, Iterable, Iterator
from contextlib import contextmanager
from threading import RLock
from typing import Any, cast
@@ -17,7 +17,7 @@ from typing import Any, cast
import orjson
import redis
from fluksio.flow.namespace import HISTORY, NAMES
from fluksio.flow.namespace import HISTORY, NAMES, Rename
#: A node may return a dict keyed by something other than a string, and the
#: stdlib encoder this replaced turned those keys into strings rather than
@@ -36,6 +36,9 @@ SOCKET_TIMEOUT_S = 5.0
CONNECT_TIMEOUT_S = 2.0
HISTORY_LIMIT = 120
#: How many keys a rename moves per round trip, and between two calls of its
#: ``progress``, which is where the rename journal can stop.
RENAME_BATCH = 500
def as_number(value: Any) -> float | None:
@@ -50,6 +53,30 @@ def as_number(value: Any) -> float | None:
return float(value)
def move_keys(client: redis.Redis, moves: list[tuple[str, str]]) -> int:
"""``RENAMENX`` each ``(key, target)`` in one round trip, which keeps the
Redis type and the remaining TTL. Returns how many moved.
A key gone since it was listed has expired and is skipped. A target that
exists is refused rather than overwritten: a rename checks that nothing
lives under the new name before it starts, so one there now is a fault.
"""
pipe = client.pipeline(transaction=False)
for key, target in moves:
pipe.renamenx(key, target)
replies = pipe.execute(raise_on_error=False)
moved = 0
for (key, target), reply in zip(moves, replies, strict=True):
if isinstance(reply, redis.ResponseError) and "no such key" in str(reply):
continue
if isinstance(reply, Exception):
raise reply
if not reply:
raise RuntimeError(f"Will not move '{key}' onto '{target}', which exists")
moved += 1
return moved
class StateBackend(ABC):
"""
Abstract base class for state storage backends.
@@ -289,6 +316,32 @@ class StateBackend(ABC):
"""
...
# -------------------------------------------------------------------------
# Renaming a flow key
# -------------------------------------------------------------------------
@abstractmethod
def rename_keys(
self, rename: Rename, progress: Callable[[str], None] | None = None
) -> int:
"""Move every key that names something of ``rename.old`` to the new
name (`Rename.state_key`), with its history.
Keys move, values do not: a message's payload and a node's private
state are its own, whatever names they mention. A key already under
the new name stays where it is, so a rerun after a crash finishes the
job. A target that exists is refused rather than overwritten.
:param progress: Called after each batch of keys.
:returns: How many keys moved.
:raises ValueError: For a bookkeeping key of an unregistered family.
"""
...
def count(self, rename: Rename) -> int:
"""How many keys `rename_keys` would move."""
return sum(rename.state_key(key) is not None for key in self.keys())
def __contains__(self, key: str) -> bool:
return self.exists(key)
@@ -459,6 +512,39 @@ class MemoryState(StateBackend):
with self._lock:
return list(self._history.get(key, ()))
def _moves(self, rename: Rename) -> tuple[dict[str, str], dict[str, str]]:
"""Where the keys of values and of histories go. Histories are kept
by bare message name here."""
data = {k: t for k in self._data if (t := rename.state_key(k)) is not None}
history = {k: rename.name(k) for k in self._history if rename.name(k) != k}
return data, history
def rename_keys(
self, rename: Rename, progress: Callable[[str], None] | None = None
) -> int:
"""One pass under the lock. A history moves as the deque it is, so it
keeps its capacity."""
with self._lock:
data, history = self._moves(rename)
for store, moves in ((self._data, data), (self._history, history)):
clash = next((t for t in moves.values() if t in store), None)
if clash is not None:
raise RuntimeError(
f"Will not move a key onto '{clash}', which exists"
)
for key, target in data.items():
self._data[target] = self._data.pop(key)
for key, target in history.items():
self._history[target] = self._history.pop(key)
if progress is not None:
progress("keys")
return len(data) + len(history)
def count(self, rename: Rename) -> int:
with self._lock:
data, history = self._moves(rename)
return len(data) + len(history)
class RedisState(StateBackend):
"""
@@ -843,3 +929,28 @@ class RedisState(StateBackend):
ts, value = orjson.loads(entry)
points.append((float(ts), float(value)))
return points
def rename_keys(
self, rename: Rename, progress: Callable[[str], None] | None = None
) -> int:
"""``RENAMENX`` in batches, then the names set in one transaction.
The keys are listed before any moves, since a SCAN running over keys
that are being renamed can return one twice.
"""
moves = [(k, t) for k in self.keys() if (t := rename.state_key(k)) is not None]
moved = 0
for start in range(0, len(moves), RENAME_BATCH):
batch = moves[start : start + RENAME_BATCH]
moved += move_keys(
self._client, [(self._key(k), self._key(t)) for k, t in batch]
)
if progress is not None:
progress("keys")
names = [name for name in self.message_names() if rename.name(name) != name]
if names:
pipe = self._client.pipeline()
pipe.srem(self._names_key(), *names)
pipe.sadd(self._names_key(), *map(rename.name, names))
pipe.execute()
return moved