Merge branch 'rename-s2'

This commit is contained in:
2026-11-02 19:58:45 +01:00
8 changed files with 1364 additions and 87 deletions
+187
View File
@@ -0,0 +1,187 @@
"""The names the engine keeps live state and queued work under, and how a
rename of a flow key maps them.
A flow's messages and nodes are named ``<key>.<name>``, and the engine's own
bookkeeping on them sits in state beside the values, under a reserved prefix
and the names it is about: ``__version__:heat.temp``,
``__last_seen__:heat.valve:hall.temp``. Renaming a flow key moves all of it to
the new name, so every family is registered here, beside the builders that
take their prefix from it, and a key of a family that is not is refused rather
than left behind.
Nothing here does I/O. It maps names; moving the keys and items is the state
and queue backends' work.
"""
from __future__ import annotations
from dataclasses import dataclass, replace
from typing import TYPE_CHECKING, Any
import orjson
from fluksio.flow.messages import flow_of
if TYPE_CHECKING:
from fluksio.flow.queue import WorkItem
#: The message names in state, as a set (`RedisState` only).
NAMES = "__names__"
#: A message's recorded points (`RedisState`; memory keeps them apart).
HISTORY = "__history__"
#: How often a message was published.
VERSION = "__version__"
#: When a message was last published.
TIMESTAMP = "__ts__"
#: The version of a message a node last ran on.
LAST_SEEN = "__last_seen__"
#: When a rate-limited input last woke a node.
DELIVERED = "__in_ts__"
#: The value a rate-limited output kept back.
HELD = "__held__"
#: When a node's rate-limit window is due to be let through.
FLUSH = "__flush__"
#: What an engine-shipped node type remembers (`Node.remember`).
NODE_STATE = "__node__"
#: A qualified message or node name, ``<flow key>.<name>``.
NAME = "name"
#: A node's own key, user data that may contain colons.
PRIVATE = "private"
#: A flow key.
FLOW = "flow"
#: A map keyed by qualified message names.
OUTPUTS = "outputs"
#: Every bookkeeping family in live state, by prefix, with the parts that
#: follow it. A key is ``<prefix>:<part>:<part>``. Names never contain a colon,
#: so the key splits at its first ones and the last part takes the rest, which
#: is how a private key keeps its own.
FAMILIES: dict[str, tuple[str, ...]] = {
NAMES: (),
HISTORY: (NAME,),
VERSION: (NAME,),
TIMESTAMP: (NAME,),
LAST_SEEN: (NAME, NAME),
DELIVERED: (NAME, NAME),
HELD: (NAME,),
FLUSH: (NAME,),
NODE_STATE: (NAME, PRIVATE),
}
#: What each `WorkItem` field holds that a rename changes, or None. A guard is
#: the node's own key, and the state it guards moves with the node.
ITEM_FIELDS: dict[str, str | None] = {
"kind": None,
"node": NAME,
"flow": FLOW,
"outputs": OUTPUTS,
"cause": None,
"not_before": None,
"guard_key": None,
"guard_value": None,
"entry_id": None,
"deliveries": None,
"enqueued_at": None,
"run_id": None,
"parent": None,
}
def _parts(key: str) -> tuple[str, list[tuple[str, str]]]:
"""A state key's family prefix and its parts, each with what it holds.
A key without a prefix is a message value, and its family is ``""``.
"""
if not key.startswith("__"):
return "", [(key, NAME)]
family, sep, rest = key.partition(":")
shape = FAMILIES.get(family)
parts = rest.split(":", len(shape) - 1) if shape is not None and sep else []
if shape is None or len(parts) != len(shape):
raise ValueError(f"State key '{key}' is in no registered family")
return family, list(zip(parts, shape, strict=True))
def touches(flow: str, key: str) -> bool:
"""Whether the state key *key* names anything of the flow key *flow*."""
_, parts = _parts(key)
return any(kind == NAME and flow_of(part) == flow for part, kind in parts)
@dataclass(frozen=True)
class Rename:
"""The flow key *old* becoming *new*, applied to the names that embed it."""
old: str
new: str
def name(self, qualified: str) -> str:
"""A qualified message or node name in the new namespace.
Only the part before the first dot is the flow, so renaming ``heat``
leaves ``heating.temp`` alone. A bare key is its own namespace.
"""
if flow_of(qualified) != self.old:
return qualified
return self.new + qualified[len(self.old) :]
def flow(self, key: str) -> str:
return self.new if key == self.old else key
def state_key(self, key: str) -> str | None:
"""Where the state key *key* moves to, or None if it stays.
:raises ValueError: For a bookkeeping key of an unregistered family.
"""
family, parts = _parts(key)
mapped = [self.name(part) if kind == NAME else part for part, kind in parts]
renamed = ":".join([family, *mapped]) if family else mapped[0]
return renamed if renamed != key else None
def item(self, item: WorkItem) -> WorkItem | None:
"""*item* with its names in the new namespace, or None if it has none.
A ``run`` item names its flow by id, which a rename leaves as it is.
"""
if item.kind == "run":
return None
changes = {}
for field, holds in ITEM_FIELDS.items():
if holds is not None:
value = getattr(item, field)
mapped = self._mapped(holds, value)
if mapped != value:
changes[field] = mapped
return replace(item, **changes) if changes else None
def fields(self, fields: dict[str, str]) -> dict[str, str] | None:
"""A serialized item (`WorkItem.to_fields`) in the new namespace, or
None if it names nothing renamed. Fields a rename does not touch keep
their exact text, so an unchanged item serializes as it did."""
if fields.get("kind") == "run":
return None
renamed = dict(fields)
for field, holds in ITEM_FIELDS.items():
if holds is None or field not in fields:
continue
if holds == OUTPUTS:
outputs = orjson.loads(fields[field] or "{}")
mapped = self._mapped(holds, outputs)
if mapped != outputs:
renamed[field] = orjson.dumps(mapped).decode()
else:
renamed[field] = self._mapped(holds, fields[field])
return renamed if renamed != fields else None
def dedup_key(self, key: str) -> str:
"""An alert's ``<event type>:<node or flow>`` (`alerts.dedup_key`)."""
event, sep, about = key.partition(":")
return f"{event}{sep}{self.name(about)}"
def _mapped(self, holds: str, value: Any) -> Any:
if holds == NAME:
return self.name(value)
if holds == FLOW:
return self.flow(value)
return {self.name(name): one for name, one in value.items()}
+2 -1
View File
@@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any
from fluksio.flow import logs
from fluksio.flow.messages import MessageSpec, qualify
from fluksio.flow.namespace import NODE_STATE
from fluksio.flow.supervision import Supervisor
if TYPE_CHECKING:
@@ -283,7 +284,7 @@ class Node:
# -------------------------------------------------------------------------
def _state_key(self, key: str) -> str:
return f"__node__:{self.id}:{key}"
return f"{NODE_STATE}:{self.id}:{key}"
def remember(self, key: str, value: Any) -> None:
"""Keep a value for the next run of this node."""
+14 -6
View File
@@ -34,6 +34,14 @@ from fluksio.flow import logs
from fluksio.flow.artifacts import is_reference
from fluksio.flow.events import EventBus
from fluksio.flow.messages import flow_of, requalify
from fluksio.flow.namespace import (
DELIVERED,
FLUSH,
HELD,
LAST_SEEN,
TIMESTAMP,
VERSION,
)
from fluksio.flow.nodes import Node
from fluksio.flow.queue import WorkQueue
from fluksio.flow.state import MemoryState, StateBackend
@@ -633,25 +641,25 @@ class Pipeline:
# -------------------------------------------------------------------------
def _version_key(self, msg_name: str) -> str:
return f"__version__:{msg_name}"
return f"{VERSION}:{msg_name}"
def _last_seen_key(self, node_name: str, msg_name: str) -> str:
return f"__last_seen__:{node_name}:{msg_name}"
return f"{LAST_SEEN}:{node_name}:{msg_name}"
def _timestamp_key(self, msg_name: str) -> str:
return f"__ts__:{msg_name}"
return f"{TIMESTAMP}:{msg_name}"
def _delivered_key(self, node_name: str, msg_name: str) -> str:
"""When a rate-limited input last woke this node."""
return f"__in_ts__:{node_name}:{msg_name}"
return f"{DELIVERED}:{node_name}:{msg_name}"
def _held_key(self, msg_name: str) -> str:
"""The value a rate-limited output kept back, waiting for its window."""
return f"__held__:{msg_name}"
return f"{HELD}:{msg_name}"
def _flush_key(self, node_name: str) -> str:
"""When a rate-limit window of this node is due to be let through."""
return f"__flush__:{node_name}"
return f"{FLUSH}:{node_name}"
def _window_split(
self, node: Node, result: dict[str, Any], now: float
+390 -9
View File
@@ -23,14 +23,15 @@ import threading
import time
from abc import ABC, abstractmethod
from collections import OrderedDict, deque
from collections.abc import Callable
from collections.abc import Callable, Iterable
from dataclasses import dataclass, field
from typing import Any, cast
import orjson
import redis
from fluksio.flow.state import CONNECT_TIMEOUT_S, SOCKET_TIMEOUT_S
from fluksio.flow.namespace import Rename
from fluksio.flow.state import CONNECT_TIMEOUT_S, SOCKET_TIMEOUT_S, move_keys
logger = logging.getLogger(__name__)
@@ -52,6 +53,25 @@ STALE_CONSUMER_IDLE_MS = 3_600_000
# The Redis one expires each after an hour; this is the same idea sized by
# count, since redelivery happens seconds after the claim and never later.
DONE_MARKERS = 10_000
# The Redis queue keeps them as ``<namespace>:__done__:<entry id>:<node>``.
DONE = "__done__"
# How many entries a rename reads, or writes, per round trip.
RENAME_PAGE = 500
#: Every structure under the Redis queue's namespace. `rename` rewrites the
#: names in all of them but the dead letters, which are history. One added
#: here holds names too, or has to say why it does not.
STRUCTURES = frozenset(
{
"__queue__",
"__due__",
"__delayed__",
"__parked__",
"__parked_flows__",
"__held__",
DONE,
"__dead__",
}
)
# A promotion is one Redis operation, including when the caller loses its
# connection before hearing the result. Write first: Lua does not roll back a
@@ -70,6 +90,24 @@ end
return #due
"""
# Work held during a rename goes into the backlog oldest first, behind
# everything already there. As in `_MOVE_DUE`, an item leaves the held list
# only once its XADD went through. 100 per call, so Redis is never blocked
# for long.
_RELEASE = """
local held = redis.call('LRANGE', KEYS[1], 0, 99)
for _, raw in ipairs(held) do
local fields = {}
for key, value in pairs(cjson.decode(raw)) do
fields[#fields + 1] = key
fields[#fields + 1] = value
end
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[1], '*', unpack(fields))
redis.call('LPOP', KEYS[1])
end
return #held
"""
@dataclass
class WorkItem:
@@ -162,6 +200,102 @@ def owned_by(item: WorkItem, flow: str) -> bool:
)
def done_marker(key: str) -> tuple[str, str]:
"""The entry id and node a done marker is for: `RedisWorkQueue._done_key`
read back. Split at the last colon, since an entry id from the due lane
has one of its own and a node id never does."""
entry_id, node = key.split(f":{DONE}:", 1)[1].rsplit(":", 1)
return entry_id, node
def _stream_id(entry_id: str) -> tuple[int, int]:
ms, _, seq = entry_id.partition("-")
return int(ms), int(seq)
def rebuild_stream(client: redis.Redis, stream: str, rename: Rename) -> None:
"""Rewrite the names in the entries of *stream* still to run, keeping
every entry id, the group and its consumers, each pending entry's owner,
delivery count and idle time, and the stream's id counters.
An entry cannot be changed in place, and one added again gets a new id,
which is what runs, events and done markers are filed under, and loses its
delivery count. So the stream is copied into a temporary key with its ids
and swapped in with one RENAME, the only commit point: a rerun after a
crash on either side of it converges. Acknowledged entries are history and
copied as they are. Nothing may write to the stream meanwhile. Needs Redis
7.0 (docs/private/id_refac_rename.md, 3.1).
"""
tmp = f"{stream}:rename"
client.delete(tmp)
info = cast(dict[str, Any], client.xinfo_stream(stream))
groups = cast(list[dict[str, Any]], client.xinfo_groups(stream))
group = next(g for g in groups if g["name"] == GROUP)
last = str(group["last-delivered-id"])
pel: list[dict[str, Any]] = []
start = "-"
while rows := cast(
list[dict[str, Any]],
client.xpending_range(stream, GROUP, min=start, max="+", count=RENAME_PAGE),
):
pel += rows
start = f"({rows[-1]['message_id']}"
pending = {row["message_id"] for row in pel}
pipe = client.pipeline(transaction=False)
# MKSTREAM, so XSETID finds the key when there is nothing to copy.
pipe.xgroup_create(tmp, GROUP, id=last, mkstream=True)
for consumer in cast(list[dict[str, Any]], client.xinfo_consumers(stream, GROUP)):
pipe.xgroup_createconsumer(tmp, GROUP, consumer["name"])
copied: set[str] = set()
start = "-"
while entries := cast(
list[tuple[str, dict[str, str]]],
client.xrange(stream, min=start, max="+", count=RENAME_PAGE),
):
for entry_id, fields in entries:
live = entry_id in pending or _stream_id(entry_id) > _stream_id(last)
renamed = rename.fields(fields) if live else None
pipe.xadd(tmp, cast(Any, renamed or fields), id=entry_id)
if entry_id in pending:
copied.add(entry_id)
pipe.execute()
start = f"({entries[-1][0]}"
# redis-py has no XSETID of its own, and leaves the generic call untyped.
cast(Any, pipe).execute_command(
"XSETID",
tmp,
info["last-generated-id"],
"ENTRIESADDED",
info["entries-added"],
"MAXDELETEDID",
info["max-deleted-entry-id"],
)
# Redis caps ENTRIESREAD at the stream's entries-added counter, which only
# XSETID has set. Unknown stays unknown.
if group["entries-read"] is not None:
pipe.xgroup_setid(tmp, GROUP, last, entries_read=group["entries-read"])
# A pending entry whose stream entry was trimmed is not recreated, as
# `_reclaim` drops it too. XCLAIM would still create its consumer.
for row in pel:
if row["message_id"] in copied:
pipe.xclaim(
tmp,
GROUP,
row["consumer"],
0,
[row["message_id"]],
idle=row["time_since_delivered"],
retrycount=row["times_delivered"],
force=True,
justid=True,
)
pipe.execute()
# On its own: a pipeline without a transaction runs every command even
# after one of them failed.
client.rename(tmp, stream)
class WorkQueue(ABC):
"""What the execution service pulls from."""
@@ -299,6 +433,52 @@ class WorkQueue(ABC):
def close(self) -> None:
"""Release whatever the queue holds."""
# -------------------------------------------------------------------------
# Renaming a flow key
# -------------------------------------------------------------------------
@abstractmethod
def hold(self) -> None:
"""Divert `add` into a held list until `release`, so a rename can
rewrite the queue while listeners keep journaling work."""
@abstractmethod
def release(self) -> None:
"""Move the held work into the backlog, oldest first and behind
everything already there, and stop diverting."""
@property
@abstractmethod
def held(self) -> bool:
"""Whether `add` is diverted."""
@abstractmethod
def release_held(self) -> int:
"""Release a held list left by a process that stopped mid-rename. For
startup, before anything claims. Returns how many items moved."""
@abstractmethod
def rename(
self, rename: Rename, progress: Callable[[str], None] | None = None
) -> None:
"""Move the names in the work still to run from ``rename.old`` to
``rename.new`` (`Rename.fields`).
Queued, claimed, delayed, parked and held items, and the done markers
that keep a replayed side effect from repeating. Entry ids, order,
deadlines, delivery counts, owners and guards stay as they are. Dead
letters and acknowledged entries are history and stay too. A rerun
after a crash completes the job. Nothing may claim, promote, reclaim,
park or unpark meanwhile.
:param progress: Called with the name of each structure once it is done.
"""
@abstractmethod
def count(self, rename: Rename) -> dict[str, int]:
"""How many entries `rename` would rewrite: ``queue_items`` (queued,
claimed or held), ``delayed``, ``parked`` and ``done_markers``."""
class MemoryWorkQueue(WorkQueue):
"""In-process queue. No durability: a crash loses whatever is in flight."""
@@ -318,6 +498,9 @@ class MemoryWorkQueue(WorkQueue):
self._done: OrderedDict[tuple[str, str], None] = OrderedDict()
# Claimed and not yet acknowledged, which is what Redis's `pending` is.
self._in_flight = 0
# What was added while a rename held the queue (`hold`).
self._held: deque[WorkItem] = deque()
self._holding = False
self._counter = 0
self._seq = 0
# The ids become `FlowRun.id`, a primary key. Counting from zero alone,
@@ -334,6 +517,9 @@ class MemoryWorkQueue(WorkQueue):
with self._wake:
if not item.entry_id:
item.entry_id = self._next_id()
if self._holding:
self._held.append(item)
return
self._items.append(item)
self._wake.notify()
@@ -429,6 +615,7 @@ class MemoryWorkQueue(WorkQueue):
*self._due,
*(entry[2] for entry in self._delayed),
*(item for held in self._parked.values() for item in held),
*self._held,
]
def backlog(self) -> int:
@@ -463,6 +650,66 @@ class MemoryWorkQueue(WorkQueue):
def close(self) -> None:
"""Nothing is held outside the process."""
def hold(self) -> None:
with self._lock:
self._holding = True
def release(self) -> None:
with self._wake:
self._holding = False
self._items.extend(self._held)
self._held.clear()
self._wake.notify()
@property
def held(self) -> bool:
return self._holding
def release_held(self) -> int:
"""Nothing held here outlives the process."""
return 0
def rename(
self, rename: Rename, progress: Callable[[str], None] | None = None
) -> None:
"""One pass under the lock. Timers keep their place in the heap,
which orders them by deadline and arrival only."""
def moved(item: WorkItem) -> WorkItem:
return rename.item(item) or item
with self._lock:
if rename.old in self._parked and rename.new in self._parked:
raise RuntimeError(f"Work is parked under '{rename.new}' already")
for lane in (self._items, self._due, self._held):
items = [moved(item) for item in lane]
lane.clear()
lane.extend(items)
self._delayed = [(at, n, moved(item)) for at, n, item in self._delayed]
self._parked = {
rename.flow(flow): [moved(item) for item in items]
for flow, items in self._parked.items()
}
self._done = OrderedDict(
((entry_id, rename.name(node)), None) for entry_id, node in self._done
)
if progress is not None:
progress("queue")
def count(self, rename: Rename) -> dict[str, int]:
"""What is claimed is gone from here, so it is not counted."""
def changed(items: Iterable[WorkItem]) -> int:
return sum(rename.item(item) is not None for item in items)
with self._lock:
return {
"queue_items": changed([*self._items, *self._due, *self._held]),
"delayed": changed(entry[2] for entry in self._delayed),
"parked": changed(i for held in self._parked.values() for i in held),
"done_markers": sum(rename.name(n) != n for _, n in self._done),
}
class RedisWorkQueue(WorkQueue):
"""Redis Streams queue: the one that survives the process.
@@ -505,7 +752,16 @@ class RedisWorkQueue(WorkQueue):
# finding them with a keyspace scan walked every state and idempotency
# key in the database, twice per health poll.
self._parked_flows_key = f"{namespace}:__parked_flows__"
# What was added while a rename held the queue (`hold`).
self._held_key = f"{namespace}:__held__"
self._move_due = self._redis.register_script(_MOVE_DUE)
self._release = self._redis.register_script(_RELEASE)
# `add` reads the flag and writes under this, and `release` empties the
# held list and clears the flag under it. So nothing added after `hold`
# lands in a stream a rename is rebuilding, and nothing added while
# held overtakes held work.
self._hold_lock = threading.Lock()
self._holding = False
self._ensure_group()
self._drop_stale_consumers()
@@ -548,15 +804,19 @@ class RedisWorkQueue(WorkQueue):
return f"{self._ns}:__parked__:{flow}"
def _done_key(self, entry_id: str, node: str) -> str:
return f"{self._ns}:__done__:{entry_id}:{node}"
return f"{self._ns}:{DONE}:{entry_id}:{node}"
def add(self, item: WorkItem) -> None:
self._redis.xadd(
self._stream,
cast(Any, item.to_fields()),
maxlen=STREAM_MAXLEN,
approximate=True,
)
with self._hold_lock:
if self._holding:
self._redis.rpush(self._held_key, orjson.dumps(item.to_fields()))
return
self._redis.xadd(
self._stream,
cast(Any, item.to_fields()),
maxlen=STREAM_MAXLEN,
approximate=True,
)
def add_delayed(self, item: WorkItem, not_before: float) -> None:
item.not_before = not_before
@@ -785,6 +1045,10 @@ class RedisWorkQueue(WorkQueue):
list[str], self._redis.lrange(self._parked_key(flow), 0, -1)
)
]
items += [
WorkItem.from_fields(orjson.loads(raw), "")
for raw in cast(list[str], self._redis.lrange(self._held_key, 0, -1))
]
return items
def _group_info(self, stream: str = "") -> dict[str, Any]:
@@ -886,3 +1150,120 @@ class RedisWorkQueue(WorkQueue):
def close(self) -> None:
self._redis.close()
def hold(self) -> None:
with self._hold_lock:
self._holding = True
def release(self) -> None:
with self._hold_lock:
self.release_held()
self._holding = False
@property
def held(self) -> bool:
return self._holding
def release_held(self) -> int:
released = 0
while moved := cast(
int,
self._release(keys=[self._held_key, self._stream], args=[STREAM_MAXLEN]),
):
released += moved
return released
def rename(
self, rename: Rename, progress: Callable[[str], None] | None = None
) -> None:
"""Each structure as section 3 of docs/private/id_refac_rename.md has
it: the streams rebuilt, then timers, lists and done markers, each
rewritten one member, slot or key at a time."""
def done(structure: str) -> None:
if progress is not None:
progress(structure)
for stream, lane in ((self._due_stream, "due"), (self._stream, "backlog")):
rebuild_stream(self._redis, stream, rename)
done(lane)
# One transaction, so each timer moves with its deadline. Promotion is
# held, and a timer added meanwhile is not touched.
pipe = self._redis.pipeline()
for raw, at in cast(
list[tuple[str, float]],
self._redis.zrange(self._delayed_key, 0, -1, withscores=True),
):
fields = rename.fields(orjson.loads(raw))
if fields is not None:
pipe.zrem(self._delayed_key, raw)
pipe.zadd(self._delayed_key, {orjson.dumps(fields): at})
pipe.execute()
done("delayed")
# Slot by slot: nothing pops while held, and `add` only appends, so
# the indices read hold.
for key in [*self._parked_keys(rename), self._held_key]:
pipe = self._redis.pipeline(transaction=False)
for i, raw in enumerate(cast(list[str], self._redis.lrange(key, 0, -1))):
fields = rename.fields(orjson.loads(raw))
if fields is not None:
pipe.lset(key, i, orjson.dumps(fields).decode())
pipe.execute()
old, new = self._parked_key(rename.old), self._parked_key(rename.new)
if self._redis.exists(old):
# Checked first: inside the transaction a refused RENAMENX would
# not stop the index from moving.
if self._redis.exists(new):
raise RuntimeError(f"Work is parked under '{rename.new}' already")
pipe = self._redis.pipeline()
pipe.renamenx(old, new)
pipe.srem(self._parked_flows_key, rename.old)
pipe.sadd(self._parked_flows_key, rename.new)
pipe.execute()
done("lists")
move_keys(self._redis, self._marker_moves(rename))
done("done markers")
def _parked_keys(self, rename: Rename) -> list[str]:
"""Every parked list, the renamed flow's under either name included."""
flows = cast(set[str], self._redis.smembers(self._parked_flows_key))
return list(
dict.fromkeys(
map(self._parked_key, [*sorted(flows), rename.old, rename.new])
)
)
def _marker_moves(self, rename: Rename) -> list[tuple[str, str]]:
moves = []
for key in self._redis.scan_iter(
match=f"{self._ns}:{DONE}:*", count=RENAME_PAGE
):
entry_id, node = done_marker(key)
if rename.name(node) != node:
moves.append((key, self._done_key(entry_id, rename.name(node))))
return moves
def count(self, rename: Rename) -> dict[str, int]:
def changed(raws: Iterable[str]) -> int:
return sum(rename.fields(orjson.loads(raw)) is not None for raw in raws)
def listed(key: str) -> list[str]:
return cast(list[str], self._redis.lrange(key, 0, -1))
live = [
fields
for stream in (self._due_stream, self._stream)
for _, fields in self._live_entries(stream)
]
return {
"queue_items": sum(rename.fields(f) is not None for f in live)
+ changed(listed(self._held_key)),
"delayed": changed(
cast(list[str], self._redis.zrange(self._delayed_key, 0, -1))
),
"parked": sum(changed(listed(key)) for key in self._parked_keys(rename)),
"done_markers": len(self._marker_moves(rename)),
}
+116 -3
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from collections import deque
from collections.abc import Iterable, Iterator
from collections.abc import Callable, Iterable, Iterator
from contextlib import contextmanager
from threading import RLock
from typing import Any, cast
@@ -17,6 +17,8 @@ from typing import Any, cast
import orjson
import redis
from fluksio.flow.namespace import HISTORY, NAMES, Rename
#: A node may return a dict keyed by something other than a string, and the
#: stdlib encoder this replaced turned those keys into strings rather than
#: refusing them. Two differences remain and are both improvements: a
@@ -34,6 +36,9 @@ SOCKET_TIMEOUT_S = 5.0
CONNECT_TIMEOUT_S = 2.0
HISTORY_LIMIT = 120
#: How many keys a rename moves per round trip, and between two calls of its
#: ``progress``, which is where the rename journal can stop.
RENAME_BATCH = 500
def as_number(value: Any) -> float | None:
@@ -48,6 +53,30 @@ def as_number(value: Any) -> float | None:
return float(value)
def move_keys(client: redis.Redis, moves: list[tuple[str, str]]) -> int:
"""``RENAMENX`` each ``(key, target)`` in one round trip, which keeps the
Redis type and the remaining TTL. Returns how many moved.
A key gone since it was listed has expired and is skipped. A target that
exists is refused rather than overwritten: a rename checks that nothing
lives under the new name before it starts, so one there now is a fault.
"""
pipe = client.pipeline(transaction=False)
for key, target in moves:
pipe.renamenx(key, target)
replies = pipe.execute(raise_on_error=False)
moved = 0
for (key, target), reply in zip(moves, replies, strict=True):
if isinstance(reply, redis.ResponseError) and "no such key" in str(reply):
continue
if isinstance(reply, Exception):
raise reply
if not reply:
raise RuntimeError(f"Will not move '{key}' onto '{target}', which exists")
moved += 1
return moved
class StateBackend(ABC):
"""
Abstract base class for state storage backends.
@@ -287,6 +316,32 @@ class StateBackend(ABC):
"""
...
# -------------------------------------------------------------------------
# Renaming a flow key
# -------------------------------------------------------------------------
@abstractmethod
def rename_keys(
self, rename: Rename, progress: Callable[[str], None] | None = None
) -> int:
"""Move every key that names something of ``rename.old`` to the new
name (`Rename.state_key`), with its history.
Keys move, values do not: a message's payload and a node's private
state are its own, whatever names they mention. A key already under
the new name stays where it is, so a rerun after a crash finishes the
job. A target that exists is refused rather than overwritten.
:param progress: Called after each batch of keys.
:returns: How many keys moved.
:raises ValueError: For a bookkeeping key of an unregistered family.
"""
...
def count(self, rename: Rename) -> int:
"""How many keys `rename_keys` would move."""
return sum(rename.state_key(key) is not None for key in self.keys())
def __contains__(self, key: str) -> bool:
return self.exists(key)
@@ -457,6 +512,39 @@ class MemoryState(StateBackend):
with self._lock:
return list(self._history.get(key, ()))
def _moves(self, rename: Rename) -> tuple[dict[str, str], dict[str, str]]:
"""Where the keys of values and of histories go. Histories are kept
by bare message name here."""
data = {k: t for k in self._data if (t := rename.state_key(k)) is not None}
history = {k: rename.name(k) for k in self._history if rename.name(k) != k}
return data, history
def rename_keys(
self, rename: Rename, progress: Callable[[str], None] | None = None
) -> int:
"""One pass under the lock. A history moves as the deque it is, so it
keeps its capacity."""
with self._lock:
data, history = self._moves(rename)
for store, moves in ((self._data, data), (self._history, history)):
clash = next((t for t in moves.values() if t in store), None)
if clash is not None:
raise RuntimeError(
f"Will not move a key onto '{clash}', which exists"
)
for key, target in data.items():
self._data[target] = self._data.pop(key)
for key, target in history.items():
self._history[target] = self._history.pop(key)
if progress is not None:
progress("keys")
return len(data) + len(history)
def count(self, rename: Rename) -> int:
with self._lock:
data, history = self._moves(rename)
return len(data) + len(history)
class RedisState(StateBackend):
"""
@@ -533,7 +621,7 @@ class RedisState(StateBackend):
return self._deserialize(data) if data is not None else default
def _names_key(self) -> str:
return self._key("__names__")
return self._key(NAMES)
def message_names(self) -> list[str]:
"""The message names, from the set kept beside them.
@@ -770,7 +858,7 @@ class RedisState(StateBackend):
self._client.close()
def _history_key(self, key: str) -> str:
return self._key(f"__history__:{key}")
return self._key(f"{HISTORY}:{key}")
def append_history(
self, values: dict[str, Any], ts: float, limits: dict[str, int] | None = None
@@ -841,3 +929,28 @@ class RedisState(StateBackend):
ts, value = orjson.loads(entry)
points.append((float(ts), float(value)))
return points
def rename_keys(
self, rename: Rename, progress: Callable[[str], None] | None = None
) -> int:
"""``RENAMENX`` in batches, then the names set in one transaction.
The keys are listed before any moves, since a SCAN running over keys
that are being renamed can return one twice.
"""
moves = [(k, t) for k in self.keys() if (t := rename.state_key(k)) is not None]
moved = 0
for start in range(0, len(moves), RENAME_BATCH):
batch = moves[start : start + RENAME_BATCH]
moved += move_keys(
self._client, [(self._key(k), self._key(t)) for k, t in batch]
)
if progress is not None:
progress("keys")
names = [name for name in self.message_names() if rename.name(name) != name]
if names:
pipe = self._client.pipeline()
pipe.srem(self._names_key(), *names)
pipe.sadd(self._names_key(), *map(rename.name, names))
pipe.execute()
return moved