Merge branch 'rename-s2'
This commit is contained in:
@@ -0,0 +1,187 @@
|
||||
"""The names the engine keeps live state and queued work under, and how a
|
||||
rename of a flow key maps them.
|
||||
|
||||
A flow's messages and nodes are named ``<key>.<name>``, and the engine's own
|
||||
bookkeeping on them sits in state beside the values, under a reserved prefix
|
||||
and the names it is about: ``__version__:heat.temp``,
|
||||
``__last_seen__:heat.valve:hall.temp``. Renaming a flow key moves all of it to
|
||||
the new name, so every family is registered here, beside the builders that
|
||||
take their prefix from it, and a key of a family that is not is refused rather
|
||||
than left behind.
|
||||
|
||||
Nothing here does I/O. It maps names; moving the keys and items is the state
|
||||
and queue backends' work.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import orjson
|
||||
|
||||
from fluksio.flow.messages import flow_of
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fluksio.flow.queue import WorkItem
|
||||
|
||||
#: The message names in state, as a set (`RedisState` only).
|
||||
NAMES = "__names__"
|
||||
#: A message's recorded points (`RedisState`; memory keeps them apart).
|
||||
HISTORY = "__history__"
|
||||
#: How often a message was published.
|
||||
VERSION = "__version__"
|
||||
#: When a message was last published.
|
||||
TIMESTAMP = "__ts__"
|
||||
#: The version of a message a node last ran on.
|
||||
LAST_SEEN = "__last_seen__"
|
||||
#: When a rate-limited input last woke a node.
|
||||
DELIVERED = "__in_ts__"
|
||||
#: The value a rate-limited output kept back.
|
||||
HELD = "__held__"
|
||||
#: When a node's rate-limit window is due to be let through.
|
||||
FLUSH = "__flush__"
|
||||
#: What an engine-shipped node type remembers (`Node.remember`).
|
||||
NODE_STATE = "__node__"
|
||||
|
||||
#: A qualified message or node name, ``<flow key>.<name>``.
|
||||
NAME = "name"
|
||||
#: A node's own key, user data that may contain colons.
|
||||
PRIVATE = "private"
|
||||
#: A flow key.
|
||||
FLOW = "flow"
|
||||
#: A map keyed by qualified message names.
|
||||
OUTPUTS = "outputs"
|
||||
|
||||
#: Every bookkeeping family in live state, by prefix, with the parts that
|
||||
#: follow it. A key is ``<prefix>:<part>:<part>``. Names never contain a colon,
|
||||
#: so the key splits at its first ones and the last part takes the rest, which
|
||||
#: is how a private key keeps its own.
|
||||
FAMILIES: dict[str, tuple[str, ...]] = {
|
||||
NAMES: (),
|
||||
HISTORY: (NAME,),
|
||||
VERSION: (NAME,),
|
||||
TIMESTAMP: (NAME,),
|
||||
LAST_SEEN: (NAME, NAME),
|
||||
DELIVERED: (NAME, NAME),
|
||||
HELD: (NAME,),
|
||||
FLUSH: (NAME,),
|
||||
NODE_STATE: (NAME, PRIVATE),
|
||||
}
|
||||
|
||||
#: What each `WorkItem` field holds that a rename changes, or None. A guard is
|
||||
#: the node's own key, and the state it guards moves with the node.
|
||||
ITEM_FIELDS: dict[str, str | None] = {
|
||||
"kind": None,
|
||||
"node": NAME,
|
||||
"flow": FLOW,
|
||||
"outputs": OUTPUTS,
|
||||
"cause": None,
|
||||
"not_before": None,
|
||||
"guard_key": None,
|
||||
"guard_value": None,
|
||||
"entry_id": None,
|
||||
"deliveries": None,
|
||||
"enqueued_at": None,
|
||||
"run_id": None,
|
||||
"parent": None,
|
||||
}
|
||||
|
||||
|
||||
def _parts(key: str) -> tuple[str, list[tuple[str, str]]]:
|
||||
"""A state key's family prefix and its parts, each with what it holds.
|
||||
|
||||
A key without a prefix is a message value, and its family is ``""``.
|
||||
"""
|
||||
if not key.startswith("__"):
|
||||
return "", [(key, NAME)]
|
||||
family, sep, rest = key.partition(":")
|
||||
shape = FAMILIES.get(family)
|
||||
parts = rest.split(":", len(shape) - 1) if shape is not None and sep else []
|
||||
if shape is None or len(parts) != len(shape):
|
||||
raise ValueError(f"State key '{key}' is in no registered family")
|
||||
return family, list(zip(parts, shape, strict=True))
|
||||
|
||||
|
||||
def touches(flow: str, key: str) -> bool:
|
||||
"""Whether the state key *key* names anything of the flow key *flow*."""
|
||||
_, parts = _parts(key)
|
||||
return any(kind == NAME and flow_of(part) == flow for part, kind in parts)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Rename:
|
||||
"""The flow key *old* becoming *new*, applied to the names that embed it."""
|
||||
|
||||
old: str
|
||||
new: str
|
||||
|
||||
def name(self, qualified: str) -> str:
|
||||
"""A qualified message or node name in the new namespace.
|
||||
|
||||
Only the part before the first dot is the flow, so renaming ``heat``
|
||||
leaves ``heating.temp`` alone. A bare key is its own namespace.
|
||||
"""
|
||||
if flow_of(qualified) != self.old:
|
||||
return qualified
|
||||
return self.new + qualified[len(self.old) :]
|
||||
|
||||
def flow(self, key: str) -> str:
|
||||
return self.new if key == self.old else key
|
||||
|
||||
def state_key(self, key: str) -> str | None:
|
||||
"""Where the state key *key* moves to, or None if it stays.
|
||||
|
||||
:raises ValueError: For a bookkeeping key of an unregistered family.
|
||||
"""
|
||||
family, parts = _parts(key)
|
||||
mapped = [self.name(part) if kind == NAME else part for part, kind in parts]
|
||||
renamed = ":".join([family, *mapped]) if family else mapped[0]
|
||||
return renamed if renamed != key else None
|
||||
|
||||
def item(self, item: WorkItem) -> WorkItem | None:
|
||||
"""*item* with its names in the new namespace, or None if it has none.
|
||||
|
||||
A ``run`` item names its flow by id, which a rename leaves as it is.
|
||||
"""
|
||||
if item.kind == "run":
|
||||
return None
|
||||
changes = {}
|
||||
for field, holds in ITEM_FIELDS.items():
|
||||
if holds is not None:
|
||||
value = getattr(item, field)
|
||||
mapped = self._mapped(holds, value)
|
||||
if mapped != value:
|
||||
changes[field] = mapped
|
||||
return replace(item, **changes) if changes else None
|
||||
|
||||
def fields(self, fields: dict[str, str]) -> dict[str, str] | None:
|
||||
"""A serialized item (`WorkItem.to_fields`) in the new namespace, or
|
||||
None if it names nothing renamed. Fields a rename does not touch keep
|
||||
their exact text, so an unchanged item serializes as it did."""
|
||||
if fields.get("kind") == "run":
|
||||
return None
|
||||
renamed = dict(fields)
|
||||
for field, holds in ITEM_FIELDS.items():
|
||||
if holds is None or field not in fields:
|
||||
continue
|
||||
if holds == OUTPUTS:
|
||||
outputs = orjson.loads(fields[field] or "{}")
|
||||
mapped = self._mapped(holds, outputs)
|
||||
if mapped != outputs:
|
||||
renamed[field] = orjson.dumps(mapped).decode()
|
||||
else:
|
||||
renamed[field] = self._mapped(holds, fields[field])
|
||||
return renamed if renamed != fields else None
|
||||
|
||||
def dedup_key(self, key: str) -> str:
|
||||
"""An alert's ``<event type>:<node or flow>`` (`alerts.dedup_key`)."""
|
||||
event, sep, about = key.partition(":")
|
||||
return f"{event}{sep}{self.name(about)}"
|
||||
|
||||
def _mapped(self, holds: str, value: Any) -> Any:
|
||||
if holds == NAME:
|
||||
return self.name(value)
|
||||
if holds == FLOW:
|
||||
return self.flow(value)
|
||||
return {self.name(name): one for name, one in value.items()}
|
||||
@@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fluksio.flow import logs
|
||||
from fluksio.flow.messages import MessageSpec, qualify
|
||||
from fluksio.flow.namespace import NODE_STATE
|
||||
from fluksio.flow.supervision import Supervisor
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -283,7 +284,7 @@ class Node:
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def _state_key(self, key: str) -> str:
|
||||
return f"__node__:{self.id}:{key}"
|
||||
return f"{NODE_STATE}:{self.id}:{key}"
|
||||
|
||||
def remember(self, key: str, value: Any) -> None:
|
||||
"""Keep a value for the next run of this node."""
|
||||
|
||||
@@ -34,6 +34,14 @@ from fluksio.flow import logs
|
||||
from fluksio.flow.artifacts import is_reference
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.messages import flow_of, requalify
|
||||
from fluksio.flow.namespace import (
|
||||
DELIVERED,
|
||||
FLUSH,
|
||||
HELD,
|
||||
LAST_SEEN,
|
||||
TIMESTAMP,
|
||||
VERSION,
|
||||
)
|
||||
from fluksio.flow.nodes import Node
|
||||
from fluksio.flow.queue import WorkQueue
|
||||
from fluksio.flow.state import MemoryState, StateBackend
|
||||
@@ -633,25 +641,25 @@ class Pipeline:
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def _version_key(self, msg_name: str) -> str:
|
||||
return f"__version__:{msg_name}"
|
||||
return f"{VERSION}:{msg_name}"
|
||||
|
||||
def _last_seen_key(self, node_name: str, msg_name: str) -> str:
|
||||
return f"__last_seen__:{node_name}:{msg_name}"
|
||||
return f"{LAST_SEEN}:{node_name}:{msg_name}"
|
||||
|
||||
def _timestamp_key(self, msg_name: str) -> str:
|
||||
return f"__ts__:{msg_name}"
|
||||
return f"{TIMESTAMP}:{msg_name}"
|
||||
|
||||
def _delivered_key(self, node_name: str, msg_name: str) -> str:
|
||||
"""When a rate-limited input last woke this node."""
|
||||
return f"__in_ts__:{node_name}:{msg_name}"
|
||||
return f"{DELIVERED}:{node_name}:{msg_name}"
|
||||
|
||||
def _held_key(self, msg_name: str) -> str:
|
||||
"""The value a rate-limited output kept back, waiting for its window."""
|
||||
return f"__held__:{msg_name}"
|
||||
return f"{HELD}:{msg_name}"
|
||||
|
||||
def _flush_key(self, node_name: str) -> str:
|
||||
"""When a rate-limit window of this node is due to be let through."""
|
||||
return f"__flush__:{node_name}"
|
||||
return f"{FLUSH}:{node_name}"
|
||||
|
||||
def _window_split(
|
||||
self, node: Node, result: dict[str, Any], now: float
|
||||
|
||||
@@ -23,14 +23,15 @@ import threading
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import OrderedDict, deque
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, cast
|
||||
|
||||
import orjson
|
||||
import redis
|
||||
|
||||
from fluksio.flow.state import CONNECT_TIMEOUT_S, SOCKET_TIMEOUT_S
|
||||
from fluksio.flow.namespace import Rename
|
||||
from fluksio.flow.state import CONNECT_TIMEOUT_S, SOCKET_TIMEOUT_S, move_keys
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -52,6 +53,25 @@ STALE_CONSUMER_IDLE_MS = 3_600_000
|
||||
# The Redis one expires each after an hour; this is the same idea sized by
|
||||
# count, since redelivery happens seconds after the claim and never later.
|
||||
DONE_MARKERS = 10_000
|
||||
# The Redis queue keeps them as ``<namespace>:__done__:<entry id>:<node>``.
|
||||
DONE = "__done__"
|
||||
# How many entries a rename reads, or writes, per round trip.
|
||||
RENAME_PAGE = 500
|
||||
#: Every structure under the Redis queue's namespace. `rename` rewrites the
|
||||
#: names in all of them but the dead letters, which are history. One added
|
||||
#: here holds names too, or has to say why it does not.
|
||||
STRUCTURES = frozenset(
|
||||
{
|
||||
"__queue__",
|
||||
"__due__",
|
||||
"__delayed__",
|
||||
"__parked__",
|
||||
"__parked_flows__",
|
||||
"__held__",
|
||||
DONE,
|
||||
"__dead__",
|
||||
}
|
||||
)
|
||||
|
||||
# A promotion is one Redis operation, including when the caller loses its
|
||||
# connection before hearing the result. Write first: Lua does not roll back a
|
||||
@@ -70,6 +90,24 @@ end
|
||||
return #due
|
||||
"""
|
||||
|
||||
# Work held during a rename goes into the backlog oldest first, behind
|
||||
# everything already there. As in `_MOVE_DUE`, an item leaves the held list
|
||||
# only once its XADD went through. 100 per call, so Redis is never blocked
|
||||
# for long.
|
||||
_RELEASE = """
|
||||
local held = redis.call('LRANGE', KEYS[1], 0, 99)
|
||||
for _, raw in ipairs(held) do
|
||||
local fields = {}
|
||||
for key, value in pairs(cjson.decode(raw)) do
|
||||
fields[#fields + 1] = key
|
||||
fields[#fields + 1] = value
|
||||
end
|
||||
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[1], '*', unpack(fields))
|
||||
redis.call('LPOP', KEYS[1])
|
||||
end
|
||||
return #held
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkItem:
|
||||
@@ -162,6 +200,102 @@ def owned_by(item: WorkItem, flow: str) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def done_marker(key: str) -> tuple[str, str]:
|
||||
"""The entry id and node a done marker is for: `RedisWorkQueue._done_key`
|
||||
read back. Split at the last colon, since an entry id from the due lane
|
||||
has one of its own and a node id never does."""
|
||||
entry_id, node = key.split(f":{DONE}:", 1)[1].rsplit(":", 1)
|
||||
return entry_id, node
|
||||
|
||||
|
||||
def _stream_id(entry_id: str) -> tuple[int, int]:
|
||||
ms, _, seq = entry_id.partition("-")
|
||||
return int(ms), int(seq)
|
||||
|
||||
|
||||
def rebuild_stream(client: redis.Redis, stream: str, rename: Rename) -> None:
|
||||
"""Rewrite the names in the entries of *stream* still to run, keeping
|
||||
every entry id, the group and its consumers, each pending entry's owner,
|
||||
delivery count and idle time, and the stream's id counters.
|
||||
|
||||
An entry cannot be changed in place, and one added again gets a new id,
|
||||
which is what runs, events and done markers are filed under, and loses its
|
||||
delivery count. So the stream is copied into a temporary key with its ids
|
||||
and swapped in with one RENAME, the only commit point: a rerun after a
|
||||
crash on either side of it converges. Acknowledged entries are history and
|
||||
copied as they are. Nothing may write to the stream meanwhile. Needs Redis
|
||||
7.0 (docs/private/id_refac_rename.md, 3.1).
|
||||
"""
|
||||
tmp = f"{stream}:rename"
|
||||
client.delete(tmp)
|
||||
info = cast(dict[str, Any], client.xinfo_stream(stream))
|
||||
groups = cast(list[dict[str, Any]], client.xinfo_groups(stream))
|
||||
group = next(g for g in groups if g["name"] == GROUP)
|
||||
last = str(group["last-delivered-id"])
|
||||
pel: list[dict[str, Any]] = []
|
||||
start = "-"
|
||||
while rows := cast(
|
||||
list[dict[str, Any]],
|
||||
client.xpending_range(stream, GROUP, min=start, max="+", count=RENAME_PAGE),
|
||||
):
|
||||
pel += rows
|
||||
start = f"({rows[-1]['message_id']}"
|
||||
pending = {row["message_id"] for row in pel}
|
||||
|
||||
pipe = client.pipeline(transaction=False)
|
||||
# MKSTREAM, so XSETID finds the key when there is nothing to copy.
|
||||
pipe.xgroup_create(tmp, GROUP, id=last, mkstream=True)
|
||||
for consumer in cast(list[dict[str, Any]], client.xinfo_consumers(stream, GROUP)):
|
||||
pipe.xgroup_createconsumer(tmp, GROUP, consumer["name"])
|
||||
copied: set[str] = set()
|
||||
start = "-"
|
||||
while entries := cast(
|
||||
list[tuple[str, dict[str, str]]],
|
||||
client.xrange(stream, min=start, max="+", count=RENAME_PAGE),
|
||||
):
|
||||
for entry_id, fields in entries:
|
||||
live = entry_id in pending or _stream_id(entry_id) > _stream_id(last)
|
||||
renamed = rename.fields(fields) if live else None
|
||||
pipe.xadd(tmp, cast(Any, renamed or fields), id=entry_id)
|
||||
if entry_id in pending:
|
||||
copied.add(entry_id)
|
||||
pipe.execute()
|
||||
start = f"({entries[-1][0]}"
|
||||
# redis-py has no XSETID of its own, and leaves the generic call untyped.
|
||||
cast(Any, pipe).execute_command(
|
||||
"XSETID",
|
||||
tmp,
|
||||
info["last-generated-id"],
|
||||
"ENTRIESADDED",
|
||||
info["entries-added"],
|
||||
"MAXDELETEDID",
|
||||
info["max-deleted-entry-id"],
|
||||
)
|
||||
# Redis caps ENTRIESREAD at the stream's entries-added counter, which only
|
||||
# XSETID has set. Unknown stays unknown.
|
||||
if group["entries-read"] is not None:
|
||||
pipe.xgroup_setid(tmp, GROUP, last, entries_read=group["entries-read"])
|
||||
# A pending entry whose stream entry was trimmed is not recreated, as
|
||||
# `_reclaim` drops it too. XCLAIM would still create its consumer.
|
||||
for row in pel:
|
||||
if row["message_id"] in copied:
|
||||
pipe.xclaim(
|
||||
tmp,
|
||||
GROUP,
|
||||
row["consumer"],
|
||||
0,
|
||||
[row["message_id"]],
|
||||
idle=row["time_since_delivered"],
|
||||
retrycount=row["times_delivered"],
|
||||
force=True,
|
||||
justid=True,
|
||||
)
|
||||
pipe.execute()
|
||||
# On its own: a pipeline without a transaction runs every command even
|
||||
# after one of them failed.
|
||||
client.rename(tmp, stream)
|
||||
|
||||
|
||||
class WorkQueue(ABC):
|
||||
"""What the execution service pulls from."""
|
||||
|
||||
@@ -299,6 +433,52 @@ class WorkQueue(ABC):
|
||||
def close(self) -> None:
|
||||
"""Release whatever the queue holds."""
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Renaming a flow key
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
def hold(self) -> None:
|
||||
"""Divert `add` into a held list until `release`, so a rename can
|
||||
rewrite the queue while listeners keep journaling work."""
|
||||
|
||||
@abstractmethod
|
||||
def release(self) -> None:
|
||||
"""Move the held work into the backlog, oldest first and behind
|
||||
everything already there, and stop diverting."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def held(self) -> bool:
|
||||
"""Whether `add` is diverted."""
|
||||
|
||||
@abstractmethod
|
||||
def release_held(self) -> int:
|
||||
"""Release a held list left by a process that stopped mid-rename. For
|
||||
startup, before anything claims. Returns how many items moved."""
|
||||
|
||||
@abstractmethod
|
||||
def rename(
|
||||
self, rename: Rename, progress: Callable[[str], None] | None = None
|
||||
) -> None:
|
||||
"""Move the names in the work still to run from ``rename.old`` to
|
||||
``rename.new`` (`Rename.fields`).
|
||||
|
||||
Queued, claimed, delayed, parked and held items, and the done markers
|
||||
that keep a replayed side effect from repeating. Entry ids, order,
|
||||
deadlines, delivery counts, owners and guards stay as they are. Dead
|
||||
letters and acknowledged entries are history and stay too. A rerun
|
||||
after a crash completes the job. Nothing may claim, promote, reclaim,
|
||||
park or unpark meanwhile.
|
||||
|
||||
:param progress: Called with the name of each structure once it is done.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def count(self, rename: Rename) -> dict[str, int]:
|
||||
"""How many entries `rename` would rewrite: ``queue_items`` (queued,
|
||||
claimed or held), ``delayed``, ``parked`` and ``done_markers``."""
|
||||
|
||||
|
||||
class MemoryWorkQueue(WorkQueue):
|
||||
"""In-process queue. No durability: a crash loses whatever is in flight."""
|
||||
@@ -318,6 +498,9 @@ class MemoryWorkQueue(WorkQueue):
|
||||
self._done: OrderedDict[tuple[str, str], None] = OrderedDict()
|
||||
# Claimed and not yet acknowledged, which is what Redis's `pending` is.
|
||||
self._in_flight = 0
|
||||
# What was added while a rename held the queue (`hold`).
|
||||
self._held: deque[WorkItem] = deque()
|
||||
self._holding = False
|
||||
self._counter = 0
|
||||
self._seq = 0
|
||||
# The ids become `FlowRun.id`, a primary key. Counting from zero alone,
|
||||
@@ -334,6 +517,9 @@ class MemoryWorkQueue(WorkQueue):
|
||||
with self._wake:
|
||||
if not item.entry_id:
|
||||
item.entry_id = self._next_id()
|
||||
if self._holding:
|
||||
self._held.append(item)
|
||||
return
|
||||
self._items.append(item)
|
||||
self._wake.notify()
|
||||
|
||||
@@ -429,6 +615,7 @@ class MemoryWorkQueue(WorkQueue):
|
||||
*self._due,
|
||||
*(entry[2] for entry in self._delayed),
|
||||
*(item for held in self._parked.values() for item in held),
|
||||
*self._held,
|
||||
]
|
||||
|
||||
def backlog(self) -> int:
|
||||
@@ -463,6 +650,66 @@ class MemoryWorkQueue(WorkQueue):
|
||||
def close(self) -> None:
|
||||
"""Nothing is held outside the process."""
|
||||
|
||||
def hold(self) -> None:
|
||||
with self._lock:
|
||||
self._holding = True
|
||||
|
||||
def release(self) -> None:
|
||||
with self._wake:
|
||||
self._holding = False
|
||||
self._items.extend(self._held)
|
||||
self._held.clear()
|
||||
self._wake.notify()
|
||||
|
||||
@property
|
||||
def held(self) -> bool:
|
||||
return self._holding
|
||||
|
||||
def release_held(self) -> int:
|
||||
"""Nothing held here outlives the process."""
|
||||
return 0
|
||||
|
||||
def rename(
|
||||
self, rename: Rename, progress: Callable[[str], None] | None = None
|
||||
) -> None:
|
||||
"""One pass under the lock. Timers keep their place in the heap,
|
||||
which orders them by deadline and arrival only."""
|
||||
|
||||
def moved(item: WorkItem) -> WorkItem:
|
||||
return rename.item(item) or item
|
||||
|
||||
with self._lock:
|
||||
if rename.old in self._parked and rename.new in self._parked:
|
||||
raise RuntimeError(f"Work is parked under '{rename.new}' already")
|
||||
for lane in (self._items, self._due, self._held):
|
||||
items = [moved(item) for item in lane]
|
||||
lane.clear()
|
||||
lane.extend(items)
|
||||
self._delayed = [(at, n, moved(item)) for at, n, item in self._delayed]
|
||||
self._parked = {
|
||||
rename.flow(flow): [moved(item) for item in items]
|
||||
for flow, items in self._parked.items()
|
||||
}
|
||||
self._done = OrderedDict(
|
||||
((entry_id, rename.name(node)), None) for entry_id, node in self._done
|
||||
)
|
||||
if progress is not None:
|
||||
progress("queue")
|
||||
|
||||
def count(self, rename: Rename) -> dict[str, int]:
|
||||
"""What is claimed is gone from here, so it is not counted."""
|
||||
|
||||
def changed(items: Iterable[WorkItem]) -> int:
|
||||
return sum(rename.item(item) is not None for item in items)
|
||||
|
||||
with self._lock:
|
||||
return {
|
||||
"queue_items": changed([*self._items, *self._due, *self._held]),
|
||||
"delayed": changed(entry[2] for entry in self._delayed),
|
||||
"parked": changed(i for held in self._parked.values() for i in held),
|
||||
"done_markers": sum(rename.name(n) != n for _, n in self._done),
|
||||
}
|
||||
|
||||
|
||||
class RedisWorkQueue(WorkQueue):
|
||||
"""Redis Streams queue: the one that survives the process.
|
||||
@@ -505,7 +752,16 @@ class RedisWorkQueue(WorkQueue):
|
||||
# finding them with a keyspace scan walked every state and idempotency
|
||||
# key in the database, twice per health poll.
|
||||
self._parked_flows_key = f"{namespace}:__parked_flows__"
|
||||
# What was added while a rename held the queue (`hold`).
|
||||
self._held_key = f"{namespace}:__held__"
|
||||
self._move_due = self._redis.register_script(_MOVE_DUE)
|
||||
self._release = self._redis.register_script(_RELEASE)
|
||||
# `add` reads the flag and writes under this, and `release` empties the
|
||||
# held list and clears the flag under it. So nothing added after `hold`
|
||||
# lands in a stream a rename is rebuilding, and nothing added while
|
||||
# held overtakes held work.
|
||||
self._hold_lock = threading.Lock()
|
||||
self._holding = False
|
||||
self._ensure_group()
|
||||
self._drop_stale_consumers()
|
||||
|
||||
@@ -548,15 +804,19 @@ class RedisWorkQueue(WorkQueue):
|
||||
return f"{self._ns}:__parked__:{flow}"
|
||||
|
||||
def _done_key(self, entry_id: str, node: str) -> str:
|
||||
return f"{self._ns}:__done__:{entry_id}:{node}"
|
||||
return f"{self._ns}:{DONE}:{entry_id}:{node}"
|
||||
|
||||
def add(self, item: WorkItem) -> None:
|
||||
self._redis.xadd(
|
||||
self._stream,
|
||||
cast(Any, item.to_fields()),
|
||||
maxlen=STREAM_MAXLEN,
|
||||
approximate=True,
|
||||
)
|
||||
with self._hold_lock:
|
||||
if self._holding:
|
||||
self._redis.rpush(self._held_key, orjson.dumps(item.to_fields()))
|
||||
return
|
||||
self._redis.xadd(
|
||||
self._stream,
|
||||
cast(Any, item.to_fields()),
|
||||
maxlen=STREAM_MAXLEN,
|
||||
approximate=True,
|
||||
)
|
||||
|
||||
def add_delayed(self, item: WorkItem, not_before: float) -> None:
|
||||
item.not_before = not_before
|
||||
@@ -785,6 +1045,10 @@ class RedisWorkQueue(WorkQueue):
|
||||
list[str], self._redis.lrange(self._parked_key(flow), 0, -1)
|
||||
)
|
||||
]
|
||||
items += [
|
||||
WorkItem.from_fields(orjson.loads(raw), "")
|
||||
for raw in cast(list[str], self._redis.lrange(self._held_key, 0, -1))
|
||||
]
|
||||
return items
|
||||
|
||||
def _group_info(self, stream: str = "") -> dict[str, Any]:
|
||||
@@ -886,3 +1150,120 @@ class RedisWorkQueue(WorkQueue):
|
||||
|
||||
def close(self) -> None:
|
||||
self._redis.close()
|
||||
|
||||
def hold(self) -> None:
|
||||
with self._hold_lock:
|
||||
self._holding = True
|
||||
|
||||
def release(self) -> None:
|
||||
with self._hold_lock:
|
||||
self.release_held()
|
||||
self._holding = False
|
||||
|
||||
@property
|
||||
def held(self) -> bool:
|
||||
return self._holding
|
||||
|
||||
def release_held(self) -> int:
|
||||
released = 0
|
||||
while moved := cast(
|
||||
int,
|
||||
self._release(keys=[self._held_key, self._stream], args=[STREAM_MAXLEN]),
|
||||
):
|
||||
released += moved
|
||||
return released
|
||||
|
||||
def rename(
|
||||
self, rename: Rename, progress: Callable[[str], None] | None = None
|
||||
) -> None:
|
||||
"""Each structure as section 3 of docs/private/id_refac_rename.md has
|
||||
it: the streams rebuilt, then timers, lists and done markers, each
|
||||
rewritten one member, slot or key at a time."""
|
||||
|
||||
def done(structure: str) -> None:
|
||||
if progress is not None:
|
||||
progress(structure)
|
||||
|
||||
for stream, lane in ((self._due_stream, "due"), (self._stream, "backlog")):
|
||||
rebuild_stream(self._redis, stream, rename)
|
||||
done(lane)
|
||||
|
||||
# One transaction, so each timer moves with its deadline. Promotion is
|
||||
# held, and a timer added meanwhile is not touched.
|
||||
pipe = self._redis.pipeline()
|
||||
for raw, at in cast(
|
||||
list[tuple[str, float]],
|
||||
self._redis.zrange(self._delayed_key, 0, -1, withscores=True),
|
||||
):
|
||||
fields = rename.fields(orjson.loads(raw))
|
||||
if fields is not None:
|
||||
pipe.zrem(self._delayed_key, raw)
|
||||
pipe.zadd(self._delayed_key, {orjson.dumps(fields): at})
|
||||
pipe.execute()
|
||||
done("delayed")
|
||||
|
||||
# Slot by slot: nothing pops while held, and `add` only appends, so
|
||||
# the indices read hold.
|
||||
for key in [*self._parked_keys(rename), self._held_key]:
|
||||
pipe = self._redis.pipeline(transaction=False)
|
||||
for i, raw in enumerate(cast(list[str], self._redis.lrange(key, 0, -1))):
|
||||
fields = rename.fields(orjson.loads(raw))
|
||||
if fields is not None:
|
||||
pipe.lset(key, i, orjson.dumps(fields).decode())
|
||||
pipe.execute()
|
||||
old, new = self._parked_key(rename.old), self._parked_key(rename.new)
|
||||
if self._redis.exists(old):
|
||||
# Checked first: inside the transaction a refused RENAMENX would
|
||||
# not stop the index from moving.
|
||||
if self._redis.exists(new):
|
||||
raise RuntimeError(f"Work is parked under '{rename.new}' already")
|
||||
pipe = self._redis.pipeline()
|
||||
pipe.renamenx(old, new)
|
||||
pipe.srem(self._parked_flows_key, rename.old)
|
||||
pipe.sadd(self._parked_flows_key, rename.new)
|
||||
pipe.execute()
|
||||
done("lists")
|
||||
|
||||
move_keys(self._redis, self._marker_moves(rename))
|
||||
done("done markers")
|
||||
|
||||
def _parked_keys(self, rename: Rename) -> list[str]:
|
||||
"""Every parked list, the renamed flow's under either name included."""
|
||||
flows = cast(set[str], self._redis.smembers(self._parked_flows_key))
|
||||
return list(
|
||||
dict.fromkeys(
|
||||
map(self._parked_key, [*sorted(flows), rename.old, rename.new])
|
||||
)
|
||||
)
|
||||
|
||||
def _marker_moves(self, rename: Rename) -> list[tuple[str, str]]:
|
||||
moves = []
|
||||
for key in self._redis.scan_iter(
|
||||
match=f"{self._ns}:{DONE}:*", count=RENAME_PAGE
|
||||
):
|
||||
entry_id, node = done_marker(key)
|
||||
if rename.name(node) != node:
|
||||
moves.append((key, self._done_key(entry_id, rename.name(node))))
|
||||
return moves
|
||||
|
||||
def count(self, rename: Rename) -> dict[str, int]:
|
||||
def changed(raws: Iterable[str]) -> int:
|
||||
return sum(rename.fields(orjson.loads(raw)) is not None for raw in raws)
|
||||
|
||||
def listed(key: str) -> list[str]:
|
||||
return cast(list[str], self._redis.lrange(key, 0, -1))
|
||||
|
||||
live = [
|
||||
fields
|
||||
for stream in (self._due_stream, self._stream)
|
||||
for _, fields in self._live_entries(stream)
|
||||
]
|
||||
return {
|
||||
"queue_items": sum(rename.fields(f) is not None for f in live)
|
||||
+ changed(listed(self._held_key)),
|
||||
"delayed": changed(
|
||||
cast(list[str], self._redis.zrange(self._delayed_key, 0, -1))
|
||||
),
|
||||
"parked": sum(changed(listed(key)) for key in self._parked_keys(rename)),
|
||||
"done_markers": len(self._marker_moves(rename)),
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import deque
|
||||
from collections.abc import Iterable, Iterator
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from threading import RLock
|
||||
from typing import Any, cast
|
||||
@@ -17,6 +17,8 @@ from typing import Any, cast
|
||||
import orjson
|
||||
import redis
|
||||
|
||||
from fluksio.flow.namespace import HISTORY, NAMES, Rename
|
||||
|
||||
#: A node may return a dict keyed by something other than a string, and the
|
||||
#: stdlib encoder this replaced turned those keys into strings rather than
|
||||
#: refusing them. Two differences remain and are both improvements: a
|
||||
@@ -34,6 +36,9 @@ SOCKET_TIMEOUT_S = 5.0
|
||||
CONNECT_TIMEOUT_S = 2.0
|
||||
|
||||
HISTORY_LIMIT = 120
|
||||
#: How many keys a rename moves per round trip, and between two calls of its
|
||||
#: ``progress``, which is where the rename journal can stop.
|
||||
RENAME_BATCH = 500
|
||||
|
||||
|
||||
def as_number(value: Any) -> float | None:
|
||||
@@ -48,6 +53,30 @@ def as_number(value: Any) -> float | None:
|
||||
return float(value)
|
||||
|
||||
|
||||
def move_keys(client: redis.Redis, moves: list[tuple[str, str]]) -> int:
|
||||
"""``RENAMENX`` each ``(key, target)`` in one round trip, which keeps the
|
||||
Redis type and the remaining TTL. Returns how many moved.
|
||||
|
||||
A key gone since it was listed has expired and is skipped. A target that
|
||||
exists is refused rather than overwritten: a rename checks that nothing
|
||||
lives under the new name before it starts, so one there now is a fault.
|
||||
"""
|
||||
pipe = client.pipeline(transaction=False)
|
||||
for key, target in moves:
|
||||
pipe.renamenx(key, target)
|
||||
replies = pipe.execute(raise_on_error=False)
|
||||
moved = 0
|
||||
for (key, target), reply in zip(moves, replies, strict=True):
|
||||
if isinstance(reply, redis.ResponseError) and "no such key" in str(reply):
|
||||
continue
|
||||
if isinstance(reply, Exception):
|
||||
raise reply
|
||||
if not reply:
|
||||
raise RuntimeError(f"Will not move '{key}' onto '{target}', which exists")
|
||||
moved += 1
|
||||
return moved
|
||||
|
||||
|
||||
class StateBackend(ABC):
|
||||
"""
|
||||
Abstract base class for state storage backends.
|
||||
@@ -287,6 +316,32 @@ class StateBackend(ABC):
|
||||
"""
|
||||
...
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Renaming a flow key
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
def rename_keys(
|
||||
self, rename: Rename, progress: Callable[[str], None] | None = None
|
||||
) -> int:
|
||||
"""Move every key that names something of ``rename.old`` to the new
|
||||
name (`Rename.state_key`), with its history.
|
||||
|
||||
Keys move, values do not: a message's payload and a node's private
|
||||
state are its own, whatever names they mention. A key already under
|
||||
the new name stays where it is, so a rerun after a crash finishes the
|
||||
job. A target that exists is refused rather than overwritten.
|
||||
|
||||
:param progress: Called after each batch of keys.
|
||||
:returns: How many keys moved.
|
||||
:raises ValueError: For a bookkeeping key of an unregistered family.
|
||||
"""
|
||||
...
|
||||
|
||||
def count(self, rename: Rename) -> int:
|
||||
"""How many keys `rename_keys` would move."""
|
||||
return sum(rename.state_key(key) is not None for key in self.keys())
|
||||
|
||||
def __contains__(self, key: str) -> bool:
|
||||
return self.exists(key)
|
||||
|
||||
@@ -457,6 +512,39 @@ class MemoryState(StateBackend):
|
||||
with self._lock:
|
||||
return list(self._history.get(key, ()))
|
||||
|
||||
def _moves(self, rename: Rename) -> tuple[dict[str, str], dict[str, str]]:
|
||||
"""Where the keys of values and of histories go. Histories are kept
|
||||
by bare message name here."""
|
||||
data = {k: t for k in self._data if (t := rename.state_key(k)) is not None}
|
||||
history = {k: rename.name(k) for k in self._history if rename.name(k) != k}
|
||||
return data, history
|
||||
|
||||
def rename_keys(
|
||||
self, rename: Rename, progress: Callable[[str], None] | None = None
|
||||
) -> int:
|
||||
"""One pass under the lock. A history moves as the deque it is, so it
|
||||
keeps its capacity."""
|
||||
with self._lock:
|
||||
data, history = self._moves(rename)
|
||||
for store, moves in ((self._data, data), (self._history, history)):
|
||||
clash = next((t for t in moves.values() if t in store), None)
|
||||
if clash is not None:
|
||||
raise RuntimeError(
|
||||
f"Will not move a key onto '{clash}', which exists"
|
||||
)
|
||||
for key, target in data.items():
|
||||
self._data[target] = self._data.pop(key)
|
||||
for key, target in history.items():
|
||||
self._history[target] = self._history.pop(key)
|
||||
if progress is not None:
|
||||
progress("keys")
|
||||
return len(data) + len(history)
|
||||
|
||||
def count(self, rename: Rename) -> int:
|
||||
with self._lock:
|
||||
data, history = self._moves(rename)
|
||||
return len(data) + len(history)
|
||||
|
||||
|
||||
class RedisState(StateBackend):
|
||||
"""
|
||||
@@ -533,7 +621,7 @@ class RedisState(StateBackend):
|
||||
return self._deserialize(data) if data is not None else default
|
||||
|
||||
def _names_key(self) -> str:
|
||||
return self._key("__names__")
|
||||
return self._key(NAMES)
|
||||
|
||||
def message_names(self) -> list[str]:
|
||||
"""The message names, from the set kept beside them.
|
||||
@@ -770,7 +858,7 @@ class RedisState(StateBackend):
|
||||
self._client.close()
|
||||
|
||||
def _history_key(self, key: str) -> str:
|
||||
return self._key(f"__history__:{key}")
|
||||
return self._key(f"{HISTORY}:{key}")
|
||||
|
||||
def append_history(
|
||||
self, values: dict[str, Any], ts: float, limits: dict[str, int] | None = None
|
||||
@@ -841,3 +929,28 @@ class RedisState(StateBackend):
|
||||
ts, value = orjson.loads(entry)
|
||||
points.append((float(ts), float(value)))
|
||||
return points
|
||||
|
||||
def rename_keys(
|
||||
self, rename: Rename, progress: Callable[[str], None] | None = None
|
||||
) -> int:
|
||||
"""``RENAMENX`` in batches, then the names set in one transaction.
|
||||
|
||||
The keys are listed before any moves, since a SCAN running over keys
|
||||
that are being renamed can return one twice.
|
||||
"""
|
||||
moves = [(k, t) for k in self.keys() if (t := rename.state_key(k)) is not None]
|
||||
moved = 0
|
||||
for start in range(0, len(moves), RENAME_BATCH):
|
||||
batch = moves[start : start + RENAME_BATCH]
|
||||
moved += move_keys(
|
||||
self._client, [(self._key(k), self._key(t)) for k, t in batch]
|
||||
)
|
||||
if progress is not None:
|
||||
progress("keys")
|
||||
names = [name for name in self.message_names() if rename.name(name) != name]
|
||||
if names:
|
||||
pipe = self._client.pipeline()
|
||||
pipe.srem(self._names_key(), *names)
|
||||
pipe.sadd(self._names_key(), *map(rename.name, names))
|
||||
pipe.execute()
|
||||
return moved
|
||||
|
||||
Reference in New Issue
Block a user