1281 lines
50 KiB
Python
1281 lines
50 KiB
Python
"""The engine's work queue: what survives a crash.
|
|
|
|
Every external event — an MQTT message, a webhook, a cron tick, a connector
|
|
poll — becomes a work item before anything runs. The item is journaled first
|
|
and acknowledged only once the wave it started has quiesced, so an engine that
|
|
dies mid-cascade picks the work up again on the way back rather than losing it.
|
|
|
|
Redis journals with ``appendfsync everysec``, so the bound is the last second:
|
|
an unclean stop of Redis or of its host loses up to a second of entries. An
|
|
engine that dies while Redis lives loses none of them.
|
|
|
|
Two implementations: Redis Streams, which is what makes the above true, and an
|
|
in-memory one for tests and for running without Redis, where "durable" degrades
|
|
honestly to "not".
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import heapq
|
|
import logging
|
|
import secrets
|
|
import threading
|
|
import time
|
|
from abc import ABC, abstractmethod
|
|
from collections import OrderedDict, deque
|
|
from collections.abc import Callable, Iterable
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, cast
|
|
|
|
import orjson
|
|
import redis
|
|
|
|
from fluksio.flow.namespace import Rename
|
|
from fluksio.flow.state import CONNECT_TIMEOUT_S, SOCKET_TIMEOUT_S, move_keys
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# The stream is a buffer, not an archive: anything this far behind is long
|
|
# superseded by a newer value of the same message.
|
|
STREAM_MAXLEN = 10_000
|
|
# An item redelivered this often is not going to succeed. Park it where someone
|
|
# can look rather than letting it loop forever.
|
|
MAX_DELIVERIES = 3
|
|
GROUP = "engine"
|
|
# Entry ids from the due lane wear this, so one id still says which stream it
|
|
# came from. Deterministic per stream, which is what keeps it usable as an
|
|
# idempotency key across redeliveries.
|
|
DUE_PREFIX = "due:"
|
|
# A consumer this far past its last read belongs to an engine that is gone. A
|
|
# live one interacts every claim, so nothing in service comes close.
|
|
STALE_CONSUMER_IDLE_MS = 3_600_000
|
|
# How many "this side effect already happened" markers the memory queue keeps.
|
|
# The Redis one expires each after an hour; this is the same idea sized by
|
|
# count, since redelivery happens seconds after the claim and never later.
|
|
DONE_MARKERS = 10_000
|
|
# The Redis queue keeps them as ``<namespace>:__done__:<entry id>:<node>``.
|
|
DONE = "__done__"
|
|
# How many entries a rename reads, or writes, per round trip.
|
|
RENAME_PAGE = 500
|
|
#: Every structure under the Redis queue's namespace. `rename` rewrites the
|
|
#: names in all of them but the dead letters, which are history. One added
|
|
#: here holds names too, or has to say why it does not.
|
|
STRUCTURES = frozenset(
|
|
{
|
|
"__queue__",
|
|
"__due__",
|
|
"__delayed__",
|
|
"__parked__",
|
|
"__parked_flows__",
|
|
"__held__",
|
|
DONE,
|
|
"__dead__",
|
|
}
|
|
)
|
|
|
|
# A promotion is one Redis operation, including when the caller loses its
|
|
# connection before hearing the result. Write first: Lua does not roll back a
|
|
# command error, so a refused XADD must leave its timer in the delayed set.
|
|
_MOVE_DUE = """
|
|
local due = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[1], 'LIMIT', 0, 100)
|
|
for _, raw in ipairs(due) do
|
|
local fields = {}
|
|
for key, value in pairs(cjson.decode(raw)) do
|
|
fields[#fields + 1] = key
|
|
fields[#fields + 1] = value
|
|
end
|
|
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[2], '*', unpack(fields))
|
|
redis.call('ZREM', KEYS[1], raw)
|
|
end
|
|
return #due
|
|
"""
|
|
|
|
# Work held during a rename goes into the backlog oldest first, behind
|
|
# everything already there. As in `_MOVE_DUE`, an item leaves the held list
|
|
# only once its XADD went through. 100 per call, so Redis is never blocked
|
|
# for long.
|
|
_RELEASE = """
|
|
local held = redis.call('LRANGE', KEYS[1], 0, 99)
|
|
for _, raw in ipairs(held) do
|
|
local fields = {}
|
|
for key, value in pairs(cjson.decode(raw)) do
|
|
fields[#fields + 1] = key
|
|
fields[#fields + 1] = value
|
|
end
|
|
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[1], '*', unpack(fields))
|
|
redis.call('LPOP', KEYS[1])
|
|
end
|
|
return #held
|
|
"""
|
|
|
|
|
|
@dataclass
|
|
class WorkItem:
|
|
"""One unit of journaled work.
|
|
|
|
:param kind: ``cascade`` replays a node's outputs and runs what is
|
|
downstream; ``emission`` is a value a node published while still
|
|
running, already in state, carried so its readers get *that* value
|
|
rather than whichever is newest when they run; ``flush`` lets out what
|
|
a node's rate limits held back; ``run`` is a whole batch run, and
|
|
carries only its id.
|
|
:param node: The node the item is about — the source for a cascade, the
|
|
one whose rate limits are released for a flush.
|
|
:param flow: The flow that node belongs to, so gating needs no lookup.
|
|
:param outputs: What the source node emitted (cascade and emission).
|
|
:param cause: Where the work came from, for logs and debugging.
|
|
:param not_before: Epoch seconds before which the item must not run.
|
|
:param guard: ``(key, value)`` the target node must still remember for
|
|
this item to be worth running — how a rescheduled wait cancels the
|
|
one it replaced.
|
|
:param entry_id: Set by the queue on claim; stable across redeliveries,
|
|
which is what makes it usable as an idempotency key.
|
|
:param deliveries: How many times this item has been handed out.
|
|
:param enqueued_at: When the item was made. Carried rather than read off
|
|
the entry id, because only the Redis queue has timestamps in its ids.
|
|
"""
|
|
|
|
kind: str
|
|
node: str
|
|
flow: str
|
|
outputs: dict[str, Any] = field(default_factory=dict)
|
|
cause: str = "system"
|
|
not_before: float = 0.0
|
|
guard_key: str = ""
|
|
guard_value: str = ""
|
|
entry_id: str = ""
|
|
deliveries: int = 1
|
|
enqueued_at: float = field(default_factory=time.time)
|
|
#: The run this item is, on a ``run`` item. Everything else about a run is
|
|
#: in its database row, so the journal only has to name it.
|
|
run_id: str = ""
|
|
#: The run that scheduled this item, on a ``flush``: the one whose value
|
|
#: the rate limit held back.
|
|
parent: str = ""
|
|
|
|
def to_fields(self) -> dict[str, str]:
|
|
return {
|
|
"kind": self.kind,
|
|
"node": self.node,
|
|
"flow": self.flow,
|
|
"outputs": orjson.dumps(self.outputs).decode(),
|
|
"cause": self.cause,
|
|
"not_before": str(self.not_before),
|
|
"guard_key": self.guard_key,
|
|
"guard_value": self.guard_value,
|
|
"enqueued_at": str(self.enqueued_at),
|
|
"run_id": self.run_id,
|
|
"parent": self.parent,
|
|
}
|
|
|
|
@classmethod
|
|
def from_fields(
|
|
cls, fields: dict[str, str], entry_id: str, deliveries: int = 1
|
|
) -> WorkItem:
|
|
return cls(
|
|
kind=fields.get("kind", "cascade"),
|
|
node=fields.get("node", ""),
|
|
flow=fields.get("flow", ""),
|
|
outputs=orjson.loads(fields.get("outputs") or "{}"),
|
|
cause=fields.get("cause", "system"),
|
|
not_before=float(fields.get("not_before") or 0.0),
|
|
guard_key=fields.get("guard_key", ""),
|
|
guard_value=fields.get("guard_value", ""),
|
|
entry_id=entry_id,
|
|
deliveries=deliveries,
|
|
enqueued_at=float(fields.get("enqueued_at") or time.time()),
|
|
run_id=fields.get("run_id", ""),
|
|
parent=fields.get("parent", ""),
|
|
)
|
|
|
|
|
|
def owned_by(item: WorkItem, flow: str) -> bool:
|
|
"""Whether *item* is work of the flow with key *flow*, not one reaching it.
|
|
|
|
A run item names its run's flow by identity rather than by key, and a
|
|
run is never removed with a flow's queued work, so it is nobody's here.
|
|
"""
|
|
return item.kind != "run" and (
|
|
item.flow == flow or item.node.split(".", 1)[0] == flow
|
|
)
|
|
|
|
|
|
def done_marker(key: str) -> tuple[str, str]:
|
|
"""The entry id and node a done marker is for: `RedisWorkQueue._done_key`
|
|
read back. Split at the last colon, since an entry id from the due lane
|
|
has one of its own and a node id never does."""
|
|
entry_id, node = key.split(f":{DONE}:", 1)[1].rsplit(":", 1)
|
|
return entry_id, node
|
|
|
|
|
|
def _stream_id(entry_id: str) -> tuple[int, int]:
|
|
ms, _, seq = entry_id.partition("-")
|
|
return int(ms), int(seq)
|
|
|
|
|
|
def rebuild_stream(client: redis.Redis, stream: str, rename: Rename) -> None:
|
|
"""Rewrite the names in the entries of *stream* still to run, keeping
|
|
every entry id, the group and its consumers, each pending entry's owner,
|
|
delivery count and idle time, and the stream's id counters.
|
|
|
|
An entry cannot be changed in place, and one added again gets a new id,
|
|
which is what runs, events and done markers are filed under, and loses its
|
|
delivery count. So the stream is copied into a temporary key with its ids
|
|
and swapped in with one RENAME, the only commit point: a rerun after a
|
|
crash on either side of it converges. Acknowledged entries are history and
|
|
copied as they are. Nothing may write to the stream meanwhile. Needs Redis
|
|
7.0 (docs/private/id_refac_rename.md, 3.1).
|
|
"""
|
|
tmp = f"{stream}:rename"
|
|
client.delete(tmp)
|
|
info = cast(dict[str, Any], client.xinfo_stream(stream))
|
|
groups = cast(list[dict[str, Any]], client.xinfo_groups(stream))
|
|
group = next(g for g in groups if g["name"] == GROUP)
|
|
last = str(group["last-delivered-id"])
|
|
pel: list[dict[str, Any]] = []
|
|
start = "-"
|
|
while rows := cast(
|
|
list[dict[str, Any]],
|
|
client.xpending_range(stream, GROUP, min=start, max="+", count=RENAME_PAGE),
|
|
):
|
|
pel += rows
|
|
start = f"({rows[-1]['message_id']}"
|
|
pending = {row["message_id"] for row in pel}
|
|
|
|
pipe = client.pipeline(transaction=False)
|
|
# MKSTREAM, so XSETID finds the key when there is nothing to copy.
|
|
pipe.xgroup_create(tmp, GROUP, id=last, mkstream=True)
|
|
for consumer in cast(list[dict[str, Any]], client.xinfo_consumers(stream, GROUP)):
|
|
pipe.xgroup_createconsumer(tmp, GROUP, consumer["name"])
|
|
copied: set[str] = set()
|
|
start = "-"
|
|
while entries := cast(
|
|
list[tuple[str, dict[str, str]]],
|
|
client.xrange(stream, min=start, max="+", count=RENAME_PAGE),
|
|
):
|
|
for entry_id, fields in entries:
|
|
live = entry_id in pending or _stream_id(entry_id) > _stream_id(last)
|
|
renamed = rename.fields(fields) if live else None
|
|
pipe.xadd(tmp, cast(Any, renamed or fields), id=entry_id)
|
|
if entry_id in pending:
|
|
copied.add(entry_id)
|
|
pipe.execute()
|
|
start = f"({entries[-1][0]}"
|
|
# redis-py has no XSETID of its own, and leaves the generic call untyped.
|
|
cast(Any, pipe).execute_command(
|
|
"XSETID",
|
|
tmp,
|
|
info["last-generated-id"],
|
|
"ENTRIESADDED",
|
|
info["entries-added"],
|
|
"MAXDELETEDID",
|
|
info["max-deleted-entry-id"],
|
|
)
|
|
# Redis caps ENTRIESREAD at the stream's entries-added counter, which only
|
|
# XSETID has set. Unknown stays unknown.
|
|
if group["entries-read"] is not None:
|
|
pipe.xgroup_setid(tmp, GROUP, last, entries_read=group["entries-read"])
|
|
# A pending entry whose stream entry was trimmed is not recreated, as
|
|
# `_reclaim` drops it too. XCLAIM would still create its consumer.
|
|
for row in pel:
|
|
if row["message_id"] in copied:
|
|
pipe.xclaim(
|
|
tmp,
|
|
GROUP,
|
|
row["consumer"],
|
|
0,
|
|
[row["message_id"]],
|
|
idle=row["time_since_delivered"],
|
|
retrycount=row["times_delivered"],
|
|
force=True,
|
|
justid=True,
|
|
)
|
|
pipe.execute()
|
|
# On its own: a pipeline without a transaction runs every command even
|
|
# after one of them failed.
|
|
client.rename(tmp, stream)
|
|
|
|
|
|
class WorkQueue(ABC):
|
|
"""What the execution service pulls from."""
|
|
|
|
#: Called after an item is journaled for later, so whoever is waiting on
|
|
#: the next deadline can find out it just moved closer. Set by the
|
|
#: execution service; a queue nobody is timing leaves it alone.
|
|
on_delayed: Callable[[], None] | None = None
|
|
|
|
def _woken(self) -> None:
|
|
"""Say a deadline changed. Never lets a listener break the enqueue."""
|
|
wake = self.on_delayed
|
|
if wake is None:
|
|
return
|
|
try:
|
|
wake()
|
|
except Exception:
|
|
logger.exception("Delayed-work listener failed")
|
|
|
|
@abstractmethod
|
|
def add(self, item: WorkItem) -> None:
|
|
"""Journal an item for immediate execution."""
|
|
|
|
@abstractmethod
|
|
def add_delayed(self, item: WorkItem, not_before: float) -> None:
|
|
"""Journal an item that must not run before ``not_before``."""
|
|
|
|
@abstractmethod
|
|
def next_due(self) -> float | None:
|
|
"""When the soonest delayed item comes due, or None if there is none.
|
|
|
|
What lets the engine sleep until a timer rather than poll for it: a
|
|
fixed poll makes every delayed item late by up to its interval, which
|
|
a motor driven for a measured number of seconds notices.
|
|
"""
|
|
|
|
@abstractmethod
|
|
def claim(
|
|
self, count: int, block_ms: int, due_only: bool = False
|
|
) -> list[WorkItem]:
|
|
"""Take up to ``count`` items, waiting up to ``block_ms`` for one.
|
|
|
|
``due_only`` takes nothing but promoted timers. A saturated engine
|
|
uses it to let a motor's stop past the cascade limit while every slot
|
|
is held by a long node.
|
|
"""
|
|
|
|
@abstractmethod
|
|
def ack(self, item: WorkItem) -> None:
|
|
"""Mark an item done, so it is never redelivered."""
|
|
|
|
def touch(self, entry_ids: list[str]) -> None: # noqa: B027
|
|
"""Say these items are still being worked on, not abandoned.
|
|
|
|
A node with no timeout may run far longer than the reclaim window, and
|
|
nothing else distinguishes that from an engine that died holding the
|
|
item. Concrete rather than abstract: a queue with no redelivery has
|
|
nothing to answer here.
|
|
"""
|
|
|
|
@abstractmethod
|
|
def reclaim_stale(self, min_idle_ms: int) -> list[WorkItem]:
|
|
"""Take back items claimed by a consumer that never acknowledged them."""
|
|
|
|
@abstractmethod
|
|
def move_due(self, now: float) -> int:
|
|
"""Promote delayed items whose time has come. Returns how many."""
|
|
|
|
@abstractmethod
|
|
def dead_letter(self, item: WorkItem, reason: str) -> None:
|
|
"""Set an item aside after it has failed too often."""
|
|
|
|
@abstractmethod
|
|
def park(self, flow: str, item: WorkItem) -> None:
|
|
"""Hold an item while its flow is paused."""
|
|
|
|
@abstractmethod
|
|
def unpark(self, flow: str) -> list[WorkItem]:
|
|
"""Return a paused flow's held items to the queue, oldest first."""
|
|
|
|
@abstractmethod
|
|
def unpark_one(self, flow: str) -> WorkItem | None:
|
|
"""Take the oldest held item, leaving the rest parked. That is a step."""
|
|
|
|
@abstractmethod
|
|
def clear_flow(self, flow: str) -> None:
|
|
"""Drop the work of a flow that no longer exists (`owned_by`).
|
|
|
|
What it parked, what waits on a timer, and what is queued or claimed
|
|
and not yet acknowledged, so a new flow under the same key never runs
|
|
it. A cascade already running finishes; its acknowledgement then has
|
|
nothing left to remove.
|
|
"""
|
|
|
|
@abstractmethod
|
|
def live_items(self) -> list[WorkItem]:
|
|
"""Every item still to run: queued, claimed, delayed or parked.
|
|
|
|
Not what was acknowledged, dead-lettered or marked done, which is
|
|
history. Reads the whole queue, so it is for rare questions.
|
|
"""
|
|
|
|
@abstractmethod
|
|
def stats(self) -> dict[str, Any]:
|
|
"""Backlog, in-flight, delayed and parked counts plus the oldest
|
|
pending age, for the health endpoint."""
|
|
|
|
def backlog(self) -> int:
|
|
"""How much work is waiting to be claimed.
|
|
|
|
Distinct from ``pending``, which is what has been handed to a consumer
|
|
and not yet acknowledged — work in progress. An engine hours behind has
|
|
a small ``pending`` and a large backlog, which is why one is not the
|
|
other. Concrete rather than abstract so the degradation watcher can ask
|
|
any queue.
|
|
"""
|
|
return 0
|
|
|
|
def backlog_flows(self, sample: int = 100) -> dict[str, int]:
|
|
"""Which flows the waiting work belongs to, as far as can be sampled."""
|
|
return {}
|
|
|
|
@abstractmethod
|
|
def dead_letters(self, count: int = 50) -> list[dict[str, Any]]:
|
|
"""What was given up on, newest first."""
|
|
|
|
@abstractmethod
|
|
def mark_done(self, entry_id: str, node: str) -> None:
|
|
"""Record that a side effect already happened for this delivery."""
|
|
|
|
@abstractmethod
|
|
def was_done(self, entry_id: str, node: str) -> bool:
|
|
"""Did this exact item already run that side-effecting node?"""
|
|
|
|
@abstractmethod
|
|
def close(self) -> None:
|
|
"""Release whatever the queue holds."""
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Renaming a flow key
|
|
# -------------------------------------------------------------------------
|
|
|
|
@abstractmethod
|
|
def hold(self) -> None:
|
|
"""Divert `add` into a held list until `release`, so a rename can
|
|
rewrite the queue while listeners keep journaling work."""
|
|
|
|
@abstractmethod
|
|
def release(self) -> None:
|
|
"""Move the held work into the backlog, oldest first and behind
|
|
everything already there, and stop diverting."""
|
|
|
|
@property
|
|
@abstractmethod
|
|
def held(self) -> bool:
|
|
"""Whether `add` is diverted."""
|
|
|
|
@abstractmethod
|
|
def release_held(self) -> int:
|
|
"""Release a held list left by a process that stopped mid-rename. For
|
|
startup, before anything claims. Returns how many items moved."""
|
|
|
|
@abstractmethod
|
|
def rename(
|
|
self, rename: Rename, progress: Callable[[str], None] | None = None
|
|
) -> None:
|
|
"""Move the names in the work still to run from ``rename.old`` to
|
|
``rename.new`` (`Rename.fields`).
|
|
|
|
Queued, claimed, delayed, parked and held items, and the done markers
|
|
that keep a replayed side effect from repeating. Entry ids, order,
|
|
deadlines, delivery counts, owners and guards stay as they are. Dead
|
|
letters and acknowledged entries are history and stay too. A rerun
|
|
after a crash completes the job. Nothing may claim, promote, reclaim,
|
|
park or unpark meanwhile.
|
|
|
|
:param progress: Called with the name of each structure once it is done.
|
|
"""
|
|
|
|
@abstractmethod
|
|
def count(self, rename: Rename) -> dict[str, int]:
|
|
"""How many entries `rename` would rewrite: ``queue_items`` (queued,
|
|
claimed or held), ``delayed``, ``parked`` and ``done_markers``."""
|
|
|
|
|
|
class MemoryWorkQueue(WorkQueue):
|
|
"""In-process queue. No durability: a crash loses whatever is in flight."""
|
|
|
|
def __init__(self) -> None:
|
|
self._items: deque[WorkItem] = deque()
|
|
# Promoted timers, kept apart from the backlog the way Redis keeps
|
|
# two streams — so they can be claimed on their own.
|
|
self._due: deque[WorkItem] = deque()
|
|
self._delayed: list[tuple[float, int, WorkItem]] = []
|
|
self._parked: dict[str, list[WorkItem]] = {}
|
|
# Insertion-ordered and capped, because this grows one entry per
|
|
# non-idempotent node per item and nothing ever removed one — the
|
|
# Redis side expires its markers after an hour, this one leaked for
|
|
# the life of the process. Redelivery is what the marker guards, and
|
|
# that happens within seconds of the claim.
|
|
self._done: OrderedDict[tuple[str, str], None] = OrderedDict()
|
|
# Claimed and not yet acknowledged, which is what Redis's `pending` is.
|
|
self._in_flight = 0
|
|
# What was added while a rename held the queue (`hold`).
|
|
self._held: deque[WorkItem] = deque()
|
|
self._holding = False
|
|
self._counter = 0
|
|
self._seq = 0
|
|
# The ids become `FlowRun.id`, a primary key. Counting from zero alone,
|
|
# a restart would overwrite the previous boot's rows.
|
|
self._boot = secrets.token_hex(4)
|
|
self._lock = threading.Lock()
|
|
self._wake = threading.Condition(self._lock)
|
|
|
|
def _next_id(self) -> str:
|
|
self._seq += 1
|
|
return f"mem-{self._boot}-{self._seq}"
|
|
|
|
def add(self, item: WorkItem) -> None:
|
|
with self._wake:
|
|
if not item.entry_id:
|
|
item.entry_id = self._next_id()
|
|
if self._holding:
|
|
self._held.append(item)
|
|
return
|
|
self._items.append(item)
|
|
self._wake.notify()
|
|
|
|
def add_delayed(self, item: WorkItem, not_before: float) -> None:
|
|
with self._wake:
|
|
if not item.entry_id:
|
|
item.entry_id = self._next_id()
|
|
item.not_before = not_before
|
|
self._counter += 1
|
|
heapq.heappush(self._delayed, (not_before, self._counter, item))
|
|
self._wake.notify()
|
|
# Outside the lock: the listener is whoever is asleep until the old
|
|
# deadline, and it comes straight back here to read the new one.
|
|
self._woken()
|
|
|
|
def next_due(self) -> float | None:
|
|
with self._lock:
|
|
return self._delayed[0][0] if self._delayed else None
|
|
|
|
def claim(
|
|
self, count: int, block_ms: int, due_only: bool = False
|
|
) -> list[WorkItem]:
|
|
deadline = time.monotonic() + block_ms / 1000.0
|
|
with self._wake:
|
|
while not self._due and not (self._items and not due_only):
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
return []
|
|
self._wake.wait(remaining)
|
|
# Due first: an item that has waited out a deadline is late by
|
|
# however long it queues here, while work merely enqueued is not
|
|
# waiting on a clock.
|
|
claimed = [self._due.popleft() for _ in range(min(count, len(self._due)))]
|
|
if not due_only:
|
|
claimed += [
|
|
self._items.popleft()
|
|
for _ in range(min(count - len(claimed), len(self._items)))
|
|
]
|
|
self._in_flight += len(claimed)
|
|
return claimed
|
|
|
|
def ack(self, item: WorkItem) -> None:
|
|
"""Nothing to take off the queue — claiming did that. Only counted."""
|
|
with self._lock:
|
|
self._in_flight = max(0, self._in_flight - 1)
|
|
|
|
def reclaim_stale(self, min_idle_ms: int) -> list[WorkItem]:
|
|
"""Nothing to reclaim: an item lost here died with the process."""
|
|
return []
|
|
|
|
def move_due(self, now: float) -> int:
|
|
due: list[WorkItem] = []
|
|
with self._wake:
|
|
while self._delayed and self._delayed[0][0] <= now:
|
|
due.append(heapq.heappop(self._delayed)[2])
|
|
if due:
|
|
self._due.extend(due)
|
|
self._wake.notify()
|
|
return len(due)
|
|
|
|
def dead_letter(self, item: WorkItem, reason: str) -> None:
|
|
logger.error("Dropping work item for '%s': %s", item.node, reason)
|
|
|
|
def park(self, flow: str, item: WorkItem) -> None:
|
|
with self._lock:
|
|
self._parked.setdefault(flow, []).append(item)
|
|
|
|
def unpark(self, flow: str) -> list[WorkItem]:
|
|
with self._lock:
|
|
return self._parked.pop(flow, [])
|
|
|
|
def unpark_one(self, flow: str) -> WorkItem | None:
|
|
with self._lock:
|
|
held = self._parked.get(flow)
|
|
return held.pop(0) if held else None
|
|
|
|
def clear_flow(self, flow: str) -> None:
|
|
with self._lock:
|
|
self._parked.pop(flow, None)
|
|
for lane in (self._items, self._due):
|
|
kept = [item for item in lane if not owned_by(item, flow)]
|
|
lane.clear()
|
|
lane.extend(kept)
|
|
self._delayed = [e for e in self._delayed if not owned_by(e[2], flow)]
|
|
heapq.heapify(self._delayed)
|
|
|
|
def live_items(self) -> list[WorkItem]:
|
|
"""What is waiting. A claimed item is gone from here once it is handed
|
|
out, so unlike Redis this cannot say what is running."""
|
|
with self._lock:
|
|
return [
|
|
*self._items,
|
|
*self._due,
|
|
*(entry[2] for entry in self._delayed),
|
|
*(item for held in self._parked.values() for item in held),
|
|
*self._held,
|
|
]
|
|
|
|
def backlog(self) -> int:
|
|
with self._lock:
|
|
return len(self._items) + len(self._due)
|
|
|
|
def stats(self) -> dict[str, Any]:
|
|
with self._lock:
|
|
return {
|
|
"backlog": len(self._items) + len(self._due),
|
|
"pending": self._in_flight,
|
|
"delayed": len(self._delayed),
|
|
"parked": sum(len(v) for v in self._parked.values()),
|
|
"oldest_pending_s": 0.0,
|
|
"durable": False,
|
|
}
|
|
|
|
def dead_letters(self, count: int = 50) -> list[dict[str, Any]]:
|
|
"""Nothing is kept: a dropped item here died with the process."""
|
|
return []
|
|
|
|
def mark_done(self, entry_id: str, node: str) -> None:
|
|
with self._lock:
|
|
self._done[(entry_id, node)] = None
|
|
while len(self._done) > DONE_MARKERS:
|
|
self._done.popitem(last=False)
|
|
|
|
def was_done(self, entry_id: str, node: str) -> bool:
|
|
with self._lock:
|
|
return (entry_id, node) in self._done
|
|
|
|
def close(self) -> None:
|
|
"""Nothing is held outside the process."""
|
|
|
|
def hold(self) -> None:
|
|
with self._lock:
|
|
self._holding = True
|
|
|
|
def release(self) -> None:
|
|
with self._wake:
|
|
self._holding = False
|
|
self._items.extend(self._held)
|
|
self._held.clear()
|
|
self._wake.notify()
|
|
|
|
@property
|
|
def held(self) -> bool:
|
|
return self._holding
|
|
|
|
def release_held(self) -> int:
|
|
"""Nothing held here outlives the process."""
|
|
return 0
|
|
|
|
def rename(
|
|
self, rename: Rename, progress: Callable[[str], None] | None = None
|
|
) -> None:
|
|
"""One pass under the lock. Timers keep their place in the heap,
|
|
which orders them by deadline and arrival only."""
|
|
|
|
def moved(item: WorkItem) -> WorkItem:
|
|
return rename.item(item) or item
|
|
|
|
with self._lock:
|
|
if rename.old in self._parked and rename.new in self._parked:
|
|
raise RuntimeError(f"Work is parked under '{rename.new}' already")
|
|
for lane in (self._items, self._due, self._held):
|
|
items = [moved(item) for item in lane]
|
|
lane.clear()
|
|
lane.extend(items)
|
|
self._delayed = [(at, n, moved(item)) for at, n, item in self._delayed]
|
|
self._parked = {
|
|
rename.flow(flow): [moved(item) for item in items]
|
|
for flow, items in self._parked.items()
|
|
}
|
|
self._done = OrderedDict(
|
|
((entry_id, rename.name(node)), None) for entry_id, node in self._done
|
|
)
|
|
if progress is not None:
|
|
progress("queue")
|
|
|
|
def count(self, rename: Rename) -> dict[str, int]:
|
|
"""What is claimed is gone from here, so it is not counted."""
|
|
|
|
def changed(items: Iterable[WorkItem]) -> int:
|
|
return sum(rename.item(item) is not None for item in items)
|
|
|
|
with self._lock:
|
|
return {
|
|
"queue_items": changed([*self._items, *self._due, *self._held]),
|
|
"delayed": changed(entry[2] for entry in self._delayed),
|
|
"parked": changed(i for held in self._parked.values() for i in held),
|
|
"done_markers": sum(rename.name(n) != n for _, n in self._done),
|
|
}
|
|
|
|
|
|
class RedisWorkQueue(WorkQueue):
|
|
"""Redis Streams queue: the one that survives the process.
|
|
|
|
Items live in a stream read through a consumer group, so an item handed to
|
|
a consumer that dies before acknowledging it stays pending and is reclaimed
|
|
by whoever comes next.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
host: str,
|
|
port: int = 6379,
|
|
# A root of its own: the state backend owns `pipeline:`, and sharing the
|
|
# prefix meant `RedisState.clear()` could DEL the queue stream.
|
|
namespace: str = "queue",
|
|
consumer: str | None = None,
|
|
) -> None:
|
|
self._redis = redis.Redis(
|
|
host=host,
|
|
port=port,
|
|
decode_responses=True,
|
|
# Longer than any block this asks for, so a claim still waits its
|
|
# second — but bounded, so a Redis that goes silent without
|
|
# closing the socket fails rather than hanging the consumer.
|
|
socket_timeout=SOCKET_TIMEOUT_S,
|
|
socket_connect_timeout=CONNECT_TIMEOUT_S,
|
|
health_check_interval=30,
|
|
)
|
|
self._ns = namespace
|
|
self._consumer = consumer or f"engine-{int(time.time() * 1000) % 1_000_000}"
|
|
self._stream = f"{namespace}:__queue__"
|
|
# Promoted timers, read before the backlog. A stream of its own rather
|
|
# than a flag on the entry, because a stream is ordered by id and there
|
|
# is no way to put a late item in front of one already in it.
|
|
self._due_stream = f"{namespace}:__due__"
|
|
self._delayed_key = f"{namespace}:__delayed__"
|
|
self._dead_key = f"{namespace}:__dead__"
|
|
# Which flows have something parked. Maintained rather than discovered:
|
|
# finding them with a keyspace scan walked every state and idempotency
|
|
# key in the database, twice per health poll.
|
|
self._parked_flows_key = f"{namespace}:__parked_flows__"
|
|
# What was added while a rename held the queue (`hold`).
|
|
self._held_key = f"{namespace}:__held__"
|
|
self._move_due = self._redis.register_script(_MOVE_DUE)
|
|
self._release = self._redis.register_script(_RELEASE)
|
|
# `add` reads the flag under this, and counts an `XADD` in `_adding`
|
|
# until it is done, so adds from parallel producers do not wait for
|
|
# each other. `hold` sets the flag and waits for `_adding` to reach 0,
|
|
# and `release` empties the held list and clears the flag under it. So
|
|
# nothing added after `hold` lands in a stream a rename is rebuilding,
|
|
# and nothing added while held overtakes held work.
|
|
self._hold_lock = threading.Condition()
|
|
self._holding = False
|
|
self._adding = 0
|
|
self._ensure_group()
|
|
self._drop_stale_consumers()
|
|
|
|
def _ensure_group(self) -> None:
|
|
for stream in (self._stream, self._due_stream):
|
|
try:
|
|
self._redis.xgroup_create(stream, GROUP, id="0", mkstream=True)
|
|
except redis.ResponseError as exc:
|
|
if "BUSYGROUP" not in str(exc):
|
|
raise
|
|
|
|
def _drop_stale_consumers(self) -> None:
|
|
"""Forget the consumers of engines that are gone.
|
|
|
|
One is registered per engine start and none was ever removed, so the
|
|
list grew without bound — and `XPENDING`, `XAUTOCLAIM` and `XINFO` all
|
|
walk it. Only consumers holding nothing and untouched for an hour are
|
|
dropped, so a live engine blocked on a read is never one of them; a
|
|
consumer is recreated by its next read anyway.
|
|
"""
|
|
for stream in (self._stream, self._due_stream):
|
|
try:
|
|
consumers = cast(
|
|
list[dict[str, Any]], self._redis.xinfo_consumers(stream, GROUP)
|
|
)
|
|
except redis.ResponseError:
|
|
continue
|
|
for row in consumers:
|
|
name = str(row.get("name", ""))
|
|
if name == self._consumer or row.get("pending"):
|
|
continue
|
|
if int(row.get("idle") or 0) < STALE_CONSUMER_IDLE_MS:
|
|
continue
|
|
try:
|
|
self._redis.xgroup_delconsumer(stream, GROUP, name)
|
|
except redis.ResponseError:
|
|
pass
|
|
|
|
def _parked_key(self, flow: str) -> str:
|
|
return f"{self._ns}:__parked__:{flow}"
|
|
|
|
def _done_key(self, entry_id: str, node: str) -> str:
|
|
return f"{self._ns}:{DONE}:{entry_id}:{node}"
|
|
|
|
def add(self, item: WorkItem) -> None:
|
|
with self._hold_lock:
|
|
if self._holding:
|
|
self._redis.rpush(self._held_key, orjson.dumps(item.to_fields()))
|
|
return
|
|
self._adding += 1
|
|
try:
|
|
self._redis.xadd(
|
|
self._stream,
|
|
cast(Any, item.to_fields()),
|
|
maxlen=STREAM_MAXLEN,
|
|
approximate=True,
|
|
)
|
|
finally:
|
|
with self._hold_lock:
|
|
self._adding -= 1
|
|
self._hold_lock.notify_all()
|
|
|
|
def add_delayed(self, item: WorkItem, not_before: float) -> None:
|
|
item.not_before = not_before
|
|
self._redis.zadd(
|
|
self._delayed_key, {orjson.dumps(item.to_fields()): not_before}
|
|
)
|
|
self._woken()
|
|
|
|
def next_due(self) -> float | None:
|
|
soonest = cast(
|
|
list[tuple[str, float]],
|
|
self._redis.zrange(self._delayed_key, 0, 0, withscores=True),
|
|
)
|
|
return soonest[0][1] if soonest else None
|
|
|
|
def _route(self, entry_id: str) -> tuple[str, str]:
|
|
"""The stream an entry id belongs to, and its plain id there."""
|
|
if entry_id.startswith(DUE_PREFIX):
|
|
return self._due_stream, entry_id[len(DUE_PREFIX) :]
|
|
return self._stream, entry_id
|
|
|
|
def claim(
|
|
self, count: int, block_ms: int, due_only: bool = False
|
|
) -> list[WorkItem]:
|
|
"""Take up to ``count`` items, due timers before anything queued.
|
|
|
|
One read over both streams rather than a read each: the block has to
|
|
cover the due lane, or an item promoted while this waits sits there
|
|
until the block expires — which is the second of lateness the sleeping
|
|
timer thread exists to remove. ``COUNT`` is per stream, so a read may
|
|
return up to twice what was asked for; the excess is bounded by the
|
|
cascade pool's own free count, and starting a due timer is worth more
|
|
than holding that number exactly.
|
|
"""
|
|
response = cast(
|
|
list[Any],
|
|
self._redis.xreadgroup(
|
|
GROUP,
|
|
self._consumer,
|
|
(
|
|
{self._due_stream: ">"}
|
|
if due_only
|
|
else {self._due_stream: ">", self._stream: ">"}
|
|
),
|
|
count=count,
|
|
block=block_ms,
|
|
),
|
|
)
|
|
due: list[WorkItem] = []
|
|
backlog: list[WorkItem] = []
|
|
for stream, entries in response or []:
|
|
is_due = stream == self._due_stream
|
|
into = due if is_due else backlog
|
|
prefix = DUE_PREFIX if is_due else ""
|
|
for entry_id, fields in entries:
|
|
into.append(WorkItem.from_fields(fields, prefix + entry_id))
|
|
return due + backlog
|
|
|
|
def ack(self, item: WorkItem) -> None:
|
|
if item.entry_id:
|
|
stream, entry_id = self._route(item.entry_id)
|
|
self._redis.xack(stream, GROUP, entry_id)
|
|
|
|
def touch(self, entry_ids: list[str]) -> None:
|
|
if not entry_ids:
|
|
return
|
|
# Claiming an entry we already hold resets how long it has been idle,
|
|
# which is the only thing `reclaim_stale` reads. `justid` keeps the
|
|
# delivery count where it is, so a long node neither redelivers nor
|
|
# spends its way towards the dead-letter cap. An entry already
|
|
# acknowledged is simply not there, and this is a no-op for it.
|
|
by_stream: dict[str, list[str]] = {}
|
|
for entry_id in entry_ids:
|
|
stream, plain = self._route(entry_id)
|
|
by_stream.setdefault(stream, []).append(plain)
|
|
for stream, ids in by_stream.items():
|
|
self._redis.xclaim(
|
|
stream,
|
|
GROUP,
|
|
self._consumer,
|
|
min_idle_time=0,
|
|
# redis-py types `message_ids` as an invariant list of every id
|
|
# shape it accepts, which a list[str] is not assignable to.
|
|
message_ids=cast("list[int | bytes | str | memoryview]", ids),
|
|
justid=True,
|
|
)
|
|
|
|
def reclaim_stale(self, min_idle_ms: int) -> list[WorkItem]:
|
|
"""Take over entries a dead consumer never acknowledged."""
|
|
return self._reclaim(self._due_stream, DUE_PREFIX, min_idle_ms) + self._reclaim(
|
|
self._stream, "", min_idle_ms
|
|
)
|
|
|
|
def _reclaim(self, stream: str, prefix: str, min_idle_ms: int) -> list[WorkItem]:
|
|
_cursor, entries, _deleted = cast(
|
|
tuple[Any, list[Any], Any],
|
|
self._redis.xautoclaim(
|
|
stream,
|
|
GROUP,
|
|
self._consumer,
|
|
min_idle_time=min_idle_ms,
|
|
count=32,
|
|
),
|
|
)
|
|
if not entries:
|
|
return []
|
|
|
|
# xautoclaim does not report delivery counts, so ask xpending for them.
|
|
counts: dict[str, int] = {}
|
|
for record in cast(
|
|
list[dict[str, Any]],
|
|
self._redis.xpending_range(stream, GROUP, min="-", max="+", count=64),
|
|
):
|
|
counts[record["message_id"]] = record["times_delivered"]
|
|
|
|
return [
|
|
WorkItem.from_fields(fields, prefix + entry_id, counts.get(entry_id, 1))
|
|
for entry_id, fields in entries
|
|
if fields
|
|
]
|
|
|
|
def move_due(self, now: float) -> int:
|
|
"""Move up to 100 timers into the due lane atomically."""
|
|
return cast(
|
|
int,
|
|
self._move_due(
|
|
keys=[self._delayed_key, self._due_stream],
|
|
args=[now, STREAM_MAXLEN],
|
|
),
|
|
)
|
|
|
|
def dead_letter(self, item: WorkItem, reason: str) -> None:
|
|
fields = item.to_fields()
|
|
fields["reason"] = reason
|
|
self._redis.xadd(
|
|
self._dead_key, cast(Any, fields), maxlen=1000, approximate=True
|
|
)
|
|
logger.error("Dead-lettered work item for '%s': %s", item.node, reason)
|
|
|
|
def park(self, flow: str, item: WorkItem) -> None:
|
|
pipe = self._redis.pipeline()
|
|
pipe.rpush(self._parked_key(flow), orjson.dumps(item.to_fields()))
|
|
pipe.sadd(self._parked_flows_key, flow)
|
|
pipe.execute()
|
|
|
|
def unpark(self, flow: str) -> list[WorkItem]:
|
|
key = self._parked_key(flow)
|
|
raw = cast(list[str], self._redis.lrange(key, 0, -1))
|
|
pipe = self._redis.pipeline()
|
|
pipe.delete(key)
|
|
pipe.srem(self._parked_flows_key, flow)
|
|
pipe.execute()
|
|
return [WorkItem.from_fields(orjson.loads(r), "") for r in raw]
|
|
|
|
def unpark_one(self, flow: str) -> WorkItem | None:
|
|
key = self._parked_key(flow)
|
|
pipe = self._redis.pipeline()
|
|
pipe.lpop(key)
|
|
pipe.llen(key)
|
|
raw, remaining = cast(tuple["str | None", int], pipe.execute())
|
|
if not remaining:
|
|
self._redis.srem(self._parked_flows_key, flow)
|
|
return WorkItem.from_fields(orjson.loads(raw), "") if raw else None
|
|
|
|
def clear_flow(self, flow: str) -> None:
|
|
pipe = self._redis.pipeline()
|
|
pipe.delete(self._parked_key(flow))
|
|
pipe.srem(self._parked_flows_key, flow)
|
|
# Timers before the streams: one promoted in between is in the due
|
|
# stream by the time that is read.
|
|
for raw in cast(list[str], self._redis.zrange(self._delayed_key, 0, -1)):
|
|
if owned_by(WorkItem.from_fields(orjson.loads(raw), ""), flow):
|
|
pipe.zrem(self._delayed_key, raw)
|
|
for stream in (self._stream, self._due_stream):
|
|
ids = [
|
|
entry_id
|
|
for entry_id, fields in self._live_entries(stream)
|
|
if owned_by(WorkItem.from_fields(fields, entry_id), flow)
|
|
]
|
|
if ids:
|
|
pipe.xack(stream, GROUP, *ids)
|
|
pipe.xdel(stream, *ids)
|
|
pipe.execute()
|
|
|
|
def _live_entries(self, stream: str) -> list[tuple[str, dict[str, str]]]:
|
|
"""The entries of *stream* still to run: claimed and unacknowledged,
|
|
then never handed out. Acknowledged ones stay in a stream until it is
|
|
trimmed, and are history."""
|
|
group = self._group_info(stream)
|
|
if not group:
|
|
return []
|
|
last = str(group.get("last-delivered-id") or "0-0")
|
|
live: list[tuple[str, dict[str, str]]] = []
|
|
pending = int(group.get("pending") or 0)
|
|
if pending:
|
|
rows = cast(
|
|
list[dict[str, Any]],
|
|
self._redis.xpending_range(
|
|
stream, GROUP, min="-", max="+", count=pending
|
|
),
|
|
)
|
|
ids = {row["message_id"] for row in rows}
|
|
if ids:
|
|
# In id order, so the first is the oldest still held.
|
|
entries = self._redis.xrange(
|
|
stream, min=rows[0]["message_id"], max=last
|
|
)
|
|
live += [(i, f) for i, f in cast(Any, entries) if i in ids]
|
|
live += cast(Any, self._redis.xrange(stream, min=f"({last}", max="+"))
|
|
return live
|
|
|
|
def live_items(self) -> list[WorkItem]:
|
|
items = [
|
|
WorkItem.from_fields(fields, prefix + entry_id)
|
|
for stream, prefix in ((self._due_stream, DUE_PREFIX), (self._stream, ""))
|
|
for entry_id, fields in self._live_entries(stream)
|
|
]
|
|
items += [
|
|
WorkItem.from_fields(orjson.loads(raw), "")
|
|
for raw in cast(list[str], self._redis.zrange(self._delayed_key, 0, -1))
|
|
]
|
|
for flow in cast(set[str], self._redis.smembers(self._parked_flows_key)):
|
|
items += [
|
|
WorkItem.from_fields(orjson.loads(raw), "")
|
|
for raw in cast(
|
|
list[str], self._redis.lrange(self._parked_key(flow), 0, -1)
|
|
)
|
|
]
|
|
items += [
|
|
WorkItem.from_fields(orjson.loads(raw), "")
|
|
for raw in cast(list[str], self._redis.lrange(self._held_key, 0, -1))
|
|
]
|
|
return items
|
|
|
|
def _group_info(self, stream: str = "") -> dict[str, Any]:
|
|
"""This consumer group's row of ``XINFO GROUPS``, or an empty one."""
|
|
try:
|
|
groups = cast(
|
|
list[dict[str, Any]], self._redis.xinfo_groups(stream or self._stream)
|
|
)
|
|
except redis.ResponseError:
|
|
# No stream yet: nothing has ever been enqueued.
|
|
return {}
|
|
return next((g for g in groups if g.get("name") == GROUP), {})
|
|
|
|
def backlog(self) -> int:
|
|
"""Entries in either stream this group has never been handed.
|
|
|
|
Redis calls it the group's ``lag``. It is nil rather than zero when the
|
|
stream has been trimmed under the group — entries that were dropped
|
|
before anyone read them — and unknown is reported as none waiting,
|
|
since the alternative is a health screen crying wolf after a trim.
|
|
"""
|
|
return sum(
|
|
int(self._group_info(stream).get("lag") or 0)
|
|
for stream in (self._stream, self._due_stream)
|
|
)
|
|
|
|
def backlog_flows(self, sample: int = 100) -> dict[str, int]:
|
|
"""Which flows the waiting work belongs to, from a sample of the tail.
|
|
|
|
The group's lag is one number for the whole stream, and the actionable
|
|
half of "the engine is behind" is always *what* is producing the work.
|
|
Reads forward from the last entry the group was handed, which is where
|
|
the undelivered entries start.
|
|
"""
|
|
after = self._group_info().get("last-delivered-id")
|
|
if not after:
|
|
return {}
|
|
entries = cast(
|
|
list[tuple[str, dict[str, str]]],
|
|
self._redis.xrange(self._stream, min=f"({after}", max="+", count=sample),
|
|
)
|
|
counts: dict[str, int] = {}
|
|
for _entry_id, fields in entries:
|
|
flow = fields.get("flow", "")
|
|
if flow:
|
|
counts[flow] = counts.get(flow, 0) + 1
|
|
return counts
|
|
|
|
def stats(self) -> dict[str, Any]:
|
|
count = 0
|
|
oldest = 0.0
|
|
for stream in (self._stream, self._due_stream):
|
|
pending = cast(dict[str, Any], self._redis.xpending(stream, GROUP))
|
|
waiting = pending.get("pending", 0) if isinstance(pending, dict) else 0
|
|
count += waiting
|
|
if not waiting:
|
|
continue
|
|
records = cast(
|
|
list[dict[str, Any]],
|
|
self._redis.xpending_range(stream, GROUP, min="-", max="+", count=1),
|
|
)
|
|
if records:
|
|
oldest = max(oldest, records[0]["time_since_delivered"] / 1000.0)
|
|
flows = cast(set[str], self._redis.smembers(self._parked_flows_key))
|
|
parked = sum(cast(int, self._redis.llen(self._parked_key(f))) for f in flows)
|
|
return {
|
|
"backlog": self.backlog(),
|
|
"pending": count,
|
|
"delayed": cast(int, self._redis.zcard(self._delayed_key)),
|
|
"parked": parked,
|
|
"oldest_pending_s": round(oldest, 1),
|
|
"durable": True,
|
|
}
|
|
|
|
def dead_letters(self, count: int = 50) -> list[dict[str, Any]]:
|
|
"""What was set aside, newest first. The entry id carries the time."""
|
|
entries = cast(
|
|
list[tuple[str, dict[str, str]]],
|
|
self._redis.xrevrange(self._dead_key, count=count),
|
|
)
|
|
return [
|
|
{
|
|
"id": entry_id,
|
|
"ts": int(entry_id.split("-")[0]) / 1000.0,
|
|
"flow": fields.get("flow", ""),
|
|
"node": fields.get("node", ""),
|
|
"cause": fields.get("cause", ""),
|
|
"reason": fields.get("reason", ""),
|
|
}
|
|
for entry_id, fields in entries
|
|
]
|
|
|
|
def mark_done(self, entry_id: str, node: str) -> None:
|
|
# An hour outlives any redelivery; after that the marker is noise.
|
|
self._redis.set(self._done_key(entry_id, node), "1", ex=3600)
|
|
|
|
def was_done(self, entry_id: str, node: str) -> bool:
|
|
return bool(self._redis.exists(self._done_key(entry_id, node)))
|
|
|
|
def close(self) -> None:
|
|
self._redis.close()
|
|
|
|
def hold(self) -> None:
|
|
with self._hold_lock:
|
|
self._holding = True
|
|
while self._adding:
|
|
self._hold_lock.wait()
|
|
|
|
def release(self) -> None:
|
|
with self._hold_lock:
|
|
self.release_held()
|
|
self._holding = False
|
|
|
|
@property
|
|
def held(self) -> bool:
|
|
return self._holding
|
|
|
|
def release_held(self) -> int:
|
|
released = 0
|
|
while moved := cast(
|
|
int,
|
|
self._release(keys=[self._held_key, self._stream], args=[STREAM_MAXLEN]),
|
|
):
|
|
released += moved
|
|
return released
|
|
|
|
def rename(
|
|
self, rename: Rename, progress: Callable[[str], None] | None = None
|
|
) -> None:
|
|
"""Each structure as section 3 of docs/private/id_refac_rename.md has
|
|
it: the streams rebuilt, then timers, lists and done markers, each
|
|
rewritten one member, slot or key at a time."""
|
|
|
|
def done(structure: str) -> None:
|
|
if progress is not None:
|
|
progress(structure)
|
|
|
|
for stream, lane in ((self._due_stream, "due"), (self._stream, "backlog")):
|
|
rebuild_stream(self._redis, stream, rename)
|
|
done(lane)
|
|
|
|
# One transaction, so each timer moves with its deadline. Promotion is
|
|
# held, and a timer added meanwhile is not touched.
|
|
pipe = self._redis.pipeline()
|
|
for raw, at in cast(
|
|
list[tuple[str, float]],
|
|
self._redis.zrange(self._delayed_key, 0, -1, withscores=True),
|
|
):
|
|
fields = rename.fields(orjson.loads(raw))
|
|
if fields is not None:
|
|
pipe.zrem(self._delayed_key, raw)
|
|
pipe.zadd(self._delayed_key, {orjson.dumps(fields): at})
|
|
pipe.execute()
|
|
done("delayed")
|
|
|
|
# Slot by slot: nothing pops while held, and `add` only appends, so
|
|
# the indices read hold.
|
|
for key in [*self._parked_keys(rename), self._held_key]:
|
|
pipe = self._redis.pipeline(transaction=False)
|
|
for i, raw in enumerate(cast(list[str], self._redis.lrange(key, 0, -1))):
|
|
fields = rename.fields(orjson.loads(raw))
|
|
if fields is not None:
|
|
pipe.lset(key, i, orjson.dumps(fields).decode())
|
|
pipe.execute()
|
|
old, new = self._parked_key(rename.old), self._parked_key(rename.new)
|
|
if self._redis.exists(old):
|
|
# Checked first: inside the transaction a refused RENAMENX would
|
|
# not stop the index from moving.
|
|
if self._redis.exists(new):
|
|
raise RuntimeError(f"Work is parked under '{rename.new}' already")
|
|
pipe = self._redis.pipeline()
|
|
pipe.renamenx(old, new)
|
|
pipe.srem(self._parked_flows_key, rename.old)
|
|
pipe.sadd(self._parked_flows_key, rename.new)
|
|
pipe.execute()
|
|
done("lists")
|
|
|
|
move_keys(self._redis, self._marker_moves(rename))
|
|
done("done markers")
|
|
|
|
def _parked_keys(self, rename: Rename) -> list[str]:
|
|
"""Every parked list, the renamed flow's under either name included."""
|
|
flows = cast(set[str], self._redis.smembers(self._parked_flows_key))
|
|
return list(
|
|
dict.fromkeys(
|
|
map(self._parked_key, [*sorted(flows), rename.old, rename.new])
|
|
)
|
|
)
|
|
|
|
def _marker_moves(self, rename: Rename) -> list[tuple[str, str]]:
|
|
moves = []
|
|
for key in self._redis.scan_iter(
|
|
match=f"{self._ns}:{DONE}:*", count=RENAME_PAGE
|
|
):
|
|
entry_id, node = done_marker(key)
|
|
if rename.name(node) != node:
|
|
moves.append((key, self._done_key(entry_id, rename.name(node))))
|
|
return moves
|
|
|
|
def count(self, rename: Rename) -> dict[str, int]:
|
|
def changed(raws: Iterable[str]) -> int:
|
|
return sum(rename.fields(orjson.loads(raw)) is not None for raw in raws)
|
|
|
|
def listed(key: str) -> list[str]:
|
|
return cast(list[str], self._redis.lrange(key, 0, -1))
|
|
|
|
live = [
|
|
fields
|
|
for stream in (self._due_stream, self._stream)
|
|
for _, fields in self._live_entries(stream)
|
|
]
|
|
return {
|
|
"queue_items": sum(rename.fields(f) is not None for f in live)
|
|
+ changed(listed(self._held_key)),
|
|
"delayed": changed(
|
|
cast(list[str], self._redis.zrange(self._delayed_key, 0, -1))
|
|
),
|
|
"parked": sum(changed(listed(key)) for key in self._parked_keys(rename)),
|
|
"done_markers": len(self._marker_moves(rename)),
|
|
}
|