Engine holds for a rename: the pipeline refuses publish and inline cascades with EngineHeld once those running have finished, the execution service stops claiming and its timers once claimed cascades have finished; both wait without a limit, report what is busy and can be aborted. REST answers 503 with Retry-After, a device frame busy. Redis adds no longer wait for each other, only a queue hold waits for them

This commit is contained in:
2026-11-02 20:21:17 +01:00
parent 46b57de4ec
commit badcf94491
10 changed files with 687 additions and 156 deletions
+175 -83
View File
@@ -17,7 +17,7 @@ from __future__ import annotations
import logging
import threading
import time
from collections import deque
from collections import Counter, deque
from collections.abc import Callable
from concurrent.futures import Executor, Future, ThreadPoolExecutor
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
@@ -26,6 +26,7 @@ from fluksio.flow.queue import MAX_DELIVERIES, WorkItem, WorkQueue
if TYPE_CHECKING:
from fluksio.flow.events import EventBus
from fluksio.flow.namespace import Rename
from fluksio.flow.pipeline import Pipeline
logger = logging.getLogger(__name__)
@@ -69,10 +70,41 @@ BACKLOG_INTERVAL_S = 5.0
BACKLOG_DEGRADED = 50
BACKLOG_STRIKES = 3
#: How often an engine hold looks at its abort while it waits.
HOLD_POLL_S = 0.5
_P = ParamSpec("_P")
_T = TypeVar("_T")
def wait_out(
lock: threading.Condition,
busy: Callable[[], list[str]],
abort: threading.Event,
report: Callable[[list[str]], None] | None = None,
) -> bool:
"""Wait until ``busy``, read under ``lock``, names nothing.
Without a limit, because an engine hold waits for as long as the work
takes. ``report`` hears each new set of names, so whoever asked can show
what it waits for, and setting ``abort`` gives up with False.
"""
said: list[str] = []
while True:
with lock:
names = busy()
while names and names == said and not abort.is_set():
lock.wait(HOLD_POLL_S)
names = busy()
if abort.is_set():
return False
if not names:
return True
said = names
if report is not None:
report(names)
class _PoolLane(Executor):
def __init__(self, pool: _ReservedPool, due: bool) -> None:
self._pool = pool
@@ -158,7 +190,8 @@ class ExecutionService:
queue.on_delayed = self._timer_wake.set
self._intake = threading.Event()
self._intake.set()
self._inflight = 0
# Cascades claimed and still running, by the node each came from.
self._running: Counter[str] = Counter()
self._inflight_lock = threading.Condition()
# Entry ids claimed and still running, under _inflight_lock.
self._active: set[str] = set()
@@ -174,6 +207,12 @@ class ExecutionService:
)
self._consumer: threading.Thread | None = None
self._timers: threading.Thread | None = None
# Held by the consumer for one pass of capacity check, claim and
# dispatch, and by the timer thread for one pass of promotion, touch
# and reclaim. `hold` takes both. Plain locks, because `release` may
# come from another thread than the one that held.
self._consumer_lock = threading.Lock()
self._timer_lock = threading.Lock()
# Consecutive backlog readings over the threshold, and whether the last
# of them said so out loud.
self._backlog_strikes = 0
@@ -225,11 +264,11 @@ class ExecutionService:
self._intake.clear()
deadline = time.monotonic() + DRAIN_TIMEOUT_S
with self._inflight_lock:
while self._inflight:
while self._running:
remaining = deadline - time.monotonic()
if remaining <= 0:
logger.warning(
"Rebuild did not wait out %d cascades", self._inflight
"Rebuild did not wait out %d cascades", self._running.total()
)
return
self._inflight_lock.wait(remaining)
@@ -237,13 +276,59 @@ class ExecutionService:
def resume_intake(self) -> None:
self._intake.set()
def hold(
self,
abort: threading.Event,
report: Callable[[list[str]], None] | None = None,
) -> bool:
"""Stop claiming, promoting, touching and reclaiming, once every
claimed cascade has finished. For a rename, which rewrites the queue.
Unlike `pause_intake` this also waits out a claim already blocking
when intake clears, and a timer pass. It waits without a limit and
tells ``report`` the nodes whose cascades are still running each time
they change. Setting ``abort`` gives up: claiming resumes and it
returns False. True means held until `release`.
"""
self._intake.clear()
# So a consumer waiting for capacity sees the intake closed.
self._capacity_changed()
self._consumer_lock.acquire()
# The timer lock only after the drain: while a cascade runs, the
# timer thread keeps touching it, so it is not reclaimed as dropped.
if not wait_out(self._inflight_lock, self._busy, abort, report):
self._consumer_lock.release()
self._intake.set()
return False
self._timer_lock.acquire()
return True
def release(self) -> None:
"""End a `hold`."""
self._timer_lock.release()
self._consumer_lock.release()
self._intake.set()
def rename_reclaimed(self, rename: Rename) -> None:
"""Move the names in reclaimed work still waiting to be run, which
the queue's own `rename` cannot reach. Under a `hold`."""
with self._inflight_lock:
self._reclaimed = deque(
rename.item(item) or item for item in self._reclaimed
)
def _busy(self) -> list[str]:
"""The nodes whose cascades are running. Under _inflight_lock."""
return sorted(self._running)
def alive(self) -> bool:
return self._consumer is not None and self._consumer.is_alive()
@property
def inflight(self) -> int:
"""Cascades claimed and still running."""
return self._inflight
with self._inflight_lock:
return self._running.total()
# -------------------------------------------------------------------------
# Threads
@@ -255,40 +340,45 @@ class ExecutionService:
if not self._intake.is_set():
self._intake.wait(timeout=0.5)
continue
free, due_only = self._await_capacity()
if not free:
continue
with self._inflight_lock:
recovered = [
item
for item in self._reclaimed
if not due_only or item.not_before > 0
][:free]
for item in recovered:
self._reclaimed.remove(item)
if recovered:
for item in recovered:
self._dispatch(item)
continue
try:
items = self.queue.claim(
free,
# Briefly, in the due-only case: this is the saturated
# engine, and a slot freeing has to be noticed promptly.
DUE_CLAIM_BLOCK_MS if due_only else CLAIM_BLOCK_MS,
due_only,
)
failures = 0
except Exception as exc:
failures += 1
logger.error("Could not claim work: %s", exc)
self._publish_unavailable(exc)
# Backing off hard: a queue that is down stays down for a while.
self._stop.wait(min(30.0, 2.0**failures))
continue
backoff = 0.0
with self._consumer_lock:
free, due_only = self._await_capacity()
if not free:
continue
with self._inflight_lock:
recovered = [
item
for item in self._reclaimed
if not due_only or item.not_before > 0
][:free]
for item in recovered:
self._reclaimed.remove(item)
if recovered:
for item in recovered:
self._dispatch(item)
continue
try:
items = self.queue.claim(
free,
# Briefly, in the due-only case: this is the saturated
# engine, and a slot freeing has to be noticed promptly.
DUE_CLAIM_BLOCK_MS if due_only else CLAIM_BLOCK_MS,
due_only,
)
failures = 0
except Exception as exc:
failures += 1
logger.error("Could not claim work: %s", exc)
self._publish_unavailable(exc)
# Backing off hard: a queue that is down stays down for a
# while. Outside the lock, so a hold does not wait it out.
backoff = min(30.0, 2.0**failures)
items = []
for item in items:
self._dispatch(item)
for item in items:
self._dispatch(item)
if backoff:
self._stop.wait(backoff)
def _sleep_until_due(self) -> bool:
"""Wait for the soonest deadline, the housekeeping cap, or a new one.
@@ -321,54 +411,56 @@ class ExecutionService:
promote = self._sleep_until_due()
if self._stop.is_set():
break
try:
if promote:
self.queue.move_due(time.time())
except Exception as exc:
logger.error("Could not promote delayed work: %s", exc)
# The item is still due, so the wait above would be zero and
# this would spin on a queue that is down. Back off to what a
# fixed poll used to cost.
self._stop.wait(DELAYED_INTERVAL_S)
now = time.monotonic()
if now - last_backlog >= BACKLOG_INTERVAL_S:
last_backlog = now
# One pass, not the sleep above: a hold waits for it to end.
with self._timer_lock:
try:
self._check_backlog()
if promote:
self.queue.move_due(time.time())
except Exception as exc:
logger.error("Could not read the queue backlog: %s", exc)
logger.error("Could not promote delayed work: %s", exc)
# The item is still due, so the wait above would be zero and
# this would spin on a queue that is down. Back off to what a
# fixed poll used to cost.
self._stop.wait(DELAYED_INTERVAL_S)
if now - last_touch >= TOUCH_INTERVAL_S:
last_touch = now
with self._inflight_lock:
running = list(self._active)
try:
self.queue.touch(running)
except Exception as exc:
logger.error("Could not touch claimed work: %s", exc)
now = time.monotonic()
if now - last_backlog >= BACKLOG_INTERVAL_S:
last_backlog = now
try:
self._check_backlog()
except Exception as exc:
logger.error("Could not read the queue backlog: %s", exc)
if now - last_reclaim < RECLAIM_INTERVAL_S:
continue
with self._inflight_lock:
if self._reclaimed or not self._intake.is_set():
if now - last_touch >= TOUCH_INTERVAL_S:
last_touch = now
with self._inflight_lock:
running = list(self._active)
try:
self.queue.touch(running)
except Exception as exc:
logger.error("Could not touch claimed work: %s", exc)
if now - last_reclaim < RECLAIM_INTERVAL_S:
continue
last_reclaim = now
try:
recovered = self.queue.reclaim_stale(RECLAIM_IDLE_MS)
for item in recovered:
logger.info(
"Reclaimed work for '%s' (delivery %d)",
item.node,
item.deliveries,
)
with self._inflight_lock:
self._reclaimed.extend(recovered)
self._active.update(
item.entry_id for item in recovered if item.entry_id
)
except Exception as exc:
logger.error("Could not reclaim stale work: %s", exc)
if self._reclaimed or not self._intake.is_set():
continue
last_reclaim = now
try:
recovered = self.queue.reclaim_stale(RECLAIM_IDLE_MS)
for item in recovered:
logger.info(
"Reclaimed work for '%s' (delivery %d)",
item.node,
item.deliveries,
)
with self._inflight_lock:
self._reclaimed.extend(recovered)
self._active.update(
item.entry_id for item in recovered if item.entry_id
)
except Exception as exc:
logger.error("Could not reclaim stale work: %s", exc)
def _check_backlog(self) -> None:
"""Say so when work has been waiting in the stream for a while.
@@ -439,7 +531,7 @@ class ExecutionService:
def _dispatch(self, item: WorkItem) -> None:
with self._inflight_lock:
self._inflight += 1
self._running[item.node] += 1
if item.entry_id:
self._active.add(item.entry_id)
try:
@@ -455,7 +547,7 @@ class ExecutionService:
def _done(self, item: WorkItem) -> None:
with self._inflight_lock:
self._inflight -= 1
self._running -= Counter((item.node,))
self._active.discard(item.entry_id)
self._inflight_lock.notify_all()
@@ -621,7 +713,7 @@ class ExecutionService:
except Exception as exc:
return {"error": str(exc), "consumer_alive": self.alive()}
stats["consumer_alive"] = self.alive()
stats["cascades_busy"] = self._inflight
stats["cascades_busy"] = self.inflight
stats["behind"] = self.behind
return stats