Engine holds for a rename: the pipeline refuses publish and inline cascades with EngineHeld once those running have finished, the execution service stops claiming and its timers once claimed cascades have finished; both wait without a limit, report what is busy and can be aborted. REST answers 503 with Retry-After, a device frame busy. Redis adds no longer wait for each other, only a queue hold waits for them
This commit is contained in:
@@ -17,7 +17,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from collections import Counter, deque
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Executor, Future, ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
|
||||
@@ -26,6 +26,7 @@ from fluksio.flow.queue import MAX_DELIVERIES, WorkItem, WorkQueue
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.namespace import Rename
|
||||
from fluksio.flow.pipeline import Pipeline
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -69,10 +70,41 @@ BACKLOG_INTERVAL_S = 5.0
|
||||
BACKLOG_DEGRADED = 50
|
||||
BACKLOG_STRIKES = 3
|
||||
|
||||
#: How often an engine hold looks at its abort while it waits.
|
||||
HOLD_POLL_S = 0.5
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
def wait_out(
|
||||
lock: threading.Condition,
|
||||
busy: Callable[[], list[str]],
|
||||
abort: threading.Event,
|
||||
report: Callable[[list[str]], None] | None = None,
|
||||
) -> bool:
|
||||
"""Wait until ``busy``, read under ``lock``, names nothing.
|
||||
|
||||
Without a limit, because an engine hold waits for as long as the work
|
||||
takes. ``report`` hears each new set of names, so whoever asked can show
|
||||
what it waits for, and setting ``abort`` gives up with False.
|
||||
"""
|
||||
said: list[str] = []
|
||||
while True:
|
||||
with lock:
|
||||
names = busy()
|
||||
while names and names == said and not abort.is_set():
|
||||
lock.wait(HOLD_POLL_S)
|
||||
names = busy()
|
||||
if abort.is_set():
|
||||
return False
|
||||
if not names:
|
||||
return True
|
||||
said = names
|
||||
if report is not None:
|
||||
report(names)
|
||||
|
||||
|
||||
class _PoolLane(Executor):
|
||||
def __init__(self, pool: _ReservedPool, due: bool) -> None:
|
||||
self._pool = pool
|
||||
@@ -158,7 +190,8 @@ class ExecutionService:
|
||||
queue.on_delayed = self._timer_wake.set
|
||||
self._intake = threading.Event()
|
||||
self._intake.set()
|
||||
self._inflight = 0
|
||||
# Cascades claimed and still running, by the node each came from.
|
||||
self._running: Counter[str] = Counter()
|
||||
self._inflight_lock = threading.Condition()
|
||||
# Entry ids claimed and still running, under _inflight_lock.
|
||||
self._active: set[str] = set()
|
||||
@@ -174,6 +207,12 @@ class ExecutionService:
|
||||
)
|
||||
self._consumer: threading.Thread | None = None
|
||||
self._timers: threading.Thread | None = None
|
||||
# Held by the consumer for one pass of capacity check, claim and
|
||||
# dispatch, and by the timer thread for one pass of promotion, touch
|
||||
# and reclaim. `hold` takes both. Plain locks, because `release` may
|
||||
# come from another thread than the one that held.
|
||||
self._consumer_lock = threading.Lock()
|
||||
self._timer_lock = threading.Lock()
|
||||
# Consecutive backlog readings over the threshold, and whether the last
|
||||
# of them said so out loud.
|
||||
self._backlog_strikes = 0
|
||||
@@ -225,11 +264,11 @@ class ExecutionService:
|
||||
self._intake.clear()
|
||||
deadline = time.monotonic() + DRAIN_TIMEOUT_S
|
||||
with self._inflight_lock:
|
||||
while self._inflight:
|
||||
while self._running:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
logger.warning(
|
||||
"Rebuild did not wait out %d cascades", self._inflight
|
||||
"Rebuild did not wait out %d cascades", self._running.total()
|
||||
)
|
||||
return
|
||||
self._inflight_lock.wait(remaining)
|
||||
@@ -237,13 +276,59 @@ class ExecutionService:
|
||||
def resume_intake(self) -> None:
|
||||
self._intake.set()
|
||||
|
||||
def hold(
|
||||
self,
|
||||
abort: threading.Event,
|
||||
report: Callable[[list[str]], None] | None = None,
|
||||
) -> bool:
|
||||
"""Stop claiming, promoting, touching and reclaiming, once every
|
||||
claimed cascade has finished. For a rename, which rewrites the queue.
|
||||
|
||||
Unlike `pause_intake` this also waits out a claim already blocking
|
||||
when intake clears, and a timer pass. It waits without a limit and
|
||||
tells ``report`` the nodes whose cascades are still running each time
|
||||
they change. Setting ``abort`` gives up: claiming resumes and it
|
||||
returns False. True means held until `release`.
|
||||
"""
|
||||
self._intake.clear()
|
||||
# So a consumer waiting for capacity sees the intake closed.
|
||||
self._capacity_changed()
|
||||
self._consumer_lock.acquire()
|
||||
# The timer lock only after the drain: while a cascade runs, the
|
||||
# timer thread keeps touching it, so it is not reclaimed as dropped.
|
||||
if not wait_out(self._inflight_lock, self._busy, abort, report):
|
||||
self._consumer_lock.release()
|
||||
self._intake.set()
|
||||
return False
|
||||
self._timer_lock.acquire()
|
||||
return True
|
||||
|
||||
def release(self) -> None:
|
||||
"""End a `hold`."""
|
||||
self._timer_lock.release()
|
||||
self._consumer_lock.release()
|
||||
self._intake.set()
|
||||
|
||||
def rename_reclaimed(self, rename: Rename) -> None:
|
||||
"""Move the names in reclaimed work still waiting to be run, which
|
||||
the queue's own `rename` cannot reach. Under a `hold`."""
|
||||
with self._inflight_lock:
|
||||
self._reclaimed = deque(
|
||||
rename.item(item) or item for item in self._reclaimed
|
||||
)
|
||||
|
||||
def _busy(self) -> list[str]:
|
||||
"""The nodes whose cascades are running. Under _inflight_lock."""
|
||||
return sorted(self._running)
|
||||
|
||||
def alive(self) -> bool:
|
||||
return self._consumer is not None and self._consumer.is_alive()
|
||||
|
||||
@property
|
||||
def inflight(self) -> int:
|
||||
"""Cascades claimed and still running."""
|
||||
return self._inflight
|
||||
with self._inflight_lock:
|
||||
return self._running.total()
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Threads
|
||||
@@ -255,40 +340,45 @@ class ExecutionService:
|
||||
if not self._intake.is_set():
|
||||
self._intake.wait(timeout=0.5)
|
||||
continue
|
||||
free, due_only = self._await_capacity()
|
||||
if not free:
|
||||
continue
|
||||
with self._inflight_lock:
|
||||
recovered = [
|
||||
item
|
||||
for item in self._reclaimed
|
||||
if not due_only or item.not_before > 0
|
||||
][:free]
|
||||
for item in recovered:
|
||||
self._reclaimed.remove(item)
|
||||
if recovered:
|
||||
for item in recovered:
|
||||
self._dispatch(item)
|
||||
continue
|
||||
try:
|
||||
items = self.queue.claim(
|
||||
free,
|
||||
# Briefly, in the due-only case: this is the saturated
|
||||
# engine, and a slot freeing has to be noticed promptly.
|
||||
DUE_CLAIM_BLOCK_MS if due_only else CLAIM_BLOCK_MS,
|
||||
due_only,
|
||||
)
|
||||
failures = 0
|
||||
except Exception as exc:
|
||||
failures += 1
|
||||
logger.error("Could not claim work: %s", exc)
|
||||
self._publish_unavailable(exc)
|
||||
# Backing off hard: a queue that is down stays down for a while.
|
||||
self._stop.wait(min(30.0, 2.0**failures))
|
||||
continue
|
||||
backoff = 0.0
|
||||
with self._consumer_lock:
|
||||
free, due_only = self._await_capacity()
|
||||
if not free:
|
||||
continue
|
||||
with self._inflight_lock:
|
||||
recovered = [
|
||||
item
|
||||
for item in self._reclaimed
|
||||
if not due_only or item.not_before > 0
|
||||
][:free]
|
||||
for item in recovered:
|
||||
self._reclaimed.remove(item)
|
||||
if recovered:
|
||||
for item in recovered:
|
||||
self._dispatch(item)
|
||||
continue
|
||||
try:
|
||||
items = self.queue.claim(
|
||||
free,
|
||||
# Briefly, in the due-only case: this is the saturated
|
||||
# engine, and a slot freeing has to be noticed promptly.
|
||||
DUE_CLAIM_BLOCK_MS if due_only else CLAIM_BLOCK_MS,
|
||||
due_only,
|
||||
)
|
||||
failures = 0
|
||||
except Exception as exc:
|
||||
failures += 1
|
||||
logger.error("Could not claim work: %s", exc)
|
||||
self._publish_unavailable(exc)
|
||||
# Backing off hard: a queue that is down stays down for a
|
||||
# while. Outside the lock, so a hold does not wait it out.
|
||||
backoff = min(30.0, 2.0**failures)
|
||||
items = []
|
||||
|
||||
for item in items:
|
||||
self._dispatch(item)
|
||||
for item in items:
|
||||
self._dispatch(item)
|
||||
if backoff:
|
||||
self._stop.wait(backoff)
|
||||
|
||||
def _sleep_until_due(self) -> bool:
|
||||
"""Wait for the soonest deadline, the housekeeping cap, or a new one.
|
||||
@@ -321,54 +411,56 @@ class ExecutionService:
|
||||
promote = self._sleep_until_due()
|
||||
if self._stop.is_set():
|
||||
break
|
||||
try:
|
||||
if promote:
|
||||
self.queue.move_due(time.time())
|
||||
except Exception as exc:
|
||||
logger.error("Could not promote delayed work: %s", exc)
|
||||
# The item is still due, so the wait above would be zero and
|
||||
# this would spin on a queue that is down. Back off to what a
|
||||
# fixed poll used to cost.
|
||||
self._stop.wait(DELAYED_INTERVAL_S)
|
||||
|
||||
now = time.monotonic()
|
||||
if now - last_backlog >= BACKLOG_INTERVAL_S:
|
||||
last_backlog = now
|
||||
# One pass, not the sleep above: a hold waits for it to end.
|
||||
with self._timer_lock:
|
||||
try:
|
||||
self._check_backlog()
|
||||
if promote:
|
||||
self.queue.move_due(time.time())
|
||||
except Exception as exc:
|
||||
logger.error("Could not read the queue backlog: %s", exc)
|
||||
logger.error("Could not promote delayed work: %s", exc)
|
||||
# The item is still due, so the wait above would be zero and
|
||||
# this would spin on a queue that is down. Back off to what a
|
||||
# fixed poll used to cost.
|
||||
self._stop.wait(DELAYED_INTERVAL_S)
|
||||
|
||||
if now - last_touch >= TOUCH_INTERVAL_S:
|
||||
last_touch = now
|
||||
with self._inflight_lock:
|
||||
running = list(self._active)
|
||||
try:
|
||||
self.queue.touch(running)
|
||||
except Exception as exc:
|
||||
logger.error("Could not touch claimed work: %s", exc)
|
||||
now = time.monotonic()
|
||||
if now - last_backlog >= BACKLOG_INTERVAL_S:
|
||||
last_backlog = now
|
||||
try:
|
||||
self._check_backlog()
|
||||
except Exception as exc:
|
||||
logger.error("Could not read the queue backlog: %s", exc)
|
||||
|
||||
if now - last_reclaim < RECLAIM_INTERVAL_S:
|
||||
continue
|
||||
with self._inflight_lock:
|
||||
if self._reclaimed or not self._intake.is_set():
|
||||
if now - last_touch >= TOUCH_INTERVAL_S:
|
||||
last_touch = now
|
||||
with self._inflight_lock:
|
||||
running = list(self._active)
|
||||
try:
|
||||
self.queue.touch(running)
|
||||
except Exception as exc:
|
||||
logger.error("Could not touch claimed work: %s", exc)
|
||||
|
||||
if now - last_reclaim < RECLAIM_INTERVAL_S:
|
||||
continue
|
||||
last_reclaim = now
|
||||
try:
|
||||
recovered = self.queue.reclaim_stale(RECLAIM_IDLE_MS)
|
||||
for item in recovered:
|
||||
logger.info(
|
||||
"Reclaimed work for '%s' (delivery %d)",
|
||||
item.node,
|
||||
item.deliveries,
|
||||
)
|
||||
with self._inflight_lock:
|
||||
self._reclaimed.extend(recovered)
|
||||
self._active.update(
|
||||
item.entry_id for item in recovered if item.entry_id
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Could not reclaim stale work: %s", exc)
|
||||
if self._reclaimed or not self._intake.is_set():
|
||||
continue
|
||||
last_reclaim = now
|
||||
try:
|
||||
recovered = self.queue.reclaim_stale(RECLAIM_IDLE_MS)
|
||||
for item in recovered:
|
||||
logger.info(
|
||||
"Reclaimed work for '%s' (delivery %d)",
|
||||
item.node,
|
||||
item.deliveries,
|
||||
)
|
||||
with self._inflight_lock:
|
||||
self._reclaimed.extend(recovered)
|
||||
self._active.update(
|
||||
item.entry_id for item in recovered if item.entry_id
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Could not reclaim stale work: %s", exc)
|
||||
|
||||
def _check_backlog(self) -> None:
|
||||
"""Say so when work has been waiting in the stream for a while.
|
||||
@@ -439,7 +531,7 @@ class ExecutionService:
|
||||
|
||||
def _dispatch(self, item: WorkItem) -> None:
|
||||
with self._inflight_lock:
|
||||
self._inflight += 1
|
||||
self._running[item.node] += 1
|
||||
if item.entry_id:
|
||||
self._active.add(item.entry_id)
|
||||
try:
|
||||
@@ -455,7 +547,7 @@ class ExecutionService:
|
||||
|
||||
def _done(self, item: WorkItem) -> None:
|
||||
with self._inflight_lock:
|
||||
self._inflight -= 1
|
||||
self._running -= Counter((item.node,))
|
||||
self._active.discard(item.entry_id)
|
||||
self._inflight_lock.notify_all()
|
||||
|
||||
@@ -621,7 +713,7 @@ class ExecutionService:
|
||||
except Exception as exc:
|
||||
return {"error": str(exc), "consumer_alive": self.alive()}
|
||||
stats["consumer_alive"] = self.alive()
|
||||
stats["cascades_busy"] = self._inflight
|
||||
stats["cascades_busy"] = self.inflight
|
||||
stats["behind"] = self.behind
|
||||
return stats
|
||||
|
||||
|
||||
Reference in New Issue
Block a user