Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
This commit is contained in:
@@ -17,8 +17,10 @@ from __future__ import annotations
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Executor, Future, ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
|
||||
|
||||
from fluksio.flow.queue import MAX_DELIVERIES, WorkItem, WorkQueue
|
||||
|
||||
@@ -41,9 +43,9 @@ TOUCH_INTERVAL_S = 20.0
|
||||
# resolution of a delay: a delayed item is waited for exactly, so what a timer
|
||||
# fires late by is a wake-up and a promotion rather than up to a whole second.
|
||||
DELAYED_INTERVAL_S = 1.0
|
||||
# Cascade slots a promoted timer may use past `max_cascades`. A due item was
|
||||
# Extra cascade and node workers reserved for promoted timers. A due item was
|
||||
# already waited for, so making it queue behind whatever long node happens to
|
||||
# hold the pool is the one lateness the sleeping timer thread cannot remove.
|
||||
# hold either pool is lateness the sleeping timer thread cannot remove.
|
||||
DUE_RESERVE = 2
|
||||
# How long the saturated engine waits on the due lane before going back to
|
||||
# check whether a cascade slot has freed.
|
||||
@@ -67,6 +69,73 @@ BACKLOG_INTERVAL_S = 5.0
|
||||
BACKLOG_DEGRADED = 50
|
||||
BACKLOG_STRIKES = 3
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
class _PoolLane(Executor):
|
||||
def __init__(self, pool: _ReservedPool, due: bool) -> None:
|
||||
self._pool = pool
|
||||
self._due = due
|
||||
|
||||
def submit(
|
||||
self, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs
|
||||
) -> Future[_T]:
|
||||
return self._pool.submit(self._due, fn, *args, **kwargs)
|
||||
|
||||
|
||||
class _ReservedPool:
|
||||
"""Due work uses idle normal workers, or its own bounded reserve."""
|
||||
|
||||
def __init__(
|
||||
self, workers: int, prefix: str, on_done: Callable[[], None] | None = None
|
||||
) -> None:
|
||||
self._limits = (workers, DUE_RESERVE)
|
||||
self._pools = (
|
||||
ThreadPoolExecutor(max_workers=workers, thread_name_prefix=prefix),
|
||||
ThreadPoolExecutor(
|
||||
max_workers=DUE_RESERVE, thread_name_prefix=f"{prefix}-due"
|
||||
),
|
||||
)
|
||||
self._pending = [0, 0]
|
||||
self._lock = threading.Lock()
|
||||
self._on_done = on_done
|
||||
self.normal = _PoolLane(self, False)
|
||||
self.due = _PoolLane(self, True)
|
||||
|
||||
def available(self) -> tuple[int, int]:
|
||||
with self._lock:
|
||||
return (
|
||||
max(0, self._limits[0] - self._pending[0]),
|
||||
max(0, self._limits[1] - self._pending[1]),
|
||||
)
|
||||
|
||||
def _done(self, lane: int) -> None:
|
||||
with self._lock:
|
||||
self._pending[lane] -= 1
|
||||
if self._on_done is not None:
|
||||
self._on_done()
|
||||
|
||||
def submit(
|
||||
self, due: bool, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs
|
||||
) -> Future[_T]:
|
||||
with self._lock:
|
||||
# Count queued work too: an idle thread with ordinary work already
|
||||
# waiting is not capacity a timer can use without waiting behind it.
|
||||
lane = int(due and self._pending[0] >= self._limits[0])
|
||||
self._pending[lane] += 1
|
||||
try:
|
||||
future = self._pools[lane].submit(fn, *args, **kwargs)
|
||||
except RuntimeError:
|
||||
self._done(lane)
|
||||
raise
|
||||
future.add_done_callback(lambda _: self._done(lane))
|
||||
return future
|
||||
|
||||
def shutdown(self) -> None:
|
||||
for pool in self._pools:
|
||||
pool.shutdown(wait=False)
|
||||
|
||||
|
||||
class ExecutionService:
|
||||
"""Owns the engine's worker threads and the queue they read from."""
|
||||
@@ -93,12 +162,15 @@ class ExecutionService:
|
||||
self._inflight_lock = threading.Condition()
|
||||
# Entry ids claimed and still running, under _inflight_lock.
|
||||
self._active: set[str] = set()
|
||||
self.node_pool = ThreadPoolExecutor(
|
||||
max_workers=MAX_WORKERS if max_workers is None else max_workers,
|
||||
thread_name_prefix="node",
|
||||
# At most one reclaim batch waits here. The timer thread must keep
|
||||
# promoting and touching work while the consumer waits for capacity.
|
||||
self._reclaimed: deque[WorkItem] = deque()
|
||||
self._node_pools = _ReservedPool(
|
||||
MAX_WORKERS if max_workers is None else max_workers, "node"
|
||||
)
|
||||
self._cascade_pool = ThreadPoolExecutor(
|
||||
max_workers=self.max_cascades, thread_name_prefix="cascade"
|
||||
self.node_pool = self._node_pools.normal
|
||||
self._cascade_pools = _ReservedPool(
|
||||
self.max_cascades, "cascade", self._capacity_changed
|
||||
)
|
||||
self._consumer: threading.Thread | None = None
|
||||
self._timers: threading.Thread | None = None
|
||||
@@ -132,14 +204,18 @@ class ExecutionService:
|
||||
thread.join(timeout=5)
|
||||
self._consumer = None
|
||||
self._timers = None
|
||||
self._cascade_pool.shutdown(wait=False)
|
||||
self.node_pool.shutdown(wait=False)
|
||||
self._cascade_pools.shutdown()
|
||||
self._node_pools.shutdown()
|
||||
self.queue.close()
|
||||
|
||||
def bind(self, pipeline: Pipeline) -> None:
|
||||
"""Point the service at the pipeline it should execute against."""
|
||||
self._pipeline = pipeline
|
||||
|
||||
def _capacity_changed(self) -> None:
|
||||
with self._inflight_lock:
|
||||
self._inflight_lock.notify_all()
|
||||
|
||||
def pause_intake(self) -> None:
|
||||
"""Stop claiming, and wait for what is already claimed to finish.
|
||||
|
||||
@@ -182,6 +258,18 @@ class ExecutionService:
|
||||
free, due_only = self._await_capacity()
|
||||
if not free:
|
||||
continue
|
||||
with self._inflight_lock:
|
||||
recovered = [
|
||||
item
|
||||
for item in self._reclaimed
|
||||
if not due_only or item.not_before > 0
|
||||
][:free]
|
||||
for item in recovered:
|
||||
self._reclaimed.remove(item)
|
||||
if recovered:
|
||||
for item in recovered:
|
||||
self._dispatch(item)
|
||||
continue
|
||||
try:
|
||||
items = self.queue.claim(
|
||||
free,
|
||||
@@ -262,21 +350,23 @@ class ExecutionService:
|
||||
|
||||
if now - last_reclaim < RECLAIM_INTERVAL_S:
|
||||
continue
|
||||
with self._inflight_lock:
|
||||
if self._reclaimed or not self._intake.is_set():
|
||||
continue
|
||||
last_reclaim = now
|
||||
try:
|
||||
for item in self.queue.reclaim_stale(RECLAIM_IDLE_MS):
|
||||
recovered = self.queue.reclaim_stale(RECLAIM_IDLE_MS)
|
||||
for item in recovered:
|
||||
logger.info(
|
||||
"Reclaimed work for '%s' (delivery %d)",
|
||||
item.node,
|
||||
item.deliveries,
|
||||
)
|
||||
# Through the same gate the main loop uses: a reclaim can
|
||||
# return sixty-odd entries at once, and dispatching them
|
||||
# all would push `_inflight` far past `max_cascades` —
|
||||
# exactly the overcommit the gate exists to prevent.
|
||||
if not self._await_capacity()[0]:
|
||||
break
|
||||
self._dispatch(item)
|
||||
with self._inflight_lock:
|
||||
self._reclaimed.extend(recovered)
|
||||
self._active.update(
|
||||
item.entry_id for item in recovered if item.entry_id
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Could not reclaim stale work: %s", exc)
|
||||
|
||||
@@ -331,7 +421,9 @@ class ExecutionService:
|
||||
"""
|
||||
with self._inflight_lock:
|
||||
while not self._stop.is_set():
|
||||
free = self.max_cascades - self._inflight
|
||||
if not self._intake.is_set():
|
||||
return 0, False
|
||||
free, _ = self._cascade_pools.available()
|
||||
if free > 0:
|
||||
return free, False
|
||||
# Only once a slot has genuinely failed to free: the due lane
|
||||
@@ -340,7 +432,7 @@ class ExecutionService:
|
||||
# read every time the pool filled up.
|
||||
if self._inflight_lock.wait(0.5):
|
||||
continue
|
||||
reserve = self.max_cascades + DUE_RESERVE - self._inflight
|
||||
_, reserve = self._cascade_pools.available()
|
||||
if reserve > 0:
|
||||
return reserve, True
|
||||
return 0, False
|
||||
@@ -351,7 +443,12 @@ class ExecutionService:
|
||||
if item.entry_id:
|
||||
self._active.add(item.entry_id)
|
||||
try:
|
||||
self._cascade_pool.submit(self._handle, item)
|
||||
pool = (
|
||||
self._cascade_pools.due
|
||||
if item.not_before > 0
|
||||
else self._cascade_pools.normal
|
||||
)
|
||||
pool.submit(self._handle, item)
|
||||
except RuntimeError:
|
||||
# Pool already shutting down.
|
||||
self._done(item)
|
||||
@@ -386,7 +483,7 @@ class ExecutionService:
|
||||
return item.node
|
||||
|
||||
def _handle(self, item: WorkItem) -> None:
|
||||
handled = True
|
||||
handled = False
|
||||
try:
|
||||
handled = self._run_item(item)
|
||||
except Exception:
|
||||
@@ -473,11 +570,12 @@ class ExecutionService:
|
||||
|
||||
replay = item.deliveries > 1
|
||||
emission = item.kind == "emission"
|
||||
node_pool = self._node_pools.due if item.not_before > 0 else self.node_pool
|
||||
try:
|
||||
if item.kind == "flush":
|
||||
# A rate-limit window ended; nothing to replay, only to let out.
|
||||
# A run of its own, so what it sets off is recorded somewhere.
|
||||
pipeline.flush(node, item.entry_id)
|
||||
pipeline.flush(node, item.entry_id, node_pool=node_pool)
|
||||
return True
|
||||
# An emission's values went into state when the node produced them;
|
||||
# this item carries them so its readers get the chunk that caused
|
||||
@@ -498,6 +596,7 @@ class ExecutionService:
|
||||
# Neither can say what changed, so neither filters on it.
|
||||
changed=None if replay or not item.outputs else published,
|
||||
overrides=item.outputs if emission else None,
|
||||
node_pool=node_pool,
|
||||
)
|
||||
finally:
|
||||
# Paired, or a cascade that raised — state backend gone, say — is a
|
||||
|
||||
Reference in New Issue
Block a user