Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s

This commit is contained in:
2026-10-29 14:24:17 +01:00
parent 20a0853f69
commit 75b3a93094
25 changed files with 1316 additions and 301 deletions
+123 -24
View File
@@ -17,8 +17,10 @@ from __future__ import annotations
import logging
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING, Any
from collections import deque
from collections.abc import Callable
from concurrent.futures import Executor, Future, ThreadPoolExecutor
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
from fluksio.flow.queue import MAX_DELIVERIES, WorkItem, WorkQueue
@@ -41,9 +43,9 @@ TOUCH_INTERVAL_S = 20.0
# resolution of a delay: a delayed item is waited for exactly, so what a timer
# fires late by is a wake-up and a promotion rather than up to a whole second.
DELAYED_INTERVAL_S = 1.0
# Cascade slots a promoted timer may use past `max_cascades`. A due item was
# Extra cascade and node workers reserved for promoted timers. A due item was
# already waited for, so making it queue behind whatever long node happens to
# hold the pool is the one lateness the sleeping timer thread cannot remove.
# hold either pool is lateness the sleeping timer thread cannot remove.
DUE_RESERVE = 2
# How long the saturated engine waits on the due lane before going back to
# check whether a cascade slot has freed.
@@ -67,6 +69,73 @@ BACKLOG_INTERVAL_S = 5.0
BACKLOG_DEGRADED = 50
BACKLOG_STRIKES = 3
_P = ParamSpec("_P")
_T = TypeVar("_T")
class _PoolLane(Executor):
def __init__(self, pool: _ReservedPool, due: bool) -> None:
self._pool = pool
self._due = due
def submit(
self, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs
) -> Future[_T]:
return self._pool.submit(self._due, fn, *args, **kwargs)
class _ReservedPool:
"""Due work uses idle normal workers, or its own bounded reserve."""
def __init__(
self, workers: int, prefix: str, on_done: Callable[[], None] | None = None
) -> None:
self._limits = (workers, DUE_RESERVE)
self._pools = (
ThreadPoolExecutor(max_workers=workers, thread_name_prefix=prefix),
ThreadPoolExecutor(
max_workers=DUE_RESERVE, thread_name_prefix=f"{prefix}-due"
),
)
self._pending = [0, 0]
self._lock = threading.Lock()
self._on_done = on_done
self.normal = _PoolLane(self, False)
self.due = _PoolLane(self, True)
def available(self) -> tuple[int, int]:
with self._lock:
return (
max(0, self._limits[0] - self._pending[0]),
max(0, self._limits[1] - self._pending[1]),
)
def _done(self, lane: int) -> None:
with self._lock:
self._pending[lane] -= 1
if self._on_done is not None:
self._on_done()
def submit(
self, due: bool, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs
) -> Future[_T]:
with self._lock:
# Count queued work too: an idle thread with ordinary work already
# waiting is not capacity a timer can use without waiting behind it.
lane = int(due and self._pending[0] >= self._limits[0])
self._pending[lane] += 1
try:
future = self._pools[lane].submit(fn, *args, **kwargs)
except RuntimeError:
self._done(lane)
raise
future.add_done_callback(lambda _: self._done(lane))
return future
def shutdown(self) -> None:
for pool in self._pools:
pool.shutdown(wait=False)
class ExecutionService:
"""Owns the engine's worker threads and the queue they read from."""
@@ -93,12 +162,15 @@ class ExecutionService:
self._inflight_lock = threading.Condition()
# Entry ids claimed and still running, under _inflight_lock.
self._active: set[str] = set()
self.node_pool = ThreadPoolExecutor(
max_workers=MAX_WORKERS if max_workers is None else max_workers,
thread_name_prefix="node",
# At most one reclaim batch waits here. The timer thread must keep
# promoting and touching work while the consumer waits for capacity.
self._reclaimed: deque[WorkItem] = deque()
self._node_pools = _ReservedPool(
MAX_WORKERS if max_workers is None else max_workers, "node"
)
self._cascade_pool = ThreadPoolExecutor(
max_workers=self.max_cascades, thread_name_prefix="cascade"
self.node_pool = self._node_pools.normal
self._cascade_pools = _ReservedPool(
self.max_cascades, "cascade", self._capacity_changed
)
self._consumer: threading.Thread | None = None
self._timers: threading.Thread | None = None
@@ -132,14 +204,18 @@ class ExecutionService:
thread.join(timeout=5)
self._consumer = None
self._timers = None
self._cascade_pool.shutdown(wait=False)
self.node_pool.shutdown(wait=False)
self._cascade_pools.shutdown()
self._node_pools.shutdown()
self.queue.close()
def bind(self, pipeline: Pipeline) -> None:
"""Point the service at the pipeline it should execute against."""
self._pipeline = pipeline
def _capacity_changed(self) -> None:
with self._inflight_lock:
self._inflight_lock.notify_all()
def pause_intake(self) -> None:
"""Stop claiming, and wait for what is already claimed to finish.
@@ -182,6 +258,18 @@ class ExecutionService:
free, due_only = self._await_capacity()
if not free:
continue
with self._inflight_lock:
recovered = [
item
for item in self._reclaimed
if not due_only or item.not_before > 0
][:free]
for item in recovered:
self._reclaimed.remove(item)
if recovered:
for item in recovered:
self._dispatch(item)
continue
try:
items = self.queue.claim(
free,
@@ -262,21 +350,23 @@ class ExecutionService:
if now - last_reclaim < RECLAIM_INTERVAL_S:
continue
with self._inflight_lock:
if self._reclaimed or not self._intake.is_set():
continue
last_reclaim = now
try:
for item in self.queue.reclaim_stale(RECLAIM_IDLE_MS):
recovered = self.queue.reclaim_stale(RECLAIM_IDLE_MS)
for item in recovered:
logger.info(
"Reclaimed work for '%s' (delivery %d)",
item.node,
item.deliveries,
)
# Through the same gate the main loop uses: a reclaim can
# return sixty-odd entries at once, and dispatching them
# all would push `_inflight` far past `max_cascades` —
# exactly the overcommit the gate exists to prevent.
if not self._await_capacity()[0]:
break
self._dispatch(item)
with self._inflight_lock:
self._reclaimed.extend(recovered)
self._active.update(
item.entry_id for item in recovered if item.entry_id
)
except Exception as exc:
logger.error("Could not reclaim stale work: %s", exc)
@@ -331,7 +421,9 @@ class ExecutionService:
"""
with self._inflight_lock:
while not self._stop.is_set():
free = self.max_cascades - self._inflight
if not self._intake.is_set():
return 0, False
free, _ = self._cascade_pools.available()
if free > 0:
return free, False
# Only once a slot has genuinely failed to free: the due lane
@@ -340,7 +432,7 @@ class ExecutionService:
# read every time the pool filled up.
if self._inflight_lock.wait(0.5):
continue
reserve = self.max_cascades + DUE_RESERVE - self._inflight
_, reserve = self._cascade_pools.available()
if reserve > 0:
return reserve, True
return 0, False
@@ -351,7 +443,12 @@ class ExecutionService:
if item.entry_id:
self._active.add(item.entry_id)
try:
self._cascade_pool.submit(self._handle, item)
pool = (
self._cascade_pools.due
if item.not_before > 0
else self._cascade_pools.normal
)
pool.submit(self._handle, item)
except RuntimeError:
# Pool already shutting down.
self._done(item)
@@ -386,7 +483,7 @@ class ExecutionService:
return item.node
def _handle(self, item: WorkItem) -> None:
handled = True
handled = False
try:
handled = self._run_item(item)
except Exception:
@@ -473,11 +570,12 @@ class ExecutionService:
replay = item.deliveries > 1
emission = item.kind == "emission"
node_pool = self._node_pools.due if item.not_before > 0 else self.node_pool
try:
if item.kind == "flush":
# A rate-limit window ended; nothing to replay, only to let out.
# A run of its own, so what it sets off is recorded somewhere.
pipeline.flush(node, item.entry_id)
pipeline.flush(node, item.entry_id, node_pool=node_pool)
return True
# An emission's values went into state when the node produced them;
# this item carries them so its readers get the chunk that caused
@@ -498,6 +596,7 @@ class ExecutionService:
# Neither can say what changed, so neither filters on it.
changed=None if replay or not item.outputs else published,
overrides=item.outputs if emission else None,
node_pool=node_pool,
)
finally:
# Paired, or a cascade that raised — state backend gone, say — is a