Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s

This commit is contained in:
2026-10-29 14:24:17 +01:00
parent 20a0853f69
commit 75b3a93094
25 changed files with 1316 additions and 301 deletions
+112 -26
View File
@@ -1,9 +1,10 @@
"""Event bus bridging the engine's worker threads to async subscribers.
Nodes execute in a thread pool; websocket clients live on the event loop.
Publishers are therefore thread-safe and never block: a subscriber that cannot
keep up loses its oldest queued events rather than stalling the engine, and
can ask whether it did (``dropped``) to catch up another way.
Publishers are therefore thread-safe and never wait for subscribers. Both
staging and subscriber queues are bounded, retaining errors and control events
ahead of routine traffic. Subscribers can ask whether they lost events
(``dropped``) to catch up another way.
"""
from __future__ import annotations
@@ -19,24 +20,88 @@ from typing import Any
logger = logging.getLogger(__name__)
QUEUE_SIZE = 256
DISPATCH_BUDGET = 1024
LOG_HISTORY = 400
#: How many of a message's last artifact digests are remembered.
RECENT_DIGESTS = 8
def _critical(event: dict[str, Any]) -> bool:
"""Keep failures, lifecycle changes and unknown events ahead of telemetry."""
kind = event.get("type")
if kind == "node_log":
return event.get("level") == "error"
return kind not in {
"message_value",
"node_started",
"node_executed",
"work_latency",
"run_metric",
}
class _EventBuffer:
"""Two FIFO lanes allow constant-time eviction without reordering events."""
def __init__(self) -> None:
self._ordinary: deque[tuple[int, dict[str, Any]]] = deque()
self._critical: deque[tuple[int, dict[str, Any]]] = deque()
self._sequence = 0
def __len__(self) -> int:
return len(self._ordinary) + len(self._critical)
def append(self, event: dict[str, Any]) -> None:
lane = self._critical if _critical(event) else self._ordinary
lane.append((self._sequence, event))
self._sequence += 1
def popleft(self) -> dict[str, Any]:
lane = self._critical
if self._ordinary and (not lane or self._ordinary[0][0] < lane[0][0]):
lane = self._ordinary
return lane.popleft()[1]
def make_room(self, event: dict[str, Any]) -> bool:
"""Evict the oldest ordinary event, or keep a full critical backlog."""
if self._ordinary:
self._ordinary.popleft()
elif _critical(event):
self._critical.popleft()
else:
return False
return True
class _EventQueue(asyncio.Queue[dict[str, Any]]):
def _init(self, maxsize: int) -> None:
self._queue = _EventBuffer()
def offer(self, event: dict[str, Any]) -> bool:
"""Enqueue without waiting and report any loss, including rejection."""
lost = self.full()
if not lost or self._queue.make_room(event):
self.put_nowait(event)
return lost
class EventBus:
"""Fan-out of engine events to any number of async subscribers."""
def __init__(self) -> None:
self._loop: asyncio.AbstractEventLoop | None = None
self._subscribers: set[asyncio.Queue[dict[str, Any]]] = set()
self._subscribers: set[_EventQueue] = set()
# Events published since the last flush was scheduled. A cascade
# publishes 13-16 of them and each used to cost its own
# `call_soon_threadsafe`; they now cost one between two turns of the
# loop, which is what a wave is.
self._pending: list[dict[str, Any]] = []
self._pending = _EventBuffer()
self._pending_dropped = False
self._pending_lock = threading.Lock()
self._flush_scheduled = False
self._batch: list[dict[str, Any]] = []
self._batch_dropped = False
self._targets: deque[_EventQueue] = deque()
# Kept whether or not anyone is listening, so opening the log panel
# shows what just happened rather than an empty box.
self.recent_logs: deque[dict[str, Any]] = deque(maxlen=LOG_HISTORY)
@@ -82,7 +147,12 @@ class EventBus:
if loop is None or not self._subscribers:
return
with self._pending_lock:
self._pending.append(event)
room = True
if len(self._pending) == QUEUE_SIZE:
self._pending_dropped = True
room = self._pending.make_room(event)
if room:
self._pending.append(event)
if self._flush_scheduled:
# A flush is already on its way and has not run yet, so it
# will find this event too.
@@ -93,33 +163,49 @@ class EventBus:
except RuntimeError:
# Loop already closed — shutting down.
with self._pending_lock:
self._pending.clear()
self._pending = _EventBuffer()
self._pending_dropped = False
self._flush_scheduled = False
def _dispatch(self) -> None:
"""Hand everything published since the last turn to every subscriber."""
with self._pending_lock:
batch = self._pending
self._pending = []
self._flush_scheduled = False
if not batch:
return
for queue in self._subscribers:
for event in batch:
if queue.full():
# Drop the oldest so a slow client never blocks the engine.
try:
queue.get_nowait()
except asyncio.QueueEmpty:
pass
"""Deliver bounded work, yielding between batches or subscriber groups."""
if not self._targets:
with self._pending_lock:
self._batch = [
self._pending.popleft() for _ in range(len(self._pending))
]
self._batch_dropped = self._pending_dropped
self._pending_dropped = False
self._targets = deque(self._subscribers)
if self._batch:
# A subscriber receives the whole bounded batch before it resumes,
# so its catch-up snapshot cannot precede the rest of that batch.
count = max(1, DISPATCH_BUDGET // len(self._batch))
for _ in range(min(count, len(self._targets))):
queue = self._targets.popleft()
if queue not in self._subscribers:
continue
if self._batch_dropped:
self._dropped.add(queue)
queue.put_nowait(event)
for event in self._batch:
if queue.offer(event):
self._dropped.add(queue)
else:
self._targets.clear()
if not self._targets:
self._batch = []
with self._pending_lock:
if not self._pending:
self._flush_scheduled = False
return
if self._loop is not None:
self._loop.call_soon(self._dispatch)
def dropped(self, queue: asyncio.Queue[dict[str, Any]]) -> bool:
"""Whether this subscriber lost events since it last asked.
A device's socket that did catches up with a snapshot: what it lost
may have been its demand, or the event that tells it it was unpaired.
This includes staging overflow before fan-out. A socket catches up
with a snapshot, since a lost value might never change again.
"""
if queue in self._dropped:
self._dropped.discard(queue)
@@ -129,7 +215,7 @@ class EventBus:
@asynccontextmanager
async def subscribe(self) -> AsyncIterator[asyncio.Queue[dict[str, Any]]]:
"""Yield a queue receiving every event published while subscribed."""
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=QUEUE_SIZE)
queue = _EventQueue(maxsize=QUEUE_SIZE)
self._subscribers.add(queue)
try:
yield queue