Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
This commit is contained in:
+112
-26
@@ -1,9 +1,10 @@
|
||||
"""Event bus bridging the engine's worker threads to async subscribers.
|
||||
|
||||
Nodes execute in a thread pool; websocket clients live on the event loop.
|
||||
Publishers are therefore thread-safe and never block: a subscriber that cannot
|
||||
keep up loses its oldest queued events rather than stalling the engine, and
|
||||
can ask whether it did (``dropped``) to catch up another way.
|
||||
Publishers are therefore thread-safe and never wait for subscribers. Both
|
||||
staging and subscriber queues are bounded, retaining errors and control events
|
||||
ahead of routine traffic. Subscribers can ask whether they lost events
|
||||
(``dropped``) to catch up another way.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,24 +20,88 @@ from typing import Any
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
QUEUE_SIZE = 256
|
||||
DISPATCH_BUDGET = 1024
|
||||
LOG_HISTORY = 400
|
||||
#: How many of a message's last artifact digests are remembered.
|
||||
RECENT_DIGESTS = 8
|
||||
|
||||
|
||||
def _critical(event: dict[str, Any]) -> bool:
|
||||
"""Keep failures, lifecycle changes and unknown events ahead of telemetry."""
|
||||
kind = event.get("type")
|
||||
if kind == "node_log":
|
||||
return event.get("level") == "error"
|
||||
return kind not in {
|
||||
"message_value",
|
||||
"node_started",
|
||||
"node_executed",
|
||||
"work_latency",
|
||||
"run_metric",
|
||||
}
|
||||
|
||||
|
||||
class _EventBuffer:
|
||||
"""Two FIFO lanes allow constant-time eviction without reordering events."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._ordinary: deque[tuple[int, dict[str, Any]]] = deque()
|
||||
self._critical: deque[tuple[int, dict[str, Any]]] = deque()
|
||||
self._sequence = 0
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._ordinary) + len(self._critical)
|
||||
|
||||
def append(self, event: dict[str, Any]) -> None:
|
||||
lane = self._critical if _critical(event) else self._ordinary
|
||||
lane.append((self._sequence, event))
|
||||
self._sequence += 1
|
||||
|
||||
def popleft(self) -> dict[str, Any]:
|
||||
lane = self._critical
|
||||
if self._ordinary and (not lane or self._ordinary[0][0] < lane[0][0]):
|
||||
lane = self._ordinary
|
||||
return lane.popleft()[1]
|
||||
|
||||
def make_room(self, event: dict[str, Any]) -> bool:
|
||||
"""Evict the oldest ordinary event, or keep a full critical backlog."""
|
||||
if self._ordinary:
|
||||
self._ordinary.popleft()
|
||||
elif _critical(event):
|
||||
self._critical.popleft()
|
||||
else:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class _EventQueue(asyncio.Queue[dict[str, Any]]):
|
||||
def _init(self, maxsize: int) -> None:
|
||||
self._queue = _EventBuffer()
|
||||
|
||||
def offer(self, event: dict[str, Any]) -> bool:
|
||||
"""Enqueue without waiting and report any loss, including rejection."""
|
||||
lost = self.full()
|
||||
if not lost or self._queue.make_room(event):
|
||||
self.put_nowait(event)
|
||||
return lost
|
||||
|
||||
|
||||
class EventBus:
|
||||
"""Fan-out of engine events to any number of async subscribers."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
self._subscribers: set[asyncio.Queue[dict[str, Any]]] = set()
|
||||
self._subscribers: set[_EventQueue] = set()
|
||||
# Events published since the last flush was scheduled. A cascade
|
||||
# publishes 13-16 of them and each used to cost its own
|
||||
# `call_soon_threadsafe`; they now cost one between two turns of the
|
||||
# loop, which is what a wave is.
|
||||
self._pending: list[dict[str, Any]] = []
|
||||
self._pending = _EventBuffer()
|
||||
self._pending_dropped = False
|
||||
self._pending_lock = threading.Lock()
|
||||
self._flush_scheduled = False
|
||||
self._batch: list[dict[str, Any]] = []
|
||||
self._batch_dropped = False
|
||||
self._targets: deque[_EventQueue] = deque()
|
||||
# Kept whether or not anyone is listening, so opening the log panel
|
||||
# shows what just happened rather than an empty box.
|
||||
self.recent_logs: deque[dict[str, Any]] = deque(maxlen=LOG_HISTORY)
|
||||
@@ -82,7 +147,12 @@ class EventBus:
|
||||
if loop is None or not self._subscribers:
|
||||
return
|
||||
with self._pending_lock:
|
||||
self._pending.append(event)
|
||||
room = True
|
||||
if len(self._pending) == QUEUE_SIZE:
|
||||
self._pending_dropped = True
|
||||
room = self._pending.make_room(event)
|
||||
if room:
|
||||
self._pending.append(event)
|
||||
if self._flush_scheduled:
|
||||
# A flush is already on its way and has not run yet, so it
|
||||
# will find this event too.
|
||||
@@ -93,33 +163,49 @@ class EventBus:
|
||||
except RuntimeError:
|
||||
# Loop already closed — shutting down.
|
||||
with self._pending_lock:
|
||||
self._pending.clear()
|
||||
self._pending = _EventBuffer()
|
||||
self._pending_dropped = False
|
||||
self._flush_scheduled = False
|
||||
|
||||
def _dispatch(self) -> None:
|
||||
"""Hand everything published since the last turn to every subscriber."""
|
||||
with self._pending_lock:
|
||||
batch = self._pending
|
||||
self._pending = []
|
||||
self._flush_scheduled = False
|
||||
if not batch:
|
||||
return
|
||||
for queue in self._subscribers:
|
||||
for event in batch:
|
||||
if queue.full():
|
||||
# Drop the oldest so a slow client never blocks the engine.
|
||||
try:
|
||||
queue.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
pass
|
||||
"""Deliver bounded work, yielding between batches or subscriber groups."""
|
||||
if not self._targets:
|
||||
with self._pending_lock:
|
||||
self._batch = [
|
||||
self._pending.popleft() for _ in range(len(self._pending))
|
||||
]
|
||||
self._batch_dropped = self._pending_dropped
|
||||
self._pending_dropped = False
|
||||
self._targets = deque(self._subscribers)
|
||||
if self._batch:
|
||||
# A subscriber receives the whole bounded batch before it resumes,
|
||||
# so its catch-up snapshot cannot precede the rest of that batch.
|
||||
count = max(1, DISPATCH_BUDGET // len(self._batch))
|
||||
for _ in range(min(count, len(self._targets))):
|
||||
queue = self._targets.popleft()
|
||||
if queue not in self._subscribers:
|
||||
continue
|
||||
if self._batch_dropped:
|
||||
self._dropped.add(queue)
|
||||
queue.put_nowait(event)
|
||||
for event in self._batch:
|
||||
if queue.offer(event):
|
||||
self._dropped.add(queue)
|
||||
else:
|
||||
self._targets.clear()
|
||||
if not self._targets:
|
||||
self._batch = []
|
||||
with self._pending_lock:
|
||||
if not self._pending:
|
||||
self._flush_scheduled = False
|
||||
return
|
||||
if self._loop is not None:
|
||||
self._loop.call_soon(self._dispatch)
|
||||
|
||||
def dropped(self, queue: asyncio.Queue[dict[str, Any]]) -> bool:
|
||||
"""Whether this subscriber lost events since it last asked.
|
||||
|
||||
A device's socket that did catches up with a snapshot: what it lost
|
||||
may have been its demand, or the event that tells it it was unpaired.
|
||||
This includes staging overflow before fan-out. A socket catches up
|
||||
with a snapshot, since a lost value might never change again.
|
||||
"""
|
||||
if queue in self._dropped:
|
||||
self._dropped.discard(queue)
|
||||
@@ -129,7 +215,7 @@ class EventBus:
|
||||
@asynccontextmanager
|
||||
async def subscribe(self) -> AsyncIterator[asyncio.Queue[dict[str, Any]]]:
|
||||
"""Yield a queue receiving every event published while subscribed."""
|
||||
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=QUEUE_SIZE)
|
||||
queue = _EventQueue(maxsize=QUEUE_SIZE)
|
||||
self._subscribers.add(queue)
|
||||
try:
|
||||
yield queue
|
||||
|
||||
Reference in New Issue
Block a user