stroblme
75b3a93094
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
228 lines
8.9 KiB
Python
228 lines
8.9 KiB
Python
"""Event bus bridging the engine's worker threads to async subscribers.
|
|
|
|
Nodes execute in a thread pool; websocket clients live on the event loop.
|
|
Publishers are therefore thread-safe and never wait for subscribers. Both
|
|
staging and subscriber queues are bounded, retaining errors and control events
|
|
ahead of routine traffic. Subscribers can ask whether they lost events
|
|
(``dropped``) to catch up another way.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import threading
|
|
from collections import deque
|
|
from collections.abc import AsyncIterator
|
|
from contextlib import asynccontextmanager
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
QUEUE_SIZE = 256
|
|
DISPATCH_BUDGET = 1024
|
|
LOG_HISTORY = 400
|
|
#: How many of a message's last artifact digests are remembered.
|
|
RECENT_DIGESTS = 8
|
|
|
|
|
|
def _critical(event: dict[str, Any]) -> bool:
|
|
"""Keep failures, lifecycle changes and unknown events ahead of telemetry."""
|
|
kind = event.get("type")
|
|
if kind == "node_log":
|
|
return event.get("level") == "error"
|
|
return kind not in {
|
|
"message_value",
|
|
"node_started",
|
|
"node_executed",
|
|
"work_latency",
|
|
"run_metric",
|
|
}
|
|
|
|
|
|
class _EventBuffer:
|
|
"""Two FIFO lanes allow constant-time eviction without reordering events."""
|
|
|
|
def __init__(self) -> None:
|
|
self._ordinary: deque[tuple[int, dict[str, Any]]] = deque()
|
|
self._critical: deque[tuple[int, dict[str, Any]]] = deque()
|
|
self._sequence = 0
|
|
|
|
def __len__(self) -> int:
|
|
return len(self._ordinary) + len(self._critical)
|
|
|
|
def append(self, event: dict[str, Any]) -> None:
|
|
lane = self._critical if _critical(event) else self._ordinary
|
|
lane.append((self._sequence, event))
|
|
self._sequence += 1
|
|
|
|
def popleft(self) -> dict[str, Any]:
|
|
lane = self._critical
|
|
if self._ordinary and (not lane or self._ordinary[0][0] < lane[0][0]):
|
|
lane = self._ordinary
|
|
return lane.popleft()[1]
|
|
|
|
def make_room(self, event: dict[str, Any]) -> bool:
|
|
"""Evict the oldest ordinary event, or keep a full critical backlog."""
|
|
if self._ordinary:
|
|
self._ordinary.popleft()
|
|
elif _critical(event):
|
|
self._critical.popleft()
|
|
else:
|
|
return False
|
|
return True
|
|
|
|
|
|
class _EventQueue(asyncio.Queue[dict[str, Any]]):
|
|
def _init(self, maxsize: int) -> None:
|
|
self._queue = _EventBuffer()
|
|
|
|
def offer(self, event: dict[str, Any]) -> bool:
|
|
"""Enqueue without waiting and report any loss, including rejection."""
|
|
lost = self.full()
|
|
if not lost or self._queue.make_room(event):
|
|
self.put_nowait(event)
|
|
return lost
|
|
|
|
|
|
class EventBus:
|
|
"""Fan-out of engine events to any number of async subscribers."""
|
|
|
|
def __init__(self) -> None:
|
|
self._loop: asyncio.AbstractEventLoop | None = None
|
|
self._subscribers: set[_EventQueue] = set()
|
|
# Events published since the last flush was scheduled. A cascade
|
|
# publishes 13-16 of them and each used to cost its own
|
|
# `call_soon_threadsafe`; they now cost one between two turns of the
|
|
# loop, which is what a wave is.
|
|
self._pending = _EventBuffer()
|
|
self._pending_dropped = False
|
|
self._pending_lock = threading.Lock()
|
|
self._flush_scheduled = False
|
|
self._batch: list[dict[str, Any]] = []
|
|
self._batch_dropped = False
|
|
self._targets: deque[_EventQueue] = deque()
|
|
# Kept whether or not anyone is listening, so opening the log panel
|
|
# shows what just happened rather than an empty box.
|
|
self.recent_logs: deque[dict[str, Any]] = deque(maxlen=LOG_HISTORY)
|
|
# How often each qualified node has emitted, for the same reason: a
|
|
# client that reconnects between two pages would otherwise start from
|
|
# zero and draw a busy graph as idle. A session count, not a metric —
|
|
# the rollups in `metrics.py` are what survives a restart.
|
|
self.emits: dict[str, int] = {}
|
|
#: The last few artifacts each message pointed at, newest last. A
|
|
#: screen fetching a camera frame by frame asks for one the message
|
|
#: may already have moved past, and a device's credential reaches
|
|
#: these as well as the current one.
|
|
self.digests: dict[str, deque[str]] = {}
|
|
#: Subscribers that lost an event since they last asked. Touched on
|
|
#: the event loop only, by the flush and the subscribers.
|
|
self._dropped: set[asyncio.Queue[dict[str, Any]]] = set()
|
|
|
|
def bind(self, loop: asyncio.AbstractEventLoop) -> None:
|
|
"""Attach the bus to the running event loop (called once at startup)."""
|
|
self._loop = loop
|
|
|
|
def publish(self, event: dict[str, Any]) -> None:
|
|
"""Publish an event from any thread."""
|
|
kind = event.get("type")
|
|
if kind == "node_log":
|
|
# deque.append is atomic, so worker threads need no lock here.
|
|
self.recent_logs.append(event)
|
|
elif kind == "node_executed" and event.get("outputs"):
|
|
# The condition the canvas animates on, so both ends agree on what
|
|
# counts as an emission. Two worker threads racing here can lose an
|
|
# increment, which is a count nobody is going to miss.
|
|
node = str(event.get("node") or "")
|
|
self.emits[node] = self.emits.get(node, 0) + 1
|
|
elif kind == "message_value":
|
|
value = event.get("value")
|
|
if isinstance(value, dict) and isinstance(value.get("digest"), str):
|
|
name = str(event.get("name") or "")
|
|
recent = self.digests.get(name)
|
|
if recent is None:
|
|
recent = self.digests[name] = deque(maxlen=RECENT_DIGESTS)
|
|
recent.append(value["digest"])
|
|
loop = self._loop
|
|
if loop is None or not self._subscribers:
|
|
return
|
|
with self._pending_lock:
|
|
room = True
|
|
if len(self._pending) == QUEUE_SIZE:
|
|
self._pending_dropped = True
|
|
room = self._pending.make_room(event)
|
|
if room:
|
|
self._pending.append(event)
|
|
if self._flush_scheduled:
|
|
# A flush is already on its way and has not run yet, so it
|
|
# will find this event too.
|
|
return
|
|
self._flush_scheduled = True
|
|
try:
|
|
loop.call_soon_threadsafe(self._dispatch)
|
|
except RuntimeError:
|
|
# Loop already closed — shutting down.
|
|
with self._pending_lock:
|
|
self._pending = _EventBuffer()
|
|
self._pending_dropped = False
|
|
self._flush_scheduled = False
|
|
|
|
def _dispatch(self) -> None:
|
|
"""Deliver bounded work, yielding between batches or subscriber groups."""
|
|
if not self._targets:
|
|
with self._pending_lock:
|
|
self._batch = [
|
|
self._pending.popleft() for _ in range(len(self._pending))
|
|
]
|
|
self._batch_dropped = self._pending_dropped
|
|
self._pending_dropped = False
|
|
self._targets = deque(self._subscribers)
|
|
if self._batch:
|
|
# A subscriber receives the whole bounded batch before it resumes,
|
|
# so its catch-up snapshot cannot precede the rest of that batch.
|
|
count = max(1, DISPATCH_BUDGET // len(self._batch))
|
|
for _ in range(min(count, len(self._targets))):
|
|
queue = self._targets.popleft()
|
|
if queue not in self._subscribers:
|
|
continue
|
|
if self._batch_dropped:
|
|
self._dropped.add(queue)
|
|
for event in self._batch:
|
|
if queue.offer(event):
|
|
self._dropped.add(queue)
|
|
else:
|
|
self._targets.clear()
|
|
if not self._targets:
|
|
self._batch = []
|
|
with self._pending_lock:
|
|
if not self._pending:
|
|
self._flush_scheduled = False
|
|
return
|
|
if self._loop is not None:
|
|
self._loop.call_soon(self._dispatch)
|
|
|
|
def dropped(self, queue: asyncio.Queue[dict[str, Any]]) -> bool:
|
|
"""Whether this subscriber lost events since it last asked.
|
|
|
|
This includes staging overflow before fan-out. A socket catches up
|
|
with a snapshot, since a lost value might never change again.
|
|
"""
|
|
if queue in self._dropped:
|
|
self._dropped.discard(queue)
|
|
return True
|
|
return False
|
|
|
|
@asynccontextmanager
|
|
async def subscribe(self) -> AsyncIterator[asyncio.Queue[dict[str, Any]]]:
|
|
"""Yield a queue receiving every event published while subscribed."""
|
|
queue = _EventQueue(maxsize=QUEUE_SIZE)
|
|
self._subscribers.add(queue)
|
|
try:
|
|
yield queue
|
|
finally:
|
|
self._subscribers.discard(queue)
|
|
self._dropped.discard(queue)
|
|
|
|
|
|
event_bus = EventBus()
|