"""Event bus bridging the engine's worker threads to async subscribers. Nodes execute in a thread pool; websocket clients live on the event loop. Publishers are therefore thread-safe and never wait for subscribers. Both staging and subscriber queues are bounded, retaining errors and control events ahead of routine traffic. Subscribers can ask whether they lost events (``dropped``) to catch up another way. """ from __future__ import annotations import asyncio import logging import threading from collections import deque from collections.abc import AsyncIterator from contextlib import asynccontextmanager from typing import Any logger = logging.getLogger(__name__) QUEUE_SIZE = 256 DISPATCH_BUDGET = 1024 LOG_HISTORY = 400 #: How many of a message's last artifact digests are remembered. RECENT_DIGESTS = 8 def _critical(event: dict[str, Any]) -> bool: """Keep failures, lifecycle changes and unknown events ahead of telemetry.""" kind = event.get("type") if kind == "node_log": return event.get("level") == "error" return kind not in { "message_value", "node_started", "node_executed", "work_latency", "run_metric", } class _EventBuffer: """Two FIFO lanes allow constant-time eviction without reordering events.""" def __init__(self) -> None: self._ordinary: deque[tuple[int, dict[str, Any]]] = deque() self._critical: deque[tuple[int, dict[str, Any]]] = deque() self._sequence = 0 def __len__(self) -> int: return len(self._ordinary) + len(self._critical) def append(self, event: dict[str, Any]) -> None: lane = self._critical if _critical(event) else self._ordinary lane.append((self._sequence, event)) self._sequence += 1 def popleft(self) -> dict[str, Any]: lane = self._critical if self._ordinary and (not lane or self._ordinary[0][0] < lane[0][0]): lane = self._ordinary return lane.popleft()[1] def make_room(self, event: dict[str, Any]) -> bool: """Evict the oldest ordinary event, or keep a full critical backlog.""" if self._ordinary: self._ordinary.popleft() elif _critical(event): self._critical.popleft() else: return False return True class _EventQueue(asyncio.Queue[dict[str, Any]]): def _init(self, maxsize: int) -> None: self._queue = _EventBuffer() def offer(self, event: dict[str, Any]) -> bool: """Enqueue without waiting and report any loss, including rejection.""" lost = self.full() if not lost or self._queue.make_room(event): self.put_nowait(event) return lost class EventBus: """Fan-out of engine events to any number of async subscribers.""" def __init__(self) -> None: self._loop: asyncio.AbstractEventLoop | None = None self._subscribers: set[_EventQueue] = set() # Events published since the last flush was scheduled. A cascade # publishes 13-16 of them and each used to cost its own # `call_soon_threadsafe`; they now cost one between two turns of the # loop, which is what a wave is. self._pending = _EventBuffer() self._pending_dropped = False self._pending_lock = threading.Lock() self._flush_scheduled = False self._batch: list[dict[str, Any]] = [] self._batch_dropped = False self._targets: deque[_EventQueue] = deque() # Kept whether or not anyone is listening, so opening the log panel # shows what just happened rather than an empty box. self.recent_logs: deque[dict[str, Any]] = deque(maxlen=LOG_HISTORY) # How often each qualified node has emitted, for the same reason: a # client that reconnects between two pages would otherwise start from # zero and draw a busy graph as idle. A session count, not a metric — # the rollups in `metrics.py` are what survives a restart. self.emits: dict[str, int] = {} #: The last few artifacts each message pointed at, newest last. A #: screen fetching a camera frame by frame asks for one the message #: may already have moved past, and a device's credential reaches #: these as well as the current one. self.digests: dict[str, deque[str]] = {} #: Subscribers that lost an event since they last asked. Touched on #: the event loop only, by the flush and the subscribers. self._dropped: set[asyncio.Queue[dict[str, Any]]] = set() def bind(self, loop: asyncio.AbstractEventLoop) -> None: """Attach the bus to the running event loop (called once at startup).""" self._loop = loop def publish(self, event: dict[str, Any]) -> None: """Publish an event from any thread.""" kind = event.get("type") if kind == "node_log": # deque.append is atomic, so worker threads need no lock here. self.recent_logs.append(event) elif kind == "node_executed" and event.get("outputs"): # The condition the canvas animates on, so both ends agree on what # counts as an emission. Two worker threads racing here can lose an # increment, which is a count nobody is going to miss. node = str(event.get("node") or "") self.emits[node] = self.emits.get(node, 0) + 1 elif kind == "message_value": value = event.get("value") if isinstance(value, dict) and isinstance(value.get("digest"), str): name = str(event.get("name") or "") recent = self.digests.get(name) if recent is None: recent = self.digests[name] = deque(maxlen=RECENT_DIGESTS) recent.append(value["digest"]) loop = self._loop if loop is None or not self._subscribers: return with self._pending_lock: room = True if len(self._pending) == QUEUE_SIZE: self._pending_dropped = True room = self._pending.make_room(event) if room: self._pending.append(event) if self._flush_scheduled: # A flush is already on its way and has not run yet, so it # will find this event too. return self._flush_scheduled = True try: loop.call_soon_threadsafe(self._dispatch) except RuntimeError: # Loop already closed — shutting down. with self._pending_lock: self._pending = _EventBuffer() self._pending_dropped = False self._flush_scheduled = False def _dispatch(self) -> None: """Deliver bounded work, yielding between batches or subscriber groups.""" if not self._targets: with self._pending_lock: self._batch = [ self._pending.popleft() for _ in range(len(self._pending)) ] self._batch_dropped = self._pending_dropped self._pending_dropped = False self._targets = deque(self._subscribers) if self._batch: # A subscriber receives the whole bounded batch before it resumes, # so its catch-up snapshot cannot precede the rest of that batch. count = max(1, DISPATCH_BUDGET // len(self._batch)) for _ in range(min(count, len(self._targets))): queue = self._targets.popleft() if queue not in self._subscribers: continue if self._batch_dropped: self._dropped.add(queue) for event in self._batch: if queue.offer(event): self._dropped.add(queue) else: self._targets.clear() if not self._targets: self._batch = [] with self._pending_lock: if not self._pending: self._flush_scheduled = False return if self._loop is not None: self._loop.call_soon(self._dispatch) def dropped(self, queue: asyncio.Queue[dict[str, Any]]) -> bool: """Whether this subscriber lost events since it last asked. This includes staging overflow before fan-out. A socket catches up with a snapshot, since a lost value might never change again. """ if queue in self._dropped: self._dropped.discard(queue) return True return False @asynccontextmanager async def subscribe(self) -> AsyncIterator[asyncio.Queue[dict[str, Any]]]: """Yield a queue receiving every event published while subscribed.""" queue = _EventQueue(maxsize=QUEUE_SIZE) self._subscribers.add(queue) try: yield queue finally: self._subscribers.discard(queue) self._dropped.discard(queue) event_bus = EventBus()