Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
This commit is contained in:
@@ -1700,8 +1700,8 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
|
||||
await websocket.send_text(orjson.dumps(payload).decode())
|
||||
return list(payload.get("demand", []))
|
||||
|
||||
async def catch_up() -> bool:
|
||||
"""Rescope a panel and send the snapshot again, demand included.
|
||||
async def rescope() -> bool:
|
||||
"""Recheck a panel and its demand before forwarding retained events.
|
||||
|
||||
False once a device whose credential stopped standing is closed with
|
||||
1008, which sends it to check: otherwise its frames would still be
|
||||
@@ -1722,12 +1722,18 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
|
||||
device_def = _device_def(websocket.app, device)
|
||||
wanted &= only
|
||||
demand.want(viewer, wanted)
|
||||
return True
|
||||
|
||||
async def catch_up() -> bool:
|
||||
if not await rescope():
|
||||
return False
|
||||
await send_snapshot()
|
||||
return True
|
||||
|
||||
told = await send_snapshot()
|
||||
|
||||
async with event_bus.subscribe() as queue:
|
||||
# Listen before reading state: a value changing while the snapshot is
|
||||
# sent must follow it on the socket, even if it never changes again.
|
||||
told = await send_snapshot()
|
||||
receiver = asyncio.create_task(_receive(websocket))
|
||||
# Where a live view this socket is one end of reaches it. Its own
|
||||
# outbox rather than the bus's queue, which drops when full.
|
||||
@@ -1741,6 +1747,7 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
|
||||
await websocket.send_text(demand_frame(ports))
|
||||
live_view.sessions.attach(device, endpoint)
|
||||
outbox = asyncio.create_task(endpoint.next())
|
||||
catching_up = False
|
||||
try:
|
||||
while True:
|
||||
sender = asyncio.create_task(queue.get())
|
||||
@@ -1845,10 +1852,13 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
|
||||
# A panel this far behind lost events the bus dropped for it:
|
||||
# its demand, a dashboard published or its own unpairing. It
|
||||
# catches up once this batch is out, as after a dashboard.
|
||||
resync = only is not None and event_bus.dropped(queue)
|
||||
# Every client catches up after overflow. A lost event may be
|
||||
# a value that never changes again, or a device's unpairing.
|
||||
lost = event_bus.dropped(queue)
|
||||
catching_up = catching_up or lost
|
||||
if lost and not await rescope():
|
||||
return
|
||||
resync = False
|
||||
if only is None:
|
||||
fresh = hidden_from(user, websocket.app)
|
||||
if fresh != hidden:
|
||||
@@ -1883,8 +1893,12 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
|
||||
for frame in media_frames(out, wanted, store):
|
||||
await websocket.send_bytes(frame)
|
||||
await _send(websocket, out)
|
||||
if resync and not await catch_up():
|
||||
return
|
||||
# Send retained events before the recovery snapshot: values
|
||||
# left in the queue would otherwise overwrite its newer state.
|
||||
if resync or (catching_up and queue.empty()):
|
||||
if not await catch_up():
|
||||
return
|
||||
catching_up = catching_up and not queue.empty()
|
||||
except (WebSocketDisconnect, RuntimeError):
|
||||
# A peer that goes away mid-send takes the RuntimeError route
|
||||
# ("websocket.send after websocket.close") rather than the clean
|
||||
|
||||
@@ -19,7 +19,6 @@ from fluksio.api.deps import (
|
||||
get_current_user,
|
||||
)
|
||||
from fluksio.flow.devices import DeviceDef, DeviceStore
|
||||
from fluksio.flow.messages import flow_of
|
||||
from fluksio.flow.pipeline import ValueSource
|
||||
from fluksio.flow.state import as_number
|
||||
|
||||
@@ -118,7 +117,7 @@ def _port_source(request: Request, name: str) -> ValueSource | None:
|
||||
|
||||
|
||||
@router.post("/{name}", response_model=MessageValue)
|
||||
async def publish_message(
|
||||
def publish_message(
|
||||
name: str, body: PublishRequest, controller: FlowControllerDep, request: Request
|
||||
) -> Any:
|
||||
"""Put a value into the graph, as a dashboard control does.
|
||||
@@ -133,7 +132,7 @@ async def publish_message(
|
||||
detail=body.source_detail,
|
||||
)
|
||||
try:
|
||||
await run_in_threadpool(controller.publish_message, name, body.value, source)
|
||||
controller.publish_message(name, body.value, source)
|
||||
except KeyError:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"No flow declares a message named '{name}'"
|
||||
@@ -141,8 +140,7 @@ async def publish_message(
|
||||
except TypeError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc))
|
||||
|
||||
values = controller.values(flow_of(name))
|
||||
current = values.get(name, {})
|
||||
current = controller.value(name)
|
||||
return MessageValue(name=name, value=current.get("value"), ts=current.get("ts"))
|
||||
|
||||
|
||||
|
||||
@@ -722,16 +722,6 @@ class CloudConnector:
|
||||
# The cameras a person may not see, looked at again per event.
|
||||
hidden = hidden_from(user, self._app, device=only is not None)
|
||||
controller = getattr(self._app.state, "flow_controller", None)
|
||||
told: list[str] = []
|
||||
if controller is not None:
|
||||
# A device here sends its frames over HTTP: the tunnel carries
|
||||
# text only.
|
||||
snapshot = snapshot_payload(controller, only, device, "http", hidden)
|
||||
told = list(snapshot.get("demand", []))
|
||||
await socket.send(
|
||||
_dump({"op": "ws_msg", "id": stream_id, "text": _dump(snapshot)})
|
||||
)
|
||||
|
||||
# The media this client's tiles are drawing count as demand on the
|
||||
# devices sending them, as a direct socket's do. The bytes themselves
|
||||
# are fetched one by one through the portal.
|
||||
@@ -743,6 +733,20 @@ class CloudConnector:
|
||||
outbox: asyncio.Task[str] | None = None
|
||||
sender: asyncio.Task[dict[str, Any]] | None = None
|
||||
|
||||
async def rescope() -> bool:
|
||||
nonlocal only, wanted
|
||||
if only is not None:
|
||||
if device and revoked(token, self._app):
|
||||
await socket.send(
|
||||
_dump({"op": "ws_close", "id": stream_id, "code": 1008})
|
||||
)
|
||||
return False
|
||||
# A deleted panel scopes to nothing, never to every message.
|
||||
only = panel_scope(token, self._app) or set()
|
||||
wanted &= only
|
||||
demand.want(viewer, wanted)
|
||||
return True
|
||||
|
||||
def heard(text: str) -> None:
|
||||
nonlocal wanted
|
||||
asked = wanted_names(text, only, hidden)
|
||||
@@ -768,6 +772,19 @@ class CloudConnector:
|
||||
live_view.sessions.attach(device, endpoint)
|
||||
try:
|
||||
async with event_bus.subscribe() as queue:
|
||||
# Subscribe before the snapshot so changes during its send
|
||||
# cannot fall between the snapshot and the live stream.
|
||||
told: list[str] = []
|
||||
if controller is not None:
|
||||
snapshot = snapshot_payload(
|
||||
controller, only, device, "http", hidden
|
||||
)
|
||||
told = list(snapshot.get("demand", []))
|
||||
await socket.send(
|
||||
_dump(
|
||||
{"op": "ws_msg", "id": stream_id, "text": _dump(snapshot)}
|
||||
)
|
||||
)
|
||||
outbox = asyncio.create_task(endpoint.next())
|
||||
if device and (ports := demand.ports(device)) != told:
|
||||
await socket.send(
|
||||
@@ -779,6 +796,7 @@ class CloudConnector:
|
||||
}
|
||||
)
|
||||
)
|
||||
catching_up = False
|
||||
while True:
|
||||
sender = asyncio.create_task(queue.get())
|
||||
done, _ = await asyncio.wait(
|
||||
@@ -796,11 +814,13 @@ class CloudConnector:
|
||||
if sender not in done:
|
||||
continue
|
||||
event = sender.result()
|
||||
# Behind far enough to lose events, a device catches up
|
||||
# as after a dashboard: the lost one may have been its
|
||||
# demand or its unpairing. A person whose cameras opened
|
||||
# or closed is sent what may be seen now.
|
||||
resync = only is not None and event_bus.dropped(queue)
|
||||
# A browser needs the same recovery as a device after
|
||||
# overflow: the lost value may never be published again.
|
||||
lost = event_bus.dropped(queue)
|
||||
catching_up = catching_up or lost
|
||||
if lost and not await rescope():
|
||||
return
|
||||
resync = False
|
||||
if only is None:
|
||||
fresh = hidden_from(user, self._app)
|
||||
if fresh != hidden:
|
||||
@@ -824,24 +844,10 @@ class CloudConnector:
|
||||
{"op": "ws_msg", "id": stream_id, "text": _dump(event)}
|
||||
)
|
||||
)
|
||||
if not resync:
|
||||
if not resync and not (catching_up and queue.empty()):
|
||||
continue
|
||||
if only is not None:
|
||||
if device and revoked(token, self._app):
|
||||
# Unpaired or deleted: 1008 sends it to check.
|
||||
await socket.send(
|
||||
_dump({"op": "ws_close", "id": stream_id, "code": 1008})
|
||||
)
|
||||
return
|
||||
# What this screen shows may have just changed under
|
||||
# it. Rescope, then resend the snapshot so a dashboard
|
||||
# it has only now been assigned draws values instead of
|
||||
# blanks. ``or set()`` is the point: a panel that was
|
||||
# deleted scopes to nothing, and None would widen this
|
||||
# socket to everything on the bus.
|
||||
only = panel_scope(token, self._app) or set()
|
||||
wanted &= only
|
||||
demand.want(viewer, wanted)
|
||||
if not await rescope():
|
||||
return
|
||||
if controller is not None:
|
||||
catch_up = snapshot_payload(
|
||||
controller, only, device, "http", hidden
|
||||
@@ -855,6 +861,10 @@ class CloudConnector:
|
||||
}
|
||||
)
|
||||
)
|
||||
# The snapshot follows the retained backlog. If sending
|
||||
# it queued more events, finish those before recovering
|
||||
# again so an older event cannot leave the client stale.
|
||||
catching_up = catching_up and not queue.empty()
|
||||
if kind == "dashboard_changed" and only is not None:
|
||||
# After the snapshot, as the device socket sends it.
|
||||
await socket.send(
|
||||
|
||||
@@ -1545,6 +1545,9 @@ class FlowController:
|
||||
return _short_error(exc)
|
||||
return None
|
||||
|
||||
def value(self, name: str) -> dict[str, Any]:
|
||||
return self.pipeline.value(name) if self.pipeline else {}
|
||||
|
||||
def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]:
|
||||
return self.pipeline.values(flow) if self.pipeline else {}
|
||||
|
||||
|
||||
+112
-26
@@ -1,9 +1,10 @@
|
||||
"""Event bus bridging the engine's worker threads to async subscribers.
|
||||
|
||||
Nodes execute in a thread pool; websocket clients live on the event loop.
|
||||
Publishers are therefore thread-safe and never block: a subscriber that cannot
|
||||
keep up loses its oldest queued events rather than stalling the engine, and
|
||||
can ask whether it did (``dropped``) to catch up another way.
|
||||
Publishers are therefore thread-safe and never wait for subscribers. Both
|
||||
staging and subscriber queues are bounded, retaining errors and control events
|
||||
ahead of routine traffic. Subscribers can ask whether they lost events
|
||||
(``dropped``) to catch up another way.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,24 +20,88 @@ from typing import Any
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
QUEUE_SIZE = 256
|
||||
DISPATCH_BUDGET = 1024
|
||||
LOG_HISTORY = 400
|
||||
#: How many of a message's last artifact digests are remembered.
|
||||
RECENT_DIGESTS = 8
|
||||
|
||||
|
||||
def _critical(event: dict[str, Any]) -> bool:
|
||||
"""Keep failures, lifecycle changes and unknown events ahead of telemetry."""
|
||||
kind = event.get("type")
|
||||
if kind == "node_log":
|
||||
return event.get("level") == "error"
|
||||
return kind not in {
|
||||
"message_value",
|
||||
"node_started",
|
||||
"node_executed",
|
||||
"work_latency",
|
||||
"run_metric",
|
||||
}
|
||||
|
||||
|
||||
class _EventBuffer:
|
||||
"""Two FIFO lanes allow constant-time eviction without reordering events."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._ordinary: deque[tuple[int, dict[str, Any]]] = deque()
|
||||
self._critical: deque[tuple[int, dict[str, Any]]] = deque()
|
||||
self._sequence = 0
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._ordinary) + len(self._critical)
|
||||
|
||||
def append(self, event: dict[str, Any]) -> None:
|
||||
lane = self._critical if _critical(event) else self._ordinary
|
||||
lane.append((self._sequence, event))
|
||||
self._sequence += 1
|
||||
|
||||
def popleft(self) -> dict[str, Any]:
|
||||
lane = self._critical
|
||||
if self._ordinary and (not lane or self._ordinary[0][0] < lane[0][0]):
|
||||
lane = self._ordinary
|
||||
return lane.popleft()[1]
|
||||
|
||||
def make_room(self, event: dict[str, Any]) -> bool:
|
||||
"""Evict the oldest ordinary event, or keep a full critical backlog."""
|
||||
if self._ordinary:
|
||||
self._ordinary.popleft()
|
||||
elif _critical(event):
|
||||
self._critical.popleft()
|
||||
else:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class _EventQueue(asyncio.Queue[dict[str, Any]]):
|
||||
def _init(self, maxsize: int) -> None:
|
||||
self._queue = _EventBuffer()
|
||||
|
||||
def offer(self, event: dict[str, Any]) -> bool:
|
||||
"""Enqueue without waiting and report any loss, including rejection."""
|
||||
lost = self.full()
|
||||
if not lost or self._queue.make_room(event):
|
||||
self.put_nowait(event)
|
||||
return lost
|
||||
|
||||
|
||||
class EventBus:
|
||||
"""Fan-out of engine events to any number of async subscribers."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
self._subscribers: set[asyncio.Queue[dict[str, Any]]] = set()
|
||||
self._subscribers: set[_EventQueue] = set()
|
||||
# Events published since the last flush was scheduled. A cascade
|
||||
# publishes 13-16 of them and each used to cost its own
|
||||
# `call_soon_threadsafe`; they now cost one between two turns of the
|
||||
# loop, which is what a wave is.
|
||||
self._pending: list[dict[str, Any]] = []
|
||||
self._pending = _EventBuffer()
|
||||
self._pending_dropped = False
|
||||
self._pending_lock = threading.Lock()
|
||||
self._flush_scheduled = False
|
||||
self._batch: list[dict[str, Any]] = []
|
||||
self._batch_dropped = False
|
||||
self._targets: deque[_EventQueue] = deque()
|
||||
# Kept whether or not anyone is listening, so opening the log panel
|
||||
# shows what just happened rather than an empty box.
|
||||
self.recent_logs: deque[dict[str, Any]] = deque(maxlen=LOG_HISTORY)
|
||||
@@ -82,7 +147,12 @@ class EventBus:
|
||||
if loop is None or not self._subscribers:
|
||||
return
|
||||
with self._pending_lock:
|
||||
self._pending.append(event)
|
||||
room = True
|
||||
if len(self._pending) == QUEUE_SIZE:
|
||||
self._pending_dropped = True
|
||||
room = self._pending.make_room(event)
|
||||
if room:
|
||||
self._pending.append(event)
|
||||
if self._flush_scheduled:
|
||||
# A flush is already on its way and has not run yet, so it
|
||||
# will find this event too.
|
||||
@@ -93,33 +163,49 @@ class EventBus:
|
||||
except RuntimeError:
|
||||
# Loop already closed — shutting down.
|
||||
with self._pending_lock:
|
||||
self._pending.clear()
|
||||
self._pending = _EventBuffer()
|
||||
self._pending_dropped = False
|
||||
self._flush_scheduled = False
|
||||
|
||||
def _dispatch(self) -> None:
|
||||
"""Hand everything published since the last turn to every subscriber."""
|
||||
with self._pending_lock:
|
||||
batch = self._pending
|
||||
self._pending = []
|
||||
self._flush_scheduled = False
|
||||
if not batch:
|
||||
return
|
||||
for queue in self._subscribers:
|
||||
for event in batch:
|
||||
if queue.full():
|
||||
# Drop the oldest so a slow client never blocks the engine.
|
||||
try:
|
||||
queue.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
pass
|
||||
"""Deliver bounded work, yielding between batches or subscriber groups."""
|
||||
if not self._targets:
|
||||
with self._pending_lock:
|
||||
self._batch = [
|
||||
self._pending.popleft() for _ in range(len(self._pending))
|
||||
]
|
||||
self._batch_dropped = self._pending_dropped
|
||||
self._pending_dropped = False
|
||||
self._targets = deque(self._subscribers)
|
||||
if self._batch:
|
||||
# A subscriber receives the whole bounded batch before it resumes,
|
||||
# so its catch-up snapshot cannot precede the rest of that batch.
|
||||
count = max(1, DISPATCH_BUDGET // len(self._batch))
|
||||
for _ in range(min(count, len(self._targets))):
|
||||
queue = self._targets.popleft()
|
||||
if queue not in self._subscribers:
|
||||
continue
|
||||
if self._batch_dropped:
|
||||
self._dropped.add(queue)
|
||||
queue.put_nowait(event)
|
||||
for event in self._batch:
|
||||
if queue.offer(event):
|
||||
self._dropped.add(queue)
|
||||
else:
|
||||
self._targets.clear()
|
||||
if not self._targets:
|
||||
self._batch = []
|
||||
with self._pending_lock:
|
||||
if not self._pending:
|
||||
self._flush_scheduled = False
|
||||
return
|
||||
if self._loop is not None:
|
||||
self._loop.call_soon(self._dispatch)
|
||||
|
||||
def dropped(self, queue: asyncio.Queue[dict[str, Any]]) -> bool:
|
||||
"""Whether this subscriber lost events since it last asked.
|
||||
|
||||
A device's socket that did catches up with a snapshot: what it lost
|
||||
may have been its demand, or the event that tells it it was unpaired.
|
||||
This includes staging overflow before fan-out. A socket catches up
|
||||
with a snapshot, since a lost value might never change again.
|
||||
"""
|
||||
if queue in self._dropped:
|
||||
self._dropped.discard(queue)
|
||||
@@ -129,7 +215,7 @@ class EventBus:
|
||||
@asynccontextmanager
|
||||
async def subscribe(self) -> AsyncIterator[asyncio.Queue[dict[str, Any]]]:
|
||||
"""Yield a queue receiving every event published while subscribed."""
|
||||
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=QUEUE_SIZE)
|
||||
queue = _EventQueue(maxsize=QUEUE_SIZE)
|
||||
self._subscribers.add(queue)
|
||||
try:
|
||||
yield queue
|
||||
|
||||
@@ -17,8 +17,10 @@ from __future__ import annotations
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Executor, Future, ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
|
||||
|
||||
from fluksio.flow.queue import MAX_DELIVERIES, WorkItem, WorkQueue
|
||||
|
||||
@@ -41,9 +43,9 @@ TOUCH_INTERVAL_S = 20.0
|
||||
# resolution of a delay: a delayed item is waited for exactly, so what a timer
|
||||
# fires late by is a wake-up and a promotion rather than up to a whole second.
|
||||
DELAYED_INTERVAL_S = 1.0
|
||||
# Cascade slots a promoted timer may use past `max_cascades`. A due item was
|
||||
# Extra cascade and node workers reserved for promoted timers. A due item was
|
||||
# already waited for, so making it queue behind whatever long node happens to
|
||||
# hold the pool is the one lateness the sleeping timer thread cannot remove.
|
||||
# hold either pool is lateness the sleeping timer thread cannot remove.
|
||||
DUE_RESERVE = 2
|
||||
# How long the saturated engine waits on the due lane before going back to
|
||||
# check whether a cascade slot has freed.
|
||||
@@ -67,6 +69,73 @@ BACKLOG_INTERVAL_S = 5.0
|
||||
BACKLOG_DEGRADED = 50
|
||||
BACKLOG_STRIKES = 3
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
class _PoolLane(Executor):
|
||||
def __init__(self, pool: _ReservedPool, due: bool) -> None:
|
||||
self._pool = pool
|
||||
self._due = due
|
||||
|
||||
def submit(
|
||||
self, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs
|
||||
) -> Future[_T]:
|
||||
return self._pool.submit(self._due, fn, *args, **kwargs)
|
||||
|
||||
|
||||
class _ReservedPool:
|
||||
"""Due work uses idle normal workers, or its own bounded reserve."""
|
||||
|
||||
def __init__(
|
||||
self, workers: int, prefix: str, on_done: Callable[[], None] | None = None
|
||||
) -> None:
|
||||
self._limits = (workers, DUE_RESERVE)
|
||||
self._pools = (
|
||||
ThreadPoolExecutor(max_workers=workers, thread_name_prefix=prefix),
|
||||
ThreadPoolExecutor(
|
||||
max_workers=DUE_RESERVE, thread_name_prefix=f"{prefix}-due"
|
||||
),
|
||||
)
|
||||
self._pending = [0, 0]
|
||||
self._lock = threading.Lock()
|
||||
self._on_done = on_done
|
||||
self.normal = _PoolLane(self, False)
|
||||
self.due = _PoolLane(self, True)
|
||||
|
||||
def available(self) -> tuple[int, int]:
|
||||
with self._lock:
|
||||
return (
|
||||
max(0, self._limits[0] - self._pending[0]),
|
||||
max(0, self._limits[1] - self._pending[1]),
|
||||
)
|
||||
|
||||
def _done(self, lane: int) -> None:
|
||||
with self._lock:
|
||||
self._pending[lane] -= 1
|
||||
if self._on_done is not None:
|
||||
self._on_done()
|
||||
|
||||
def submit(
|
||||
self, due: bool, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs
|
||||
) -> Future[_T]:
|
||||
with self._lock:
|
||||
# Count queued work too: an idle thread with ordinary work already
|
||||
# waiting is not capacity a timer can use without waiting behind it.
|
||||
lane = int(due and self._pending[0] >= self._limits[0])
|
||||
self._pending[lane] += 1
|
||||
try:
|
||||
future = self._pools[lane].submit(fn, *args, **kwargs)
|
||||
except RuntimeError:
|
||||
self._done(lane)
|
||||
raise
|
||||
future.add_done_callback(lambda _: self._done(lane))
|
||||
return future
|
||||
|
||||
def shutdown(self) -> None:
|
||||
for pool in self._pools:
|
||||
pool.shutdown(wait=False)
|
||||
|
||||
|
||||
class ExecutionService:
|
||||
"""Owns the engine's worker threads and the queue they read from."""
|
||||
@@ -93,12 +162,15 @@ class ExecutionService:
|
||||
self._inflight_lock = threading.Condition()
|
||||
# Entry ids claimed and still running, under _inflight_lock.
|
||||
self._active: set[str] = set()
|
||||
self.node_pool = ThreadPoolExecutor(
|
||||
max_workers=MAX_WORKERS if max_workers is None else max_workers,
|
||||
thread_name_prefix="node",
|
||||
# At most one reclaim batch waits here. The timer thread must keep
|
||||
# promoting and touching work while the consumer waits for capacity.
|
||||
self._reclaimed: deque[WorkItem] = deque()
|
||||
self._node_pools = _ReservedPool(
|
||||
MAX_WORKERS if max_workers is None else max_workers, "node"
|
||||
)
|
||||
self._cascade_pool = ThreadPoolExecutor(
|
||||
max_workers=self.max_cascades, thread_name_prefix="cascade"
|
||||
self.node_pool = self._node_pools.normal
|
||||
self._cascade_pools = _ReservedPool(
|
||||
self.max_cascades, "cascade", self._capacity_changed
|
||||
)
|
||||
self._consumer: threading.Thread | None = None
|
||||
self._timers: threading.Thread | None = None
|
||||
@@ -132,14 +204,18 @@ class ExecutionService:
|
||||
thread.join(timeout=5)
|
||||
self._consumer = None
|
||||
self._timers = None
|
||||
self._cascade_pool.shutdown(wait=False)
|
||||
self.node_pool.shutdown(wait=False)
|
||||
self._cascade_pools.shutdown()
|
||||
self._node_pools.shutdown()
|
||||
self.queue.close()
|
||||
|
||||
def bind(self, pipeline: Pipeline) -> None:
|
||||
"""Point the service at the pipeline it should execute against."""
|
||||
self._pipeline = pipeline
|
||||
|
||||
def _capacity_changed(self) -> None:
|
||||
with self._inflight_lock:
|
||||
self._inflight_lock.notify_all()
|
||||
|
||||
def pause_intake(self) -> None:
|
||||
"""Stop claiming, and wait for what is already claimed to finish.
|
||||
|
||||
@@ -182,6 +258,18 @@ class ExecutionService:
|
||||
free, due_only = self._await_capacity()
|
||||
if not free:
|
||||
continue
|
||||
with self._inflight_lock:
|
||||
recovered = [
|
||||
item
|
||||
for item in self._reclaimed
|
||||
if not due_only or item.not_before > 0
|
||||
][:free]
|
||||
for item in recovered:
|
||||
self._reclaimed.remove(item)
|
||||
if recovered:
|
||||
for item in recovered:
|
||||
self._dispatch(item)
|
||||
continue
|
||||
try:
|
||||
items = self.queue.claim(
|
||||
free,
|
||||
@@ -262,21 +350,23 @@ class ExecutionService:
|
||||
|
||||
if now - last_reclaim < RECLAIM_INTERVAL_S:
|
||||
continue
|
||||
with self._inflight_lock:
|
||||
if self._reclaimed or not self._intake.is_set():
|
||||
continue
|
||||
last_reclaim = now
|
||||
try:
|
||||
for item in self.queue.reclaim_stale(RECLAIM_IDLE_MS):
|
||||
recovered = self.queue.reclaim_stale(RECLAIM_IDLE_MS)
|
||||
for item in recovered:
|
||||
logger.info(
|
||||
"Reclaimed work for '%s' (delivery %d)",
|
||||
item.node,
|
||||
item.deliveries,
|
||||
)
|
||||
# Through the same gate the main loop uses: a reclaim can
|
||||
# return sixty-odd entries at once, and dispatching them
|
||||
# all would push `_inflight` far past `max_cascades` —
|
||||
# exactly the overcommit the gate exists to prevent.
|
||||
if not self._await_capacity()[0]:
|
||||
break
|
||||
self._dispatch(item)
|
||||
with self._inflight_lock:
|
||||
self._reclaimed.extend(recovered)
|
||||
self._active.update(
|
||||
item.entry_id for item in recovered if item.entry_id
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Could not reclaim stale work: %s", exc)
|
||||
|
||||
@@ -331,7 +421,9 @@ class ExecutionService:
|
||||
"""
|
||||
with self._inflight_lock:
|
||||
while not self._stop.is_set():
|
||||
free = self.max_cascades - self._inflight
|
||||
if not self._intake.is_set():
|
||||
return 0, False
|
||||
free, _ = self._cascade_pools.available()
|
||||
if free > 0:
|
||||
return free, False
|
||||
# Only once a slot has genuinely failed to free: the due lane
|
||||
@@ -340,7 +432,7 @@ class ExecutionService:
|
||||
# read every time the pool filled up.
|
||||
if self._inflight_lock.wait(0.5):
|
||||
continue
|
||||
reserve = self.max_cascades + DUE_RESERVE - self._inflight
|
||||
_, reserve = self._cascade_pools.available()
|
||||
if reserve > 0:
|
||||
return reserve, True
|
||||
return 0, False
|
||||
@@ -351,7 +443,12 @@ class ExecutionService:
|
||||
if item.entry_id:
|
||||
self._active.add(item.entry_id)
|
||||
try:
|
||||
self._cascade_pool.submit(self._handle, item)
|
||||
pool = (
|
||||
self._cascade_pools.due
|
||||
if item.not_before > 0
|
||||
else self._cascade_pools.normal
|
||||
)
|
||||
pool.submit(self._handle, item)
|
||||
except RuntimeError:
|
||||
# Pool already shutting down.
|
||||
self._done(item)
|
||||
@@ -386,7 +483,7 @@ class ExecutionService:
|
||||
return item.node
|
||||
|
||||
def _handle(self, item: WorkItem) -> None:
|
||||
handled = True
|
||||
handled = False
|
||||
try:
|
||||
handled = self._run_item(item)
|
||||
except Exception:
|
||||
@@ -473,11 +570,12 @@ class ExecutionService:
|
||||
|
||||
replay = item.deliveries > 1
|
||||
emission = item.kind == "emission"
|
||||
node_pool = self._node_pools.due if item.not_before > 0 else self.node_pool
|
||||
try:
|
||||
if item.kind == "flush":
|
||||
# A rate-limit window ended; nothing to replay, only to let out.
|
||||
# A run of its own, so what it sets off is recorded somewhere.
|
||||
pipeline.flush(node, item.entry_id)
|
||||
pipeline.flush(node, item.entry_id, node_pool=node_pool)
|
||||
return True
|
||||
# An emission's values went into state when the node produced them;
|
||||
# this item carries them so its readers get the chunk that caused
|
||||
@@ -498,6 +596,7 @@ class ExecutionService:
|
||||
# Neither can say what changed, so neither filters on it.
|
||||
changed=None if replay or not item.outputs else published,
|
||||
overrides=item.outputs if emission else None,
|
||||
node_pool=node_pool,
|
||||
)
|
||||
finally:
|
||||
# Paired, or a cascade that raised — state backend gone, say — is a
|
||||
|
||||
@@ -394,13 +394,8 @@ class MetricsCollector:
|
||||
prune: bool,
|
||||
) -> None:
|
||||
with Session(engine) as session:
|
||||
for (flow, node, minute), agg in buckets.items():
|
||||
statement = insert(MetricBucket).values(
|
||||
flow=flow,
|
||||
node=node,
|
||||
bucket=datetime.fromtimestamp(minute, UTC),
|
||||
**agg,
|
||||
)
|
||||
if buckets:
|
||||
statement = insert(MetricBucket)
|
||||
# The same minute is written several times, so the counters add
|
||||
# and the maxima take whichever is larger. Columns are read by
|
||||
# subscript: `excluded.items` is the collection's own method.
|
||||
@@ -418,21 +413,36 @@ class MetricsCollector:
|
||||
name: func.max(col(getattr(MetricBucket, name)), new[name])
|
||||
for name in MAXIMA
|
||||
},
|
||||
)
|
||||
),
|
||||
[
|
||||
{
|
||||
"flow": flow,
|
||||
"node": node,
|
||||
"bucket": datetime.fromtimestamp(minute, UTC),
|
||||
**agg,
|
||||
}
|
||||
for (flow, node, minute), agg in buckets.items()
|
||||
],
|
||||
)
|
||||
|
||||
for run in runs:
|
||||
values = {k: v for k, v in run.items() if k != "started_ts"}
|
||||
statement = insert(FlowRun).values(**values)
|
||||
if runs:
|
||||
values = [
|
||||
{k: v for k, v in run.items() if k != "started_ts"} for run in runs
|
||||
]
|
||||
statement = insert(FlowRun)
|
||||
session.execute(
|
||||
statement.on_conflict_do_update(
|
||||
index_elements=["id"],
|
||||
set_={
|
||||
key: statement.excluded[key]
|
||||
for key in values
|
||||
for key in values[0]
|
||||
if key != "id"
|
||||
},
|
||||
)
|
||||
),
|
||||
values,
|
||||
# A redelivery clears finished_at. Include nulls in the
|
||||
# batch so open and finished runs share the same statement.
|
||||
execution_options={"render_nulls": True},
|
||||
)
|
||||
|
||||
session.add_all(pending)
|
||||
|
||||
@@ -21,10 +21,11 @@ import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from collections.abc import Callable, Iterator
|
||||
from concurrent.futures import Future, ThreadPoolExecutor, wait
|
||||
from concurrent.futures import Executor, Future, ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from functools import lru_cache
|
||||
from queue import SimpleQueue
|
||||
from typing import Any, Literal, Protocol
|
||||
|
||||
from pydantic import BaseModel, model_validator
|
||||
@@ -275,6 +276,7 @@ class Pipeline:
|
||||
"_paused",
|
||||
"_stepping",
|
||||
"_gate_lock",
|
||||
"_gate_revision",
|
||||
"_graph_lock",
|
||||
"_queue",
|
||||
"_node_pool",
|
||||
@@ -294,7 +296,7 @@ class Pipeline:
|
||||
initial_values: dict[str, Any] | None = None,
|
||||
disabled_flows: set[str] | None = None,
|
||||
work_queue: WorkQueue | None = None,
|
||||
node_pool: ThreadPoolExecutor | None = None,
|
||||
node_pool: Executor | None = None,
|
||||
observer: Callable[[NodeOutcome], None] | None = None,
|
||||
emission_observer: Callable[[str, dict[str, Any]], None] | None = None,
|
||||
run_cache: RunCacheLookup | None = None,
|
||||
@@ -307,6 +309,7 @@ class Pipeline:
|
||||
self._paused: set[str] = set()
|
||||
self._stepping: set[str] = set()
|
||||
self._gate_lock = threading.Lock()
|
||||
self._gate_revision = 0
|
||||
# Held only while the graph maps are read or swapped together, never
|
||||
# across an await, a node call or any I/O. Reentrant because the lazy
|
||||
# `edges` build re-enters through `_graph`. It is never nested with
|
||||
@@ -448,6 +451,7 @@ class Pipeline:
|
||||
with self._gate_lock:
|
||||
self._paused.discard(flow)
|
||||
self._stepping.discard(flow)
|
||||
self._gate_revision += 1
|
||||
|
||||
def remove_flow(self, flow: str) -> None:
|
||||
"""Take a deleted flow out of the graph."""
|
||||
@@ -742,7 +746,9 @@ class Pipeline:
|
||||
self._state[self._flush_key(node.id)] = at
|
||||
self.defer(node, {}, at - now, kind="flush", parent=entry_id)
|
||||
|
||||
def flush(self, node: Node, entry_id: str = "") -> None:
|
||||
def flush(
|
||||
self, node: Node, entry_id: str = "", *, node_pool: Executor | None = None
|
||||
) -> None:
|
||||
"""Let through what this node's rate limits held back.
|
||||
|
||||
Runs when a window ends: the last value a limited output kept back is
|
||||
@@ -766,7 +772,9 @@ class Pipeline:
|
||||
# nothing came out and there is nothing downstream to wake.
|
||||
published = self.apply_outputs(node, held, entry_id)
|
||||
if published:
|
||||
self.run_downstream(node, entry_id=entry_id, changed=published)
|
||||
self.run_downstream(
|
||||
node, entry_id=entry_id, changed=published, node_pool=node_pool
|
||||
)
|
||||
if any(spec.interval > 0 for spec in node.requires.values()):
|
||||
# A wake-up that was held back, not a value that just changed —
|
||||
# so this one deliberately walks everything reachable.
|
||||
@@ -775,6 +783,7 @@ class Pipeline:
|
||||
self._state,
|
||||
check_ready=True,
|
||||
entry_id=entry_id,
|
||||
node_pool=node_pool,
|
||||
)
|
||||
|
||||
def _increment_message_versions(self, outputs: dict[str, Any]) -> None:
|
||||
@@ -1252,7 +1261,9 @@ class Pipeline:
|
||||
|
||||
def set_disabled(self, flows: set[str]) -> None:
|
||||
"""Which flows are stopped. Swapped whole, so a reader never sees half."""
|
||||
self._disabled = frozenset(flows)
|
||||
with self._gate_lock:
|
||||
self._disabled = frozenset(flows)
|
||||
self._gate_revision += 1
|
||||
|
||||
def paused_flows(self) -> list[str]:
|
||||
with self._gate_lock:
|
||||
@@ -1262,11 +1273,13 @@ class Pipeline:
|
||||
"""Hold this flow's nodes. Values still arrive; nothing acts on them."""
|
||||
with self._gate_lock:
|
||||
self._paused.add(flow)
|
||||
self._gate_revision += 1
|
||||
self._publish({"type": "flow_paused", "flow": flow, "paused": True})
|
||||
|
||||
def resume(self, flow: str) -> None:
|
||||
with self._gate_lock:
|
||||
self._paused.discard(flow)
|
||||
self._gate_revision += 1
|
||||
self._publish({"type": "flow_paused", "flow": flow, "paused": False})
|
||||
# Whatever was held back is free to run now.
|
||||
self._execute_parallel(self.flow_nodes(flow), self._state, check_ready=True)
|
||||
@@ -1280,11 +1293,13 @@ class Pipeline:
|
||||
"""
|
||||
with self._gate_lock:
|
||||
self._stepping.add(flow)
|
||||
self._gate_revision += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with self._gate_lock:
|
||||
self._stepping.discard(flow)
|
||||
self._gate_revision += 1
|
||||
|
||||
def _gate_blocks(self, node: Node) -> bool:
|
||||
"""Is this node's flow held back from executing?"""
|
||||
@@ -1311,6 +1326,7 @@ class Pipeline:
|
||||
replay: bool = False,
|
||||
changed: set[str] | None = None,
|
||||
overrides: dict[str, Any] | None = None,
|
||||
node_pool: Executor | None = None,
|
||||
) -> StateBackend:
|
||||
"""Execute nodes concurrently, scheduling each as its inputs arrive.
|
||||
|
||||
@@ -1344,9 +1360,11 @@ class Pipeline:
|
||||
# Copied: the caller's set is not this wave's to grow.
|
||||
fresh = set(changed) if changed is not None else None
|
||||
|
||||
submitted: set[Node] = set()
|
||||
skipped: set[Node] = set()
|
||||
node_futures: dict[Node, Future[dict[str, Any] | None]] = {}
|
||||
ready_nodes = deque(n for n, count in in_degree.items() if count == 0)
|
||||
blocked: set[Node] = set()
|
||||
gate_revision = self._gate_revision
|
||||
node_futures: dict[Future[dict[str, Any] | None], Node] = {}
|
||||
finished: SimpleQueue[Future[dict[str, Any] | None]] = SimpleQueue()
|
||||
|
||||
def untouched(n: Node) -> bool:
|
||||
"""Did this wave publish nothing this node waits on?"""
|
||||
@@ -1356,108 +1374,72 @@ class Pipeline:
|
||||
)
|
||||
|
||||
def complete(n: Node) -> None:
|
||||
"""Count a node as done without running it, freeing its consumers."""
|
||||
skipped.add(n)
|
||||
"""Release consumers when their last dependency finishes or skips."""
|
||||
for consumer in edges[n]:
|
||||
if consumer in target_nodes:
|
||||
in_degree[consumer] -= 1
|
||||
if in_degree[consumer] == 0:
|
||||
ready_nodes.append(consumer)
|
||||
|
||||
def submit_ready(executor: ThreadPoolExecutor) -> None:
|
||||
# To a fixpoint: completing a node without running it frees its
|
||||
# consumers, and one this pass has already walked past would
|
||||
# otherwise be stranded — there is no future outstanding to bring
|
||||
# the drain loop back round for it.
|
||||
progressed = True
|
||||
while progressed:
|
||||
progressed = False
|
||||
for n in target_nodes:
|
||||
if n in submitted or n in skipped:
|
||||
continue
|
||||
# Gate before the readiness check, so a held-back node does
|
||||
# not spend the synchronous claim it needs once it may run.
|
||||
if self._gate_blocks(n) or in_degree[n] != 0:
|
||||
continue
|
||||
if n.passive:
|
||||
# Something outside publishes for it; a run is not it.
|
||||
def submit_ready(executor: Executor) -> None:
|
||||
nonlocal gate_revision
|
||||
with self._gate_lock:
|
||||
revision = self._gate_revision
|
||||
if blocked and revision != gate_revision:
|
||||
# A flow can resume while another branch of this wave runs.
|
||||
# Revisit only its held candidates, and only if a gate moved.
|
||||
ready_nodes.extend(blocked)
|
||||
blocked.clear()
|
||||
gate_revision = revision
|
||||
while ready_nodes:
|
||||
n = ready_nodes.popleft()
|
||||
# Gate before readiness, so a pause spends no synchronous claim.
|
||||
if self._gate_blocks(n):
|
||||
blocked.add(n)
|
||||
continue
|
||||
if n.passive:
|
||||
complete(n)
|
||||
continue
|
||||
if fresh is not None and untouched(n):
|
||||
complete(n)
|
||||
continue
|
||||
ready_inputs: dict[str, Any] | None = None
|
||||
if check_ready:
|
||||
ready, ready_inputs = self._is_node_ready(n, state, entry_id)
|
||||
if not ready:
|
||||
# Its old output remains usable by a consumer whose
|
||||
# other producer did publish during this wave.
|
||||
complete(n)
|
||||
progressed = True
|
||||
continue
|
||||
if fresh is not None and untouched(n):
|
||||
# Ahead of the readiness check, so an unchanged node
|
||||
# spends neither a delivery stamp nor a synchronous
|
||||
# claim on a run it is not going to make.
|
||||
complete(n)
|
||||
progressed = True
|
||||
continue
|
||||
ready_inputs: dict[str, Any] | None = None
|
||||
if check_ready:
|
||||
# The values are kept whether or not the answer is yes:
|
||||
# a node that may run reads them below instead of
|
||||
# asking state for the same keys again.
|
||||
ready, ready_inputs = self._is_node_ready(n, state, entry_id)
|
||||
if not ready:
|
||||
# Completed, not merely passed over: a later trigger
|
||||
# may still make it ready, but not running is not
|
||||
# the same as blocking. Whatever it last published
|
||||
# is in state, and a consumer reading that alongside
|
||||
# something this wave *did* publish has every right
|
||||
# to run on the pair.
|
||||
#
|
||||
# Passing over it held that consumer for the whole
|
||||
# wave, and a node that can never be ready — an
|
||||
# input nothing has ever published — took its
|
||||
# consumers out for good. On this house
|
||||
# `plugs.pump_run` waits on a watering pulse that
|
||||
# only exists at 02:00, and it sits upstream of
|
||||
# `dmx.switches`: every boiler, plug and appliance
|
||||
# command reached its own message and stopped there,
|
||||
# reaching the DMX universe only in the occasional
|
||||
# wave the pump node happened not to be part of.
|
||||
complete(n)
|
||||
progressed = True
|
||||
continue
|
||||
if replay and entry_id and self._already_done(entry_id, n):
|
||||
# Its side effect happened on an earlier delivery; its
|
||||
# outputs are still in state, so downstream carries on.
|
||||
complete(n)
|
||||
progressed = True
|
||||
continue
|
||||
submitted.add(n)
|
||||
node_futures[n] = executor.submit(
|
||||
self._execute_node, n, state, entry_id, overrides, ready_inputs
|
||||
)
|
||||
if replay and entry_id and self._already_done(entry_id, n):
|
||||
complete(n)
|
||||
continue
|
||||
future = executor.submit(
|
||||
self._execute_node, n, state, entry_id, overrides, ready_inputs
|
||||
)
|
||||
node_futures[future] = n
|
||||
future.add_done_callback(finished.put)
|
||||
|
||||
def drain(executor: ThreadPoolExecutor) -> None:
|
||||
def drain(executor: Executor) -> None:
|
||||
submit_ready(executor)
|
||||
|
||||
while node_futures:
|
||||
done, _ = wait(node_futures.values(), return_when="FIRST_COMPLETED")
|
||||
completed = [n for n, f in node_futures.items() if f in done]
|
||||
# A completion queue also avoids scanning every outstanding
|
||||
# future whenever one node finishes.
|
||||
future = finished.get()
|
||||
n = node_futures.pop(future)
|
||||
result = future.result()
|
||||
if result is not None and fresh is not None:
|
||||
fresh.update(result)
|
||||
# Silent, failed and throttled producers still release their
|
||||
# consumers. Fresh inputs decide whether those consumers run.
|
||||
complete(n)
|
||||
submit_ready(executor)
|
||||
|
||||
for n in completed:
|
||||
result = node_futures.pop(n).result()
|
||||
# Before the decrement: a consumer freed by this node is
|
||||
# judged on what it just published.
|
||||
if result is not None and fresh is not None:
|
||||
fresh.update(result)
|
||||
# Decremented whatever it returned. A node that published
|
||||
# nothing — rate limited, unchanged, or failed — has not
|
||||
# *blocked* its consumers; it has merely given them nothing
|
||||
# to read, and `untouched` above is the accurate test of
|
||||
# that. Holding the count instead made a quiet producer
|
||||
# veto a noisy sibling: `dmx.switches` reads both boilers
|
||||
# through report-by-exception nodes, so the kitchen one
|
||||
# staying silent — which is almost always — kept the main
|
||||
# boiler's command from ever reaching the DMX universe.
|
||||
for consumer in edges[n]:
|
||||
if consumer in target_nodes:
|
||||
in_degree[consumer] -= 1
|
||||
submit_ready(executor)
|
||||
|
||||
if self._node_pool is not None:
|
||||
pool = node_pool if node_pool is not None else self._node_pool
|
||||
if pool is not None:
|
||||
# The execution service owns a long-lived pool; building one per
|
||||
# wave is what used to spawn threads without bound under load.
|
||||
drain(self._node_pool)
|
||||
drain(pool)
|
||||
else:
|
||||
with ThreadPoolExecutor(max_workers=self._max_workers) as executor:
|
||||
drain(executor)
|
||||
@@ -1519,6 +1501,7 @@ class Pipeline:
|
||||
replay: bool = False,
|
||||
changed: set[str] | None = None,
|
||||
overrides: dict[str, Any] | None = None,
|
||||
node_pool: Executor | None = None,
|
||||
) -> StateBackend:
|
||||
"""Run everything downstream of a node that has just published.
|
||||
|
||||
@@ -1552,6 +1535,7 @@ class Pipeline:
|
||||
replay=replay,
|
||||
changed=changed,
|
||||
overrides=overrides,
|
||||
node_pool=node_pool,
|
||||
)
|
||||
|
||||
def trigger(
|
||||
@@ -1796,6 +1780,12 @@ class Pipeline:
|
||||
# Losing the value outright would be worse than running it here.
|
||||
self._run_here(node, outputs, cause="external", delivered=kind == "emission")
|
||||
|
||||
def value(self, name: str) -> dict[str, Any]:
|
||||
"""One message and its timestamp, read together without listing state."""
|
||||
stamp = self._timestamp_key(name)
|
||||
found = self._state.get_multi([name, stamp])
|
||||
return {"value": found.get(name), "ts": found.get(stamp)}
|
||||
|
||||
def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]:
|
||||
"""Last value and timestamp of every message, optionally one flow's.
|
||||
|
||||
|
||||
@@ -53,6 +53,23 @@ STALE_CONSUMER_IDLE_MS = 3_600_000
|
||||
# count, since redelivery happens seconds after the claim and never later.
|
||||
DONE_MARKERS = 10_000
|
||||
|
||||
# A promotion is one Redis operation, including when the caller loses its
|
||||
# connection before hearing the result. Write first: Lua does not roll back a
|
||||
# command error, so a refused XADD must leave its timer in the delayed set.
|
||||
_MOVE_DUE = """
|
||||
local due = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[1], 'LIMIT', 0, 100)
|
||||
for _, raw in ipairs(due) do
|
||||
local fields = {}
|
||||
for key, value in pairs(cjson.decode(raw)) do
|
||||
fields[#fields + 1] = key
|
||||
fields[#fields + 1] = value
|
||||
end
|
||||
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[2], '*', unpack(fields))
|
||||
redis.call('ZREM', KEYS[1], raw)
|
||||
end
|
||||
return #due
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkItem:
|
||||
@@ -446,6 +463,7 @@ class RedisWorkQueue(WorkQueue):
|
||||
# finding them with a keyspace scan walked every state and idempotency
|
||||
# key in the database, twice per health poll.
|
||||
self._parked_flows_key = f"{namespace}:__parked_flows__"
|
||||
self._move_due = self._redis.register_script(_MOVE_DUE)
|
||||
self._ensure_group()
|
||||
self._drop_stale_consumers()
|
||||
|
||||
@@ -619,36 +637,14 @@ class RedisWorkQueue(WorkQueue):
|
||||
]
|
||||
|
||||
def move_due(self, now: float) -> int:
|
||||
due = cast(
|
||||
list[str],
|
||||
self._redis.zrangebyscore(self._delayed_key, "-inf", now, start=0, num=100),
|
||||
"""Move up to 100 timers into the due lane atomically."""
|
||||
return cast(
|
||||
int,
|
||||
self._move_due(
|
||||
keys=[self._delayed_key, self._due_stream],
|
||||
args=[now, STREAM_MAXLEN],
|
||||
),
|
||||
)
|
||||
if not due:
|
||||
return 0
|
||||
# Whoever removes it owns it: a second engine gets 0 for that member and
|
||||
# leaves it alone. One round trip for the batch rather than one each,
|
||||
# which matters because this runs every second.
|
||||
pipe = self._redis.pipeline()
|
||||
for raw in due:
|
||||
pipe.zrem(self._delayed_key, raw)
|
||||
claimed = cast(list[int], pipe.execute())
|
||||
|
||||
pipe = self._redis.pipeline()
|
||||
moved = 0
|
||||
for raw, owned in zip(due, claimed, strict=True):
|
||||
if owned:
|
||||
# Into the due lane, which `claim` reads first: what has waited
|
||||
# out a deadline should not then queue behind a backlog.
|
||||
pipe.xadd(
|
||||
self._due_stream,
|
||||
cast(Any, orjson.loads(raw)),
|
||||
maxlen=STREAM_MAXLEN,
|
||||
approximate=True,
|
||||
)
|
||||
moved += 1
|
||||
if moved:
|
||||
pipe.execute()
|
||||
return moved
|
||||
|
||||
def dead_letter(self, item: WorkItem, reason: str) -> None:
|
||||
fields = item.to_fields()
|
||||
|
||||
+28
-3
@@ -15,7 +15,7 @@ import mimetypes
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from starlette.middleware.gzip import GZipMiddleware
|
||||
from starlette.datastructures import Headers
|
||||
from starlette.responses import FileResponse
|
||||
from starlette.types import ASGIApp, Receive, Scope, Send
|
||||
|
||||
@@ -26,6 +26,24 @@ API_PREFIXES = ("/api/", "/hooks/", "/mcp", "/.well-known/", "/docs", "/redoc")
|
||||
mimetypes.add_type("application/manifest+json", ".webmanifest")
|
||||
|
||||
|
||||
def _accepts_gzip(header: str) -> bool:
|
||||
"""An explicit refusal overrides a wildcard, including on a cached asset."""
|
||||
qualities: dict[str, float] = {}
|
||||
for entry in header.lower().split(","):
|
||||
encoding, *parameters = entry.strip().split(";")
|
||||
quality = 1.0
|
||||
for parameter in parameters:
|
||||
key, _, value = parameter.strip().partition("=")
|
||||
if key == "q":
|
||||
try:
|
||||
quality = float(value)
|
||||
except ValueError:
|
||||
quality = 0.0
|
||||
qualities[encoding.strip()] = quality if 0.0 <= quality <= 1.0 else 0.0
|
||||
gzip = qualities.get("gzip", qualities.get("*", 0.0))
|
||||
return gzip > 0 and gzip >= qualities.get("identity", 0.0)
|
||||
|
||||
|
||||
def content_security_policy(index_html: str) -> str:
|
||||
"""The policy for the dashboard, admitting its inline scripts by hash.
|
||||
|
||||
@@ -84,5 +102,12 @@ class Dashboard:
|
||||
# assets, so a stale copy pins the browser to the previous build.
|
||||
file = self.index
|
||||
headers = {**self.headers, "Cache-Control": "no-cache"}
|
||||
response = FileResponse(file, headers=headers)
|
||||
await GZipMiddleware(response, minimum_size=1024)(scope, receive, send)
|
||||
media_type = mimetypes.guess_type(file.name)[0] or "text/plain"
|
||||
compressed = file.with_name(file.name + ".gz").resolve()
|
||||
if compressed.is_relative_to(self.dist) and compressed.is_file():
|
||||
headers["Vary"] = "Accept-Encoding"
|
||||
if _accepts_gzip(Headers(scope=scope).get("accept-encoding", "")):
|
||||
file = compressed
|
||||
headers["Content-Encoding"] = "gzip"
|
||||
response = FileResponse(file, headers=headers, media_type=media_type)
|
||||
await response(scope, receive, send)
|
||||
|
||||
Reference in New Issue
Block a user