Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s

This commit is contained in:
2026-10-29 14:24:17 +01:00
parent 20a0853f69
commit 75b3a93094
25 changed files with 1316 additions and 301 deletions
+24 -10
View File
@@ -1700,8 +1700,8 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
await websocket.send_text(orjson.dumps(payload).decode()) await websocket.send_text(orjson.dumps(payload).decode())
return list(payload.get("demand", [])) return list(payload.get("demand", []))
async def catch_up() -> bool: async def rescope() -> bool:
"""Rescope a panel and send the snapshot again, demand included. """Recheck a panel and its demand before forwarding retained events.
False once a device whose credential stopped standing is closed with False once a device whose credential stopped standing is closed with
1008, which sends it to check: otherwise its frames would still be 1008, which sends it to check: otherwise its frames would still be
@@ -1722,12 +1722,18 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
device_def = _device_def(websocket.app, device) device_def = _device_def(websocket.app, device)
wanted &= only wanted &= only
demand.want(viewer, wanted) demand.want(viewer, wanted)
return True
async def catch_up() -> bool:
if not await rescope():
return False
await send_snapshot() await send_snapshot()
return True return True
told = await send_snapshot()
async with event_bus.subscribe() as queue: async with event_bus.subscribe() as queue:
# Listen before reading state: a value changing while the snapshot is
# sent must follow it on the socket, even if it never changes again.
told = await send_snapshot()
receiver = asyncio.create_task(_receive(websocket)) receiver = asyncio.create_task(_receive(websocket))
# Where a live view this socket is one end of reaches it. Its own # Where a live view this socket is one end of reaches it. Its own
# outbox rather than the bus's queue, which drops when full. # outbox rather than the bus's queue, which drops when full.
@@ -1741,6 +1747,7 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
await websocket.send_text(demand_frame(ports)) await websocket.send_text(demand_frame(ports))
live_view.sessions.attach(device, endpoint) live_view.sessions.attach(device, endpoint)
outbox = asyncio.create_task(endpoint.next()) outbox = asyncio.create_task(endpoint.next())
catching_up = False
try: try:
while True: while True:
sender = asyncio.create_task(queue.get()) sender = asyncio.create_task(queue.get())
@@ -1845,10 +1852,13 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
except asyncio.QueueEmpty: except asyncio.QueueEmpty:
break break
# A panel this far behind lost events the bus dropped for it: # Every client catches up after overflow. A lost event may be
# its demand, a dashboard published or its own unpairing. It # a value that never changes again, or a device's unpairing.
# catches up once this batch is out, as after a dashboard. lost = event_bus.dropped(queue)
resync = only is not None and event_bus.dropped(queue) catching_up = catching_up or lost
if lost and not await rescope():
return
resync = False
if only is None: if only is None:
fresh = hidden_from(user, websocket.app) fresh = hidden_from(user, websocket.app)
if fresh != hidden: if fresh != hidden:
@@ -1883,8 +1893,12 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
for frame in media_frames(out, wanted, store): for frame in media_frames(out, wanted, store):
await websocket.send_bytes(frame) await websocket.send_bytes(frame)
await _send(websocket, out) await _send(websocket, out)
if resync and not await catch_up(): # Send retained events before the recovery snapshot: values
return # left in the queue would otherwise overwrite its newer state.
if resync or (catching_up and queue.empty()):
if not await catch_up():
return
catching_up = catching_up and not queue.empty()
except (WebSocketDisconnect, RuntimeError): except (WebSocketDisconnect, RuntimeError):
# A peer that goes away mid-send takes the RuntimeError route # A peer that goes away mid-send takes the RuntimeError route
# ("websocket.send after websocket.close") rather than the clean # ("websocket.send after websocket.close") rather than the clean
+3 -5
View File
@@ -19,7 +19,6 @@ from fluksio.api.deps import (
get_current_user, get_current_user,
) )
from fluksio.flow.devices import DeviceDef, DeviceStore from fluksio.flow.devices import DeviceDef, DeviceStore
from fluksio.flow.messages import flow_of
from fluksio.flow.pipeline import ValueSource from fluksio.flow.pipeline import ValueSource
from fluksio.flow.state import as_number from fluksio.flow.state import as_number
@@ -118,7 +117,7 @@ def _port_source(request: Request, name: str) -> ValueSource | None:
@router.post("/{name}", response_model=MessageValue) @router.post("/{name}", response_model=MessageValue)
async def publish_message( def publish_message(
name: str, body: PublishRequest, controller: FlowControllerDep, request: Request name: str, body: PublishRequest, controller: FlowControllerDep, request: Request
) -> Any: ) -> Any:
"""Put a value into the graph, as a dashboard control does. """Put a value into the graph, as a dashboard control does.
@@ -133,7 +132,7 @@ async def publish_message(
detail=body.source_detail, detail=body.source_detail,
) )
try: try:
await run_in_threadpool(controller.publish_message, name, body.value, source) controller.publish_message(name, body.value, source)
except KeyError: except KeyError:
raise HTTPException( raise HTTPException(
status_code=404, detail=f"No flow declares a message named '{name}'" status_code=404, detail=f"No flow declares a message named '{name}'"
@@ -141,8 +140,7 @@ async def publish_message(
except TypeError as exc: except TypeError as exc:
raise HTTPException(status_code=422, detail=str(exc)) raise HTTPException(status_code=422, detail=str(exc))
values = controller.values(flow_of(name)) current = controller.value(name)
current = values.get(name, {})
return MessageValue(name=name, value=current.get("value"), ts=current.get("ts")) return MessageValue(name=name, value=current.get("value"), ts=current.get("ts"))
+42 -32
View File
@@ -722,16 +722,6 @@ class CloudConnector:
# The cameras a person may not see, looked at again per event. # The cameras a person may not see, looked at again per event.
hidden = hidden_from(user, self._app, device=only is not None) hidden = hidden_from(user, self._app, device=only is not None)
controller = getattr(self._app.state, "flow_controller", None) controller = getattr(self._app.state, "flow_controller", None)
told: list[str] = []
if controller is not None:
# A device here sends its frames over HTTP: the tunnel carries
# text only.
snapshot = snapshot_payload(controller, only, device, "http", hidden)
told = list(snapshot.get("demand", []))
await socket.send(
_dump({"op": "ws_msg", "id": stream_id, "text": _dump(snapshot)})
)
# The media this client's tiles are drawing count as demand on the # The media this client's tiles are drawing count as demand on the
# devices sending them, as a direct socket's do. The bytes themselves # devices sending them, as a direct socket's do. The bytes themselves
# are fetched one by one through the portal. # are fetched one by one through the portal.
@@ -743,6 +733,20 @@ class CloudConnector:
outbox: asyncio.Task[str] | None = None outbox: asyncio.Task[str] | None = None
sender: asyncio.Task[dict[str, Any]] | None = None sender: asyncio.Task[dict[str, Any]] | None = None
async def rescope() -> bool:
nonlocal only, wanted
if only is not None:
if device and revoked(token, self._app):
await socket.send(
_dump({"op": "ws_close", "id": stream_id, "code": 1008})
)
return False
# A deleted panel scopes to nothing, never to every message.
only = panel_scope(token, self._app) or set()
wanted &= only
demand.want(viewer, wanted)
return True
def heard(text: str) -> None: def heard(text: str) -> None:
nonlocal wanted nonlocal wanted
asked = wanted_names(text, only, hidden) asked = wanted_names(text, only, hidden)
@@ -768,6 +772,19 @@ class CloudConnector:
live_view.sessions.attach(device, endpoint) live_view.sessions.attach(device, endpoint)
try: try:
async with event_bus.subscribe() as queue: async with event_bus.subscribe() as queue:
# Subscribe before the snapshot so changes during its send
# cannot fall between the snapshot and the live stream.
told: list[str] = []
if controller is not None:
snapshot = snapshot_payload(
controller, only, device, "http", hidden
)
told = list(snapshot.get("demand", []))
await socket.send(
_dump(
{"op": "ws_msg", "id": stream_id, "text": _dump(snapshot)}
)
)
outbox = asyncio.create_task(endpoint.next()) outbox = asyncio.create_task(endpoint.next())
if device and (ports := demand.ports(device)) != told: if device and (ports := demand.ports(device)) != told:
await socket.send( await socket.send(
@@ -779,6 +796,7 @@ class CloudConnector:
} }
) )
) )
catching_up = False
while True: while True:
sender = asyncio.create_task(queue.get()) sender = asyncio.create_task(queue.get())
done, _ = await asyncio.wait( done, _ = await asyncio.wait(
@@ -796,11 +814,13 @@ class CloudConnector:
if sender not in done: if sender not in done:
continue continue
event = sender.result() event = sender.result()
# Behind far enough to lose events, a device catches up # A browser needs the same recovery as a device after
# as after a dashboard: the lost one may have been its # overflow: the lost value may never be published again.
# demand or its unpairing. A person whose cameras opened lost = event_bus.dropped(queue)
# or closed is sent what may be seen now. catching_up = catching_up or lost
resync = only is not None and event_bus.dropped(queue) if lost and not await rescope():
return
resync = False
if only is None: if only is None:
fresh = hidden_from(user, self._app) fresh = hidden_from(user, self._app)
if fresh != hidden: if fresh != hidden:
@@ -824,24 +844,10 @@ class CloudConnector:
{"op": "ws_msg", "id": stream_id, "text": _dump(event)} {"op": "ws_msg", "id": stream_id, "text": _dump(event)}
) )
) )
if not resync: if not resync and not (catching_up and queue.empty()):
continue continue
if only is not None: if not await rescope():
if device and revoked(token, self._app): return
# Unpaired or deleted: 1008 sends it to check.
await socket.send(
_dump({"op": "ws_close", "id": stream_id, "code": 1008})
)
return
# What this screen shows may have just changed under
# it. Rescope, then resend the snapshot so a dashboard
# it has only now been assigned draws values instead of
# blanks. ``or set()`` is the point: a panel that was
# deleted scopes to nothing, and None would widen this
# socket to everything on the bus.
only = panel_scope(token, self._app) or set()
wanted &= only
demand.want(viewer, wanted)
if controller is not None: if controller is not None:
catch_up = snapshot_payload( catch_up = snapshot_payload(
controller, only, device, "http", hidden controller, only, device, "http", hidden
@@ -855,6 +861,10 @@ class CloudConnector:
} }
) )
) )
# The snapshot follows the retained backlog. If sending
# it queued more events, finish those before recovering
# again so an older event cannot leave the client stale.
catching_up = catching_up and not queue.empty()
if kind == "dashboard_changed" and only is not None: if kind == "dashboard_changed" and only is not None:
# After the snapshot, as the device socket sends it. # After the snapshot, as the device socket sends it.
await socket.send( await socket.send(
+3
View File
@@ -1545,6 +1545,9 @@ class FlowController:
return _short_error(exc) return _short_error(exc)
return None return None
def value(self, name: str) -> dict[str, Any]:
return self.pipeline.value(name) if self.pipeline else {}
def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]: def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]:
return self.pipeline.values(flow) if self.pipeline else {} return self.pipeline.values(flow) if self.pipeline else {}
+112 -26
View File
@@ -1,9 +1,10 @@
"""Event bus bridging the engine's worker threads to async subscribers. """Event bus bridging the engine's worker threads to async subscribers.
Nodes execute in a thread pool; websocket clients live on the event loop. Nodes execute in a thread pool; websocket clients live on the event loop.
Publishers are therefore thread-safe and never block: a subscriber that cannot Publishers are therefore thread-safe and never wait for subscribers. Both
keep up loses its oldest queued events rather than stalling the engine, and staging and subscriber queues are bounded, retaining errors and control events
can ask whether it did (``dropped``) to catch up another way. ahead of routine traffic. Subscribers can ask whether they lost events
(``dropped``) to catch up another way.
""" """
from __future__ import annotations from __future__ import annotations
@@ -19,24 +20,88 @@ from typing import Any
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
QUEUE_SIZE = 256 QUEUE_SIZE = 256
DISPATCH_BUDGET = 1024
LOG_HISTORY = 400 LOG_HISTORY = 400
#: How many of a message's last artifact digests are remembered. #: How many of a message's last artifact digests are remembered.
RECENT_DIGESTS = 8 RECENT_DIGESTS = 8
def _critical(event: dict[str, Any]) -> bool:
"""Keep failures, lifecycle changes and unknown events ahead of telemetry."""
kind = event.get("type")
if kind == "node_log":
return event.get("level") == "error"
return kind not in {
"message_value",
"node_started",
"node_executed",
"work_latency",
"run_metric",
}
class _EventBuffer:
"""Two FIFO lanes allow constant-time eviction without reordering events."""
def __init__(self) -> None:
self._ordinary: deque[tuple[int, dict[str, Any]]] = deque()
self._critical: deque[tuple[int, dict[str, Any]]] = deque()
self._sequence = 0
def __len__(self) -> int:
return len(self._ordinary) + len(self._critical)
def append(self, event: dict[str, Any]) -> None:
lane = self._critical if _critical(event) else self._ordinary
lane.append((self._sequence, event))
self._sequence += 1
def popleft(self) -> dict[str, Any]:
lane = self._critical
if self._ordinary and (not lane or self._ordinary[0][0] < lane[0][0]):
lane = self._ordinary
return lane.popleft()[1]
def make_room(self, event: dict[str, Any]) -> bool:
"""Evict the oldest ordinary event, or keep a full critical backlog."""
if self._ordinary:
self._ordinary.popleft()
elif _critical(event):
self._critical.popleft()
else:
return False
return True
class _EventQueue(asyncio.Queue[dict[str, Any]]):
def _init(self, maxsize: int) -> None:
self._queue = _EventBuffer()
def offer(self, event: dict[str, Any]) -> bool:
"""Enqueue without waiting and report any loss, including rejection."""
lost = self.full()
if not lost or self._queue.make_room(event):
self.put_nowait(event)
return lost
class EventBus: class EventBus:
"""Fan-out of engine events to any number of async subscribers.""" """Fan-out of engine events to any number of async subscribers."""
def __init__(self) -> None: def __init__(self) -> None:
self._loop: asyncio.AbstractEventLoop | None = None self._loop: asyncio.AbstractEventLoop | None = None
self._subscribers: set[asyncio.Queue[dict[str, Any]]] = set() self._subscribers: set[_EventQueue] = set()
# Events published since the last flush was scheduled. A cascade # Events published since the last flush was scheduled. A cascade
# publishes 13-16 of them and each used to cost its own # publishes 13-16 of them and each used to cost its own
# `call_soon_threadsafe`; they now cost one between two turns of the # `call_soon_threadsafe`; they now cost one between two turns of the
# loop, which is what a wave is. # loop, which is what a wave is.
self._pending: list[dict[str, Any]] = [] self._pending = _EventBuffer()
self._pending_dropped = False
self._pending_lock = threading.Lock() self._pending_lock = threading.Lock()
self._flush_scheduled = False self._flush_scheduled = False
self._batch: list[dict[str, Any]] = []
self._batch_dropped = False
self._targets: deque[_EventQueue] = deque()
# Kept whether or not anyone is listening, so opening the log panel # Kept whether or not anyone is listening, so opening the log panel
# shows what just happened rather than an empty box. # shows what just happened rather than an empty box.
self.recent_logs: deque[dict[str, Any]] = deque(maxlen=LOG_HISTORY) self.recent_logs: deque[dict[str, Any]] = deque(maxlen=LOG_HISTORY)
@@ -82,7 +147,12 @@ class EventBus:
if loop is None or not self._subscribers: if loop is None or not self._subscribers:
return return
with self._pending_lock: with self._pending_lock:
self._pending.append(event) room = True
if len(self._pending) == QUEUE_SIZE:
self._pending_dropped = True
room = self._pending.make_room(event)
if room:
self._pending.append(event)
if self._flush_scheduled: if self._flush_scheduled:
# A flush is already on its way and has not run yet, so it # A flush is already on its way and has not run yet, so it
# will find this event too. # will find this event too.
@@ -93,33 +163,49 @@ class EventBus:
except RuntimeError: except RuntimeError:
# Loop already closed — shutting down. # Loop already closed — shutting down.
with self._pending_lock: with self._pending_lock:
self._pending.clear() self._pending = _EventBuffer()
self._pending_dropped = False
self._flush_scheduled = False self._flush_scheduled = False
def _dispatch(self) -> None: def _dispatch(self) -> None:
"""Hand everything published since the last turn to every subscriber.""" """Deliver bounded work, yielding between batches or subscriber groups."""
with self._pending_lock: if not self._targets:
batch = self._pending with self._pending_lock:
self._pending = [] self._batch = [
self._flush_scheduled = False self._pending.popleft() for _ in range(len(self._pending))
if not batch: ]
return self._batch_dropped = self._pending_dropped
for queue in self._subscribers: self._pending_dropped = False
for event in batch: self._targets = deque(self._subscribers)
if queue.full(): if self._batch:
# Drop the oldest so a slow client never blocks the engine. # A subscriber receives the whole bounded batch before it resumes,
try: # so its catch-up snapshot cannot precede the rest of that batch.
queue.get_nowait() count = max(1, DISPATCH_BUDGET // len(self._batch))
except asyncio.QueueEmpty: for _ in range(min(count, len(self._targets))):
pass queue = self._targets.popleft()
if queue not in self._subscribers:
continue
if self._batch_dropped:
self._dropped.add(queue) self._dropped.add(queue)
queue.put_nowait(event) for event in self._batch:
if queue.offer(event):
self._dropped.add(queue)
else:
self._targets.clear()
if not self._targets:
self._batch = []
with self._pending_lock:
if not self._pending:
self._flush_scheduled = False
return
if self._loop is not None:
self._loop.call_soon(self._dispatch)
def dropped(self, queue: asyncio.Queue[dict[str, Any]]) -> bool: def dropped(self, queue: asyncio.Queue[dict[str, Any]]) -> bool:
"""Whether this subscriber lost events since it last asked. """Whether this subscriber lost events since it last asked.
A device's socket that did catches up with a snapshot: what it lost This includes staging overflow before fan-out. A socket catches up
may have been its demand, or the event that tells it it was unpaired. with a snapshot, since a lost value might never change again.
""" """
if queue in self._dropped: if queue in self._dropped:
self._dropped.discard(queue) self._dropped.discard(queue)
@@ -129,7 +215,7 @@ class EventBus:
@asynccontextmanager @asynccontextmanager
async def subscribe(self) -> AsyncIterator[asyncio.Queue[dict[str, Any]]]: async def subscribe(self) -> AsyncIterator[asyncio.Queue[dict[str, Any]]]:
"""Yield a queue receiving every event published while subscribed.""" """Yield a queue receiving every event published while subscribed."""
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=QUEUE_SIZE) queue = _EventQueue(maxsize=QUEUE_SIZE)
self._subscribers.add(queue) self._subscribers.add(queue)
try: try:
yield queue yield queue
+123 -24
View File
@@ -17,8 +17,10 @@ from __future__ import annotations
import logging import logging
import threading import threading
import time import time
from concurrent.futures import ThreadPoolExecutor from collections import deque
from typing import TYPE_CHECKING, Any from collections.abc import Callable
from concurrent.futures import Executor, Future, ThreadPoolExecutor
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
from fluksio.flow.queue import MAX_DELIVERIES, WorkItem, WorkQueue from fluksio.flow.queue import MAX_DELIVERIES, WorkItem, WorkQueue
@@ -41,9 +43,9 @@ TOUCH_INTERVAL_S = 20.0
# resolution of a delay: a delayed item is waited for exactly, so what a timer # resolution of a delay: a delayed item is waited for exactly, so what a timer
# fires late by is a wake-up and a promotion rather than up to a whole second. # fires late by is a wake-up and a promotion rather than up to a whole second.
DELAYED_INTERVAL_S = 1.0 DELAYED_INTERVAL_S = 1.0
# Cascade slots a promoted timer may use past `max_cascades`. A due item was # Extra cascade and node workers reserved for promoted timers. A due item was
# already waited for, so making it queue behind whatever long node happens to # already waited for, so making it queue behind whatever long node happens to
# hold the pool is the one lateness the sleeping timer thread cannot remove. # hold either pool is lateness the sleeping timer thread cannot remove.
DUE_RESERVE = 2 DUE_RESERVE = 2
# How long the saturated engine waits on the due lane before going back to # How long the saturated engine waits on the due lane before going back to
# check whether a cascade slot has freed. # check whether a cascade slot has freed.
@@ -67,6 +69,73 @@ BACKLOG_INTERVAL_S = 5.0
BACKLOG_DEGRADED = 50 BACKLOG_DEGRADED = 50
BACKLOG_STRIKES = 3 BACKLOG_STRIKES = 3
_P = ParamSpec("_P")
_T = TypeVar("_T")
class _PoolLane(Executor):
def __init__(self, pool: _ReservedPool, due: bool) -> None:
self._pool = pool
self._due = due
def submit(
self, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs
) -> Future[_T]:
return self._pool.submit(self._due, fn, *args, **kwargs)
class _ReservedPool:
"""Due work uses idle normal workers, or its own bounded reserve."""
def __init__(
self, workers: int, prefix: str, on_done: Callable[[], None] | None = None
) -> None:
self._limits = (workers, DUE_RESERVE)
self._pools = (
ThreadPoolExecutor(max_workers=workers, thread_name_prefix=prefix),
ThreadPoolExecutor(
max_workers=DUE_RESERVE, thread_name_prefix=f"{prefix}-due"
),
)
self._pending = [0, 0]
self._lock = threading.Lock()
self._on_done = on_done
self.normal = _PoolLane(self, False)
self.due = _PoolLane(self, True)
def available(self) -> tuple[int, int]:
with self._lock:
return (
max(0, self._limits[0] - self._pending[0]),
max(0, self._limits[1] - self._pending[1]),
)
def _done(self, lane: int) -> None:
with self._lock:
self._pending[lane] -= 1
if self._on_done is not None:
self._on_done()
def submit(
self, due: bool, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs
) -> Future[_T]:
with self._lock:
# Count queued work too: an idle thread with ordinary work already
# waiting is not capacity a timer can use without waiting behind it.
lane = int(due and self._pending[0] >= self._limits[0])
self._pending[lane] += 1
try:
future = self._pools[lane].submit(fn, *args, **kwargs)
except RuntimeError:
self._done(lane)
raise
future.add_done_callback(lambda _: self._done(lane))
return future
def shutdown(self) -> None:
for pool in self._pools:
pool.shutdown(wait=False)
class ExecutionService: class ExecutionService:
"""Owns the engine's worker threads and the queue they read from.""" """Owns the engine's worker threads and the queue they read from."""
@@ -93,12 +162,15 @@ class ExecutionService:
self._inflight_lock = threading.Condition() self._inflight_lock = threading.Condition()
# Entry ids claimed and still running, under _inflight_lock. # Entry ids claimed and still running, under _inflight_lock.
self._active: set[str] = set() self._active: set[str] = set()
self.node_pool = ThreadPoolExecutor( # At most one reclaim batch waits here. The timer thread must keep
max_workers=MAX_WORKERS if max_workers is None else max_workers, # promoting and touching work while the consumer waits for capacity.
thread_name_prefix="node", self._reclaimed: deque[WorkItem] = deque()
self._node_pools = _ReservedPool(
MAX_WORKERS if max_workers is None else max_workers, "node"
) )
self._cascade_pool = ThreadPoolExecutor( self.node_pool = self._node_pools.normal
max_workers=self.max_cascades, thread_name_prefix="cascade" self._cascade_pools = _ReservedPool(
self.max_cascades, "cascade", self._capacity_changed
) )
self._consumer: threading.Thread | None = None self._consumer: threading.Thread | None = None
self._timers: threading.Thread | None = None self._timers: threading.Thread | None = None
@@ -132,14 +204,18 @@ class ExecutionService:
thread.join(timeout=5) thread.join(timeout=5)
self._consumer = None self._consumer = None
self._timers = None self._timers = None
self._cascade_pool.shutdown(wait=False) self._cascade_pools.shutdown()
self.node_pool.shutdown(wait=False) self._node_pools.shutdown()
self.queue.close() self.queue.close()
def bind(self, pipeline: Pipeline) -> None: def bind(self, pipeline: Pipeline) -> None:
"""Point the service at the pipeline it should execute against.""" """Point the service at the pipeline it should execute against."""
self._pipeline = pipeline self._pipeline = pipeline
def _capacity_changed(self) -> None:
with self._inflight_lock:
self._inflight_lock.notify_all()
def pause_intake(self) -> None: def pause_intake(self) -> None:
"""Stop claiming, and wait for what is already claimed to finish. """Stop claiming, and wait for what is already claimed to finish.
@@ -182,6 +258,18 @@ class ExecutionService:
free, due_only = self._await_capacity() free, due_only = self._await_capacity()
if not free: if not free:
continue continue
with self._inflight_lock:
recovered = [
item
for item in self._reclaimed
if not due_only or item.not_before > 0
][:free]
for item in recovered:
self._reclaimed.remove(item)
if recovered:
for item in recovered:
self._dispatch(item)
continue
try: try:
items = self.queue.claim( items = self.queue.claim(
free, free,
@@ -262,21 +350,23 @@ class ExecutionService:
if now - last_reclaim < RECLAIM_INTERVAL_S: if now - last_reclaim < RECLAIM_INTERVAL_S:
continue continue
with self._inflight_lock:
if self._reclaimed or not self._intake.is_set():
continue
last_reclaim = now last_reclaim = now
try: try:
for item in self.queue.reclaim_stale(RECLAIM_IDLE_MS): recovered = self.queue.reclaim_stale(RECLAIM_IDLE_MS)
for item in recovered:
logger.info( logger.info(
"Reclaimed work for '%s' (delivery %d)", "Reclaimed work for '%s' (delivery %d)",
item.node, item.node,
item.deliveries, item.deliveries,
) )
# Through the same gate the main loop uses: a reclaim can with self._inflight_lock:
# return sixty-odd entries at once, and dispatching them self._reclaimed.extend(recovered)
# all would push `_inflight` far past `max_cascades` — self._active.update(
# exactly the overcommit the gate exists to prevent. item.entry_id for item in recovered if item.entry_id
if not self._await_capacity()[0]: )
break
self._dispatch(item)
except Exception as exc: except Exception as exc:
logger.error("Could not reclaim stale work: %s", exc) logger.error("Could not reclaim stale work: %s", exc)
@@ -331,7 +421,9 @@ class ExecutionService:
""" """
with self._inflight_lock: with self._inflight_lock:
while not self._stop.is_set(): while not self._stop.is_set():
free = self.max_cascades - self._inflight if not self._intake.is_set():
return 0, False
free, _ = self._cascade_pools.available()
if free > 0: if free > 0:
return free, False return free, False
# Only once a slot has genuinely failed to free: the due lane # Only once a slot has genuinely failed to free: the due lane
@@ -340,7 +432,7 @@ class ExecutionService:
# read every time the pool filled up. # read every time the pool filled up.
if self._inflight_lock.wait(0.5): if self._inflight_lock.wait(0.5):
continue continue
reserve = self.max_cascades + DUE_RESERVE - self._inflight _, reserve = self._cascade_pools.available()
if reserve > 0: if reserve > 0:
return reserve, True return reserve, True
return 0, False return 0, False
@@ -351,7 +443,12 @@ class ExecutionService:
if item.entry_id: if item.entry_id:
self._active.add(item.entry_id) self._active.add(item.entry_id)
try: try:
self._cascade_pool.submit(self._handle, item) pool = (
self._cascade_pools.due
if item.not_before > 0
else self._cascade_pools.normal
)
pool.submit(self._handle, item)
except RuntimeError: except RuntimeError:
# Pool already shutting down. # Pool already shutting down.
self._done(item) self._done(item)
@@ -386,7 +483,7 @@ class ExecutionService:
return item.node return item.node
def _handle(self, item: WorkItem) -> None: def _handle(self, item: WorkItem) -> None:
handled = True handled = False
try: try:
handled = self._run_item(item) handled = self._run_item(item)
except Exception: except Exception:
@@ -473,11 +570,12 @@ class ExecutionService:
replay = item.deliveries > 1 replay = item.deliveries > 1
emission = item.kind == "emission" emission = item.kind == "emission"
node_pool = self._node_pools.due if item.not_before > 0 else self.node_pool
try: try:
if item.kind == "flush": if item.kind == "flush":
# A rate-limit window ended; nothing to replay, only to let out. # A rate-limit window ended; nothing to replay, only to let out.
# A run of its own, so what it sets off is recorded somewhere. # A run of its own, so what it sets off is recorded somewhere.
pipeline.flush(node, item.entry_id) pipeline.flush(node, item.entry_id, node_pool=node_pool)
return True return True
# An emission's values went into state when the node produced them; # An emission's values went into state when the node produced them;
# this item carries them so its readers get the chunk that caused # this item carries them so its readers get the chunk that caused
@@ -498,6 +596,7 @@ class ExecutionService:
# Neither can say what changed, so neither filters on it. # Neither can say what changed, so neither filters on it.
changed=None if replay or not item.outputs else published, changed=None if replay or not item.outputs else published,
overrides=item.outputs if emission else None, overrides=item.outputs if emission else None,
node_pool=node_pool,
) )
finally: finally:
# Paired, or a cascade that raised — state backend gone, say — is a # Paired, or a cascade that raised — state backend gone, say — is a
+23 -13
View File
@@ -394,13 +394,8 @@ class MetricsCollector:
prune: bool, prune: bool,
) -> None: ) -> None:
with Session(engine) as session: with Session(engine) as session:
for (flow, node, minute), agg in buckets.items(): if buckets:
statement = insert(MetricBucket).values( statement = insert(MetricBucket)
flow=flow,
node=node,
bucket=datetime.fromtimestamp(minute, UTC),
**agg,
)
# The same minute is written several times, so the counters add # The same minute is written several times, so the counters add
# and the maxima take whichever is larger. Columns are read by # and the maxima take whichever is larger. Columns are read by
# subscript: `excluded.items` is the collection's own method. # subscript: `excluded.items` is the collection's own method.
@@ -418,21 +413,36 @@ class MetricsCollector:
name: func.max(col(getattr(MetricBucket, name)), new[name]) name: func.max(col(getattr(MetricBucket, name)), new[name])
for name in MAXIMA for name in MAXIMA
}, },
) ),
[
{
"flow": flow,
"node": node,
"bucket": datetime.fromtimestamp(minute, UTC),
**agg,
}
for (flow, node, minute), agg in buckets.items()
],
) )
for run in runs: if runs:
values = {k: v for k, v in run.items() if k != "started_ts"} values = [
statement = insert(FlowRun).values(**values) {k: v for k, v in run.items() if k != "started_ts"} for run in runs
]
statement = insert(FlowRun)
session.execute( session.execute(
statement.on_conflict_do_update( statement.on_conflict_do_update(
index_elements=["id"], index_elements=["id"],
set_={ set_={
key: statement.excluded[key] key: statement.excluded[key]
for key in values for key in values[0]
if key != "id" if key != "id"
}, },
) ),
values,
# A redelivery clears finished_at. Include nulls in the
# batch so open and finished runs share the same statement.
execution_options={"render_nulls": True},
) )
session.add_all(pending) session.add_all(pending)
+88 -98
View File
@@ -21,10 +21,11 @@ import time
import uuid import uuid
from collections import deque from collections import deque
from collections.abc import Callable, Iterator from collections.abc import Callable, Iterator
from concurrent.futures import Future, ThreadPoolExecutor, wait from concurrent.futures import Executor, Future, ThreadPoolExecutor
from contextlib import contextmanager from contextlib import contextmanager
from datetime import UTC, datetime from datetime import UTC, datetime
from functools import lru_cache from functools import lru_cache
from queue import SimpleQueue
from typing import Any, Literal, Protocol from typing import Any, Literal, Protocol
from pydantic import BaseModel, model_validator from pydantic import BaseModel, model_validator
@@ -275,6 +276,7 @@ class Pipeline:
"_paused", "_paused",
"_stepping", "_stepping",
"_gate_lock", "_gate_lock",
"_gate_revision",
"_graph_lock", "_graph_lock",
"_queue", "_queue",
"_node_pool", "_node_pool",
@@ -294,7 +296,7 @@ class Pipeline:
initial_values: dict[str, Any] | None = None, initial_values: dict[str, Any] | None = None,
disabled_flows: set[str] | None = None, disabled_flows: set[str] | None = None,
work_queue: WorkQueue | None = None, work_queue: WorkQueue | None = None,
node_pool: ThreadPoolExecutor | None = None, node_pool: Executor | None = None,
observer: Callable[[NodeOutcome], None] | None = None, observer: Callable[[NodeOutcome], None] | None = None,
emission_observer: Callable[[str, dict[str, Any]], None] | None = None, emission_observer: Callable[[str, dict[str, Any]], None] | None = None,
run_cache: RunCacheLookup | None = None, run_cache: RunCacheLookup | None = None,
@@ -307,6 +309,7 @@ class Pipeline:
self._paused: set[str] = set() self._paused: set[str] = set()
self._stepping: set[str] = set() self._stepping: set[str] = set()
self._gate_lock = threading.Lock() self._gate_lock = threading.Lock()
self._gate_revision = 0
# Held only while the graph maps are read or swapped together, never # Held only while the graph maps are read or swapped together, never
# across an await, a node call or any I/O. Reentrant because the lazy # across an await, a node call or any I/O. Reentrant because the lazy
# `edges` build re-enters through `_graph`. It is never nested with # `edges` build re-enters through `_graph`. It is never nested with
@@ -448,6 +451,7 @@ class Pipeline:
with self._gate_lock: with self._gate_lock:
self._paused.discard(flow) self._paused.discard(flow)
self._stepping.discard(flow) self._stepping.discard(flow)
self._gate_revision += 1
def remove_flow(self, flow: str) -> None: def remove_flow(self, flow: str) -> None:
"""Take a deleted flow out of the graph.""" """Take a deleted flow out of the graph."""
@@ -742,7 +746,9 @@ class Pipeline:
self._state[self._flush_key(node.id)] = at self._state[self._flush_key(node.id)] = at
self.defer(node, {}, at - now, kind="flush", parent=entry_id) self.defer(node, {}, at - now, kind="flush", parent=entry_id)
def flush(self, node: Node, entry_id: str = "") -> None: def flush(
self, node: Node, entry_id: str = "", *, node_pool: Executor | None = None
) -> None:
"""Let through what this node's rate limits held back. """Let through what this node's rate limits held back.
Runs when a window ends: the last value a limited output kept back is Runs when a window ends: the last value a limited output kept back is
@@ -766,7 +772,9 @@ class Pipeline:
# nothing came out and there is nothing downstream to wake. # nothing came out and there is nothing downstream to wake.
published = self.apply_outputs(node, held, entry_id) published = self.apply_outputs(node, held, entry_id)
if published: if published:
self.run_downstream(node, entry_id=entry_id, changed=published) self.run_downstream(
node, entry_id=entry_id, changed=published, node_pool=node_pool
)
if any(spec.interval > 0 for spec in node.requires.values()): if any(spec.interval > 0 for spec in node.requires.values()):
# A wake-up that was held back, not a value that just changed — # A wake-up that was held back, not a value that just changed —
# so this one deliberately walks everything reachable. # so this one deliberately walks everything reachable.
@@ -775,6 +783,7 @@ class Pipeline:
self._state, self._state,
check_ready=True, check_ready=True,
entry_id=entry_id, entry_id=entry_id,
node_pool=node_pool,
) )
def _increment_message_versions(self, outputs: dict[str, Any]) -> None: def _increment_message_versions(self, outputs: dict[str, Any]) -> None:
@@ -1252,7 +1261,9 @@ class Pipeline:
def set_disabled(self, flows: set[str]) -> None: def set_disabled(self, flows: set[str]) -> None:
"""Which flows are stopped. Swapped whole, so a reader never sees half.""" """Which flows are stopped. Swapped whole, so a reader never sees half."""
self._disabled = frozenset(flows) with self._gate_lock:
self._disabled = frozenset(flows)
self._gate_revision += 1
def paused_flows(self) -> list[str]: def paused_flows(self) -> list[str]:
with self._gate_lock: with self._gate_lock:
@@ -1262,11 +1273,13 @@ class Pipeline:
"""Hold this flow's nodes. Values still arrive; nothing acts on them.""" """Hold this flow's nodes. Values still arrive; nothing acts on them."""
with self._gate_lock: with self._gate_lock:
self._paused.add(flow) self._paused.add(flow)
self._gate_revision += 1
self._publish({"type": "flow_paused", "flow": flow, "paused": True}) self._publish({"type": "flow_paused", "flow": flow, "paused": True})
def resume(self, flow: str) -> None: def resume(self, flow: str) -> None:
with self._gate_lock: with self._gate_lock:
self._paused.discard(flow) self._paused.discard(flow)
self._gate_revision += 1
self._publish({"type": "flow_paused", "flow": flow, "paused": False}) self._publish({"type": "flow_paused", "flow": flow, "paused": False})
# Whatever was held back is free to run now. # Whatever was held back is free to run now.
self._execute_parallel(self.flow_nodes(flow), self._state, check_ready=True) self._execute_parallel(self.flow_nodes(flow), self._state, check_ready=True)
@@ -1280,11 +1293,13 @@ class Pipeline:
""" """
with self._gate_lock: with self._gate_lock:
self._stepping.add(flow) self._stepping.add(flow)
self._gate_revision += 1
try: try:
yield yield
finally: finally:
with self._gate_lock: with self._gate_lock:
self._stepping.discard(flow) self._stepping.discard(flow)
self._gate_revision += 1
def _gate_blocks(self, node: Node) -> bool: def _gate_blocks(self, node: Node) -> bool:
"""Is this node's flow held back from executing?""" """Is this node's flow held back from executing?"""
@@ -1311,6 +1326,7 @@ class Pipeline:
replay: bool = False, replay: bool = False,
changed: set[str] | None = None, changed: set[str] | None = None,
overrides: dict[str, Any] | None = None, overrides: dict[str, Any] | None = None,
node_pool: Executor | None = None,
) -> StateBackend: ) -> StateBackend:
"""Execute nodes concurrently, scheduling each as its inputs arrive. """Execute nodes concurrently, scheduling each as its inputs arrive.
@@ -1344,9 +1360,11 @@ class Pipeline:
# Copied: the caller's set is not this wave's to grow. # Copied: the caller's set is not this wave's to grow.
fresh = set(changed) if changed is not None else None fresh = set(changed) if changed is not None else None
submitted: set[Node] = set() ready_nodes = deque(n for n, count in in_degree.items() if count == 0)
skipped: set[Node] = set() blocked: set[Node] = set()
node_futures: dict[Node, Future[dict[str, Any] | None]] = {} gate_revision = self._gate_revision
node_futures: dict[Future[dict[str, Any] | None], Node] = {}
finished: SimpleQueue[Future[dict[str, Any] | None]] = SimpleQueue()
def untouched(n: Node) -> bool: def untouched(n: Node) -> bool:
"""Did this wave publish nothing this node waits on?""" """Did this wave publish nothing this node waits on?"""
@@ -1356,108 +1374,72 @@ class Pipeline:
) )
def complete(n: Node) -> None: def complete(n: Node) -> None:
"""Count a node as done without running it, freeing its consumers.""" """Release consumers when their last dependency finishes or skips."""
skipped.add(n)
for consumer in edges[n]: for consumer in edges[n]:
if consumer in target_nodes: if consumer in target_nodes:
in_degree[consumer] -= 1 in_degree[consumer] -= 1
if in_degree[consumer] == 0:
ready_nodes.append(consumer)
def submit_ready(executor: ThreadPoolExecutor) -> None: def submit_ready(executor: Executor) -> None:
# To a fixpoint: completing a node without running it frees its nonlocal gate_revision
# consumers, and one this pass has already walked past would with self._gate_lock:
# otherwise be stranded — there is no future outstanding to bring revision = self._gate_revision
# the drain loop back round for it. if blocked and revision != gate_revision:
progressed = True # A flow can resume while another branch of this wave runs.
while progressed: # Revisit only its held candidates, and only if a gate moved.
progressed = False ready_nodes.extend(blocked)
for n in target_nodes: blocked.clear()
if n in submitted or n in skipped: gate_revision = revision
continue while ready_nodes:
# Gate before the readiness check, so a held-back node does n = ready_nodes.popleft()
# not spend the synchronous claim it needs once it may run. # Gate before readiness, so a pause spends no synchronous claim.
if self._gate_blocks(n) or in_degree[n] != 0: if self._gate_blocks(n):
continue blocked.add(n)
if n.passive: continue
# Something outside publishes for it; a run is not it. if n.passive:
complete(n)
continue
if fresh is not None and untouched(n):
complete(n)
continue
ready_inputs: dict[str, Any] | None = None
if check_ready:
ready, ready_inputs = self._is_node_ready(n, state, entry_id)
if not ready:
# Its old output remains usable by a consumer whose
# other producer did publish during this wave.
complete(n) complete(n)
progressed = True
continue continue
if fresh is not None and untouched(n): if replay and entry_id and self._already_done(entry_id, n):
# Ahead of the readiness check, so an unchanged node complete(n)
# spends neither a delivery stamp nor a synchronous continue
# claim on a run it is not going to make. future = executor.submit(
complete(n) self._execute_node, n, state, entry_id, overrides, ready_inputs
progressed = True )
continue node_futures[future] = n
ready_inputs: dict[str, Any] | None = None future.add_done_callback(finished.put)
if check_ready:
# The values are kept whether or not the answer is yes:
# a node that may run reads them below instead of
# asking state for the same keys again.
ready, ready_inputs = self._is_node_ready(n, state, entry_id)
if not ready:
# Completed, not merely passed over: a later trigger
# may still make it ready, but not running is not
# the same as blocking. Whatever it last published
# is in state, and a consumer reading that alongside
# something this wave *did* publish has every right
# to run on the pair.
#
# Passing over it held that consumer for the whole
# wave, and a node that can never be ready — an
# input nothing has ever published — took its
# consumers out for good. On this house
# `plugs.pump_run` waits on a watering pulse that
# only exists at 02:00, and it sits upstream of
# `dmx.switches`: every boiler, plug and appliance
# command reached its own message and stopped there,
# reaching the DMX universe only in the occasional
# wave the pump node happened not to be part of.
complete(n)
progressed = True
continue
if replay and entry_id and self._already_done(entry_id, n):
# Its side effect happened on an earlier delivery; its
# outputs are still in state, so downstream carries on.
complete(n)
progressed = True
continue
submitted.add(n)
node_futures[n] = executor.submit(
self._execute_node, n, state, entry_id, overrides, ready_inputs
)
def drain(executor: ThreadPoolExecutor) -> None: def drain(executor: Executor) -> None:
submit_ready(executor) submit_ready(executor)
while node_futures: while node_futures:
done, _ = wait(node_futures.values(), return_when="FIRST_COMPLETED") # A completion queue also avoids scanning every outstanding
completed = [n for n, f in node_futures.items() if f in done] # future whenever one node finishes.
future = finished.get()
n = node_futures.pop(future)
result = future.result()
if result is not None and fresh is not None:
fresh.update(result)
# Silent, failed and throttled producers still release their
# consumers. Fresh inputs decide whether those consumers run.
complete(n)
submit_ready(executor)
for n in completed: pool = node_pool if node_pool is not None else self._node_pool
result = node_futures.pop(n).result() if pool is not None:
# Before the decrement: a consumer freed by this node is
# judged on what it just published.
if result is not None and fresh is not None:
fresh.update(result)
# Decremented whatever it returned. A node that published
# nothing — rate limited, unchanged, or failed — has not
# *blocked* its consumers; it has merely given them nothing
# to read, and `untouched` above is the accurate test of
# that. Holding the count instead made a quiet producer
# veto a noisy sibling: `dmx.switches` reads both boilers
# through report-by-exception nodes, so the kitchen one
# staying silent — which is almost always — kept the main
# boiler's command from ever reaching the DMX universe.
for consumer in edges[n]:
if consumer in target_nodes:
in_degree[consumer] -= 1
submit_ready(executor)
if self._node_pool is not None:
# The execution service owns a long-lived pool; building one per # The execution service owns a long-lived pool; building one per
# wave is what used to spawn threads without bound under load. # wave is what used to spawn threads without bound under load.
drain(self._node_pool) drain(pool)
else: else:
with ThreadPoolExecutor(max_workers=self._max_workers) as executor: with ThreadPoolExecutor(max_workers=self._max_workers) as executor:
drain(executor) drain(executor)
@@ -1519,6 +1501,7 @@ class Pipeline:
replay: bool = False, replay: bool = False,
changed: set[str] | None = None, changed: set[str] | None = None,
overrides: dict[str, Any] | None = None, overrides: dict[str, Any] | None = None,
node_pool: Executor | None = None,
) -> StateBackend: ) -> StateBackend:
"""Run everything downstream of a node that has just published. """Run everything downstream of a node that has just published.
@@ -1552,6 +1535,7 @@ class Pipeline:
replay=replay, replay=replay,
changed=changed, changed=changed,
overrides=overrides, overrides=overrides,
node_pool=node_pool,
) )
def trigger( def trigger(
@@ -1796,6 +1780,12 @@ class Pipeline:
# Losing the value outright would be worse than running it here. # Losing the value outright would be worse than running it here.
self._run_here(node, outputs, cause="external", delivered=kind == "emission") self._run_here(node, outputs, cause="external", delivered=kind == "emission")
def value(self, name: str) -> dict[str, Any]:
"""One message and its timestamp, read together without listing state."""
stamp = self._timestamp_key(name)
found = self._state.get_multi([name, stamp])
return {"value": found.get(name), "ts": found.get(stamp)}
def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]: def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]:
"""Last value and timestamp of every message, optionally one flow's. """Last value and timestamp of every message, optionally one flow's.
+25 -29
View File
@@ -53,6 +53,23 @@ STALE_CONSUMER_IDLE_MS = 3_600_000
# count, since redelivery happens seconds after the claim and never later. # count, since redelivery happens seconds after the claim and never later.
DONE_MARKERS = 10_000 DONE_MARKERS = 10_000
# A promotion is one Redis operation, including when the caller loses its
# connection before hearing the result. Write first: Lua does not roll back a
# command error, so a refused XADD must leave its timer in the delayed set.
_MOVE_DUE = """
local due = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[1], 'LIMIT', 0, 100)
for _, raw in ipairs(due) do
local fields = {}
for key, value in pairs(cjson.decode(raw)) do
fields[#fields + 1] = key
fields[#fields + 1] = value
end
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[2], '*', unpack(fields))
redis.call('ZREM', KEYS[1], raw)
end
return #due
"""
@dataclass @dataclass
class WorkItem: class WorkItem:
@@ -446,6 +463,7 @@ class RedisWorkQueue(WorkQueue):
# finding them with a keyspace scan walked every state and idempotency # finding them with a keyspace scan walked every state and idempotency
# key in the database, twice per health poll. # key in the database, twice per health poll.
self._parked_flows_key = f"{namespace}:__parked_flows__" self._parked_flows_key = f"{namespace}:__parked_flows__"
self._move_due = self._redis.register_script(_MOVE_DUE)
self._ensure_group() self._ensure_group()
self._drop_stale_consumers() self._drop_stale_consumers()
@@ -619,36 +637,14 @@ class RedisWorkQueue(WorkQueue):
] ]
def move_due(self, now: float) -> int: def move_due(self, now: float) -> int:
due = cast( """Move up to 100 timers into the due lane atomically."""
list[str], return cast(
self._redis.zrangebyscore(self._delayed_key, "-inf", now, start=0, num=100), int,
self._move_due(
keys=[self._delayed_key, self._due_stream],
args=[now, STREAM_MAXLEN],
),
) )
if not due:
return 0
# Whoever removes it owns it: a second engine gets 0 for that member and
# leaves it alone. One round trip for the batch rather than one each,
# which matters because this runs every second.
pipe = self._redis.pipeline()
for raw in due:
pipe.zrem(self._delayed_key, raw)
claimed = cast(list[int], pipe.execute())
pipe = self._redis.pipeline()
moved = 0
for raw, owned in zip(due, claimed, strict=True):
if owned:
# Into the due lane, which `claim` reads first: what has waited
# out a deadline should not then queue behind a backlog.
pipe.xadd(
self._due_stream,
cast(Any, orjson.loads(raw)),
maxlen=STREAM_MAXLEN,
approximate=True,
)
moved += 1
if moved:
pipe.execute()
return moved
def dead_letter(self, item: WorkItem, reason: str) -> None: def dead_letter(self, item: WorkItem, reason: str) -> None:
fields = item.to_fields() fields = item.to_fields()
+28 -3
View File
@@ -15,7 +15,7 @@ import mimetypes
import re import re
from pathlib import Path from pathlib import Path
from starlette.middleware.gzip import GZipMiddleware from starlette.datastructures import Headers
from starlette.responses import FileResponse from starlette.responses import FileResponse
from starlette.types import ASGIApp, Receive, Scope, Send from starlette.types import ASGIApp, Receive, Scope, Send
@@ -26,6 +26,24 @@ API_PREFIXES = ("/api/", "/hooks/", "/mcp", "/.well-known/", "/docs", "/redoc")
mimetypes.add_type("application/manifest+json", ".webmanifest") mimetypes.add_type("application/manifest+json", ".webmanifest")
def _accepts_gzip(header: str) -> bool:
"""An explicit refusal overrides a wildcard, including on a cached asset."""
qualities: dict[str, float] = {}
for entry in header.lower().split(","):
encoding, *parameters = entry.strip().split(";")
quality = 1.0
for parameter in parameters:
key, _, value = parameter.strip().partition("=")
if key == "q":
try:
quality = float(value)
except ValueError:
quality = 0.0
qualities[encoding.strip()] = quality if 0.0 <= quality <= 1.0 else 0.0
gzip = qualities.get("gzip", qualities.get("*", 0.0))
return gzip > 0 and gzip >= qualities.get("identity", 0.0)
def content_security_policy(index_html: str) -> str: def content_security_policy(index_html: str) -> str:
"""The policy for the dashboard, admitting its inline scripts by hash. """The policy for the dashboard, admitting its inline scripts by hash.
@@ -84,5 +102,12 @@ class Dashboard:
# assets, so a stale copy pins the browser to the previous build. # assets, so a stale copy pins the browser to the previous build.
file = self.index file = self.index
headers = {**self.headers, "Cache-Control": "no-cache"} headers = {**self.headers, "Cache-Control": "no-cache"}
response = FileResponse(file, headers=headers) media_type = mimetypes.guess_type(file.name)[0] or "text/plain"
await GZipMiddleware(response, minimum_size=1024)(scope, receive, send) compressed = file.with_name(file.name + ".gz").resolve()
if compressed.is_relative_to(self.dist) and compressed.is_file():
headers["Vary"] = "Accept-Encoding"
if _accepts_gzip(Headers(scope=scope).get("accept-encoding", "")):
file = compressed
headers["Content-Encoding"] = "gzip"
response = FileResponse(file, headers=headers, media_type=media_type)
await response(scope, receive, send)
+92 -1
View File
@@ -2,8 +2,11 @@
import asyncio import asyncio
import threading import threading
from contextlib import AsyncExitStack
from fluksio.flow.events import QUEUE_SIZE, EventBus import pytest
from fluksio.flow.events import DISPATCH_BUDGET, QUEUE_SIZE, EventBus
def _event(i: int) -> dict[str, object]: def _event(i: int) -> dict[str, object]:
@@ -100,3 +103,91 @@ def test_nothing_is_buffered_with_nobody_listening() -> None:
return len(bus._pending) return len(bus._pending)
assert asyncio.run(run()) == 0 assert asyncio.run(run()) == 0
@pytest.mark.parametrize("flush", [False, True])
def test_overflow_keeps_errors_and_control_events_in_order(flush: bool) -> None:
"""The staging bound and a slow subscriber use the same eviction policy."""
async def run() -> None:
bus = EventBus()
bus.bind(asyncio.get_running_loop())
critical = [
{"type": "node_log", "level": "error"},
{"type": "node_error"},
{"type": "run_started"},
{"type": "audit"},
{"type": "dashboard_changed"},
{"type": "future_control_event"},
]
async with bus.subscribe() as first, bus.subscribe() as second:
sent = [*critical, *(_event(i) for i in range(QUEUE_SIZE + 50))]
sent.append({"type": "run_finished"})
for sequence, event in enumerate(sent):
event["sequence"] = sequence
bus.publish(event)
assert len(bus._pending) <= QUEUE_SIZE
if flush:
# Queue overflow, with no staging overflow.
await asyncio.sleep(0)
await asyncio.sleep(0)
for queue in (first, second):
received = [queue.get_nowait() for _ in range(queue.qsize())]
assert len(received) == QUEUE_SIZE
assert received[: len(critical)] == critical
assert received[-1]["type"] == "run_finished"
assert received == sorted(received, key=lambda e: e["sequence"])
assert bus.dropped(queue)
assert not bus.dropped(queue)
asyncio.run(run())
def test_a_full_critical_backlog_rejects_telemetry_and_reports_loss() -> None:
async def run() -> None:
bus = EventBus()
bus.bind(asyncio.get_running_loop())
async with bus.subscribe() as queue:
for i in range(QUEUE_SIZE + 1):
bus.publish({"type": "node_error", "sequence": i})
bus.publish(_event(0))
assert len(bus._pending) == QUEUE_SIZE
await asyncio.sleep(0)
assert bus.dropped(queue)
assert (await queue.get())["sequence"] == 1
bus.publish({"type": "audit"})
await asyncio.sleep(0)
bus.publish(_event(1))
await asyncio.sleep(0)
assert bus.dropped(queue)
assert queue.qsize() == QUEUE_SIZE
assert all(
queue.get_nowait()["type"] != "node_executed" for _ in range(QUEUE_SIZE)
)
asyncio.run(run())
def test_fanout_yields_before_all_subscribers_are_served() -> None:
async def run() -> None:
bus = EventBus()
loop = asyncio.get_running_loop()
bus.bind(loop)
async with AsyncExitStack() as stack:
queues = [
await stack.enter_async_context(bus.subscribe())
for _ in range(DISPATCH_BUDGET // QUEUE_SIZE + 1)
]
for i in range(QUEUE_SIZE):
bus.publish(_event(i))
heartbeat: asyncio.Future[int] = loop.create_future()
loop.call_soon(
lambda: heartbeat.set_result(sum(queue.qsize() for queue in queues))
)
assert await heartbeat == DISPATCH_BUDGET
await asyncio.sleep(0)
assert all(queue.qsize() == QUEUE_SIZE for queue in queues)
assert not any(bus.dropped(queue) for queue in queues)
assert not bus._flush_scheduled
asyncio.run(run())
+42
View File
@@ -242,6 +242,48 @@ def test_a_trigger_is_journaled_rather_than_run_on_the_spot():
assert state["f.doubled"] == 6.0 assert state["f.doubled"] == 6.0
def test_a_state_failure_leaves_the_delivery_unacknowledged(monkeypatch):
pipeline, source, state, seen = _pipeline_with_a_consumer()
service = ExecutionService(pipeline._queue)
service.bind(pipeline)
def unavailable(*args, **kwargs):
raise ConnectionError("State unavailable")
monkeypatch.setattr(state, "record", unavailable)
source.inject({"reading": 3.0})
item = pipeline._queue.claim(1, 10)[0]
try:
service._dispatch(item)
service.pause_intake()
assert seen == []
assert pipeline._queue.stats()["pending"] == 1
assert service.inflight == 0
finally:
service.stop()
def test_a_node_failure_still_acknowledges_its_delivery(monkeypatch):
pipeline, source, _state, seen = _pipeline_with_a_consumer()
service = ExecutionService(pipeline._queue)
service.bind(pipeline)
def broken(*args, **kwargs):
raise ValueError("Node failed")
monkeypatch.setattr(pipeline.get_node_by_id("f.consumer"), "f", broken)
source.inject({"reading": 3.0})
item = pipeline._queue.claim(1, 10)[0]
try:
service._dispatch(item)
service.pause_intake()
assert seen == []
assert pipeline._queue.stats()["pending"] == 0
assert service.inflight == 0
finally:
service.stop()
def test_work_for_a_paused_flow_is_held_and_released_on_resume(): def test_work_for_a_paused_flow_is_held_and_released_on_resume():
pipeline, source, state, seen = _pipeline_with_a_consumer() pipeline, source, state, seen = _pipeline_with_a_consumer()
service = ExecutionService(pipeline._queue) service = ExecutionService(pipeline._queue)
+68
View File
@@ -0,0 +1,68 @@
"""Redis delivery checks, enabled by FLUKSIO_TEST_REDIS_HOST (and optional PORT)."""
import os
import uuid
from concurrent.futures import ThreadPoolExecutor
import pytest
import redis
from fluksio.flow.queue import RedisWorkQueue, WorkItem
@pytest.fixture
def queue():
host = os.environ.get("FLUKSIO_TEST_REDIS_HOST")
if not host:
pytest.skip("Set FLUKSIO_TEST_REDIS_HOST to run Redis integration checks")
queue = RedisWorkQueue(
host=host,
port=int(os.environ.get("FLUKSIO_TEST_REDIS_PORT", "6379")),
namespace=f"test-queue-{uuid.uuid4().hex}",
)
try:
yield queue
finally:
keys = list(queue._redis.scan_iter(match=f"{queue._ns}:*"))
if keys:
queue._redis.delete(*keys)
queue.close()
def test_concurrent_promotions_deliver_each_due_item_once(queue):
for i in range(150):
queue.add_delayed(
WorkItem(kind="cascade", node=f"f.n{i}", flow="f", outputs={"f.x": i}),
100.0,
)
queue.add_delayed(WorkItem(kind="cascade", node="f.later", flow="f"), 300.0)
with ThreadPoolExecutor(max_workers=2) as pool:
moved = list(pool.map(queue.move_due, [200.0, 200.0]))
assert sorted(moved) == [50, 100]
assert queue.move_due(200.0) == 0
assert queue.next_due() == 300.0
claimed = queue.claim(200, 10, due_only=True)
assert len(claimed) == 150
assert {item.outputs["f.x"] for item in claimed} == set(range(150))
assert all(item.not_before == 100.0 for item in claimed)
assert all(item.entry_id.startswith("due:") for item in claimed)
def test_a_refused_stream_write_keeps_the_timer_for_retry(queue):
queue.add_delayed(WorkItem(kind="cascade", node="f.stop", flow="f"), 100.0)
# A Redis command error does not roll a Lua script back. The stream write
# must succeed before the delayed entry is removed.
queue._redis.delete(queue._due_stream)
queue._redis.set(queue._due_stream, "not a stream")
with pytest.raises(redis.ResponseError, match="WRONGTYPE"):
queue.move_due(200.0)
assert queue.next_due() == 100.0
queue._redis.delete(queue._due_stream)
queue._ensure_group()
assert queue.move_due(200.0) == 1
assert queue.move_due(200.0) == 0
assert [item.node for item in queue.claim(10, 10, due_only=True)] == ["f.stop"]
+21
View File
@@ -122,3 +122,24 @@ def test_a_hop_costs_a_bounded_number_of_state_operations() -> None:
total = sum(state.calls[name] for name in COUNTED) total = sum(state.calls[name] for name in COUNTED)
assert total <= 8, f"state operations per hop grew: {dict(state.calls)}" assert total <= 8, f"state operations per hop grew: {dict(state.calls)}"
def test_reading_one_value_does_not_list_or_read_other_messages(monkeypatch) -> None:
state = MemoryState()
state.update({f"f.m{i}": i for i in range(1000)})
state.update({"f.m0": None, "__ts__:f.m0": 123.0})
pipeline = Pipeline([], state=state)
reads = []
get_multi = state.get_multi
def read(keys):
reads.append(keys)
return get_multi(keys)
def no_listing():
raise AssertionError("A single-message read must not enumerate state")
monkeypatch.setattr(MemoryState, "get_multi", lambda self, keys: read(keys))
monkeypatch.setattr(MemoryState, "message_names", lambda self: no_listing())
assert pipeline.value("f.m0") == {"value": None, "ts": 123.0}
assert reads == [["f.m0", "__ts__:f.m0"]]
+84
View File
@@ -0,0 +1,84 @@
"""Scheduling work scales with the graph and observes runtime gates."""
import threading
from concurrent.futures import ThreadPoolExecutor
from contextlib import nullcontext
import pytest
from fluksio.flow.messages import DType, MessageSpec
from fluksio.flow.nodes import Node
from fluksio.flow.pipeline import Pipeline
def _node(name, function, requires=(), provides=(), flow="f"):
node = Node(
f=function,
requires=[MessageSpec(name=key, dtype=DType.FLOAT) for key in requires],
provides=[MessageSpec(name=key, dtype=DType.FLOAT) for key in provides],
name=name,
)
node.assign_flow(flow, name)
return node
def test_a_long_chain_checks_each_candidate_once():
class CountingPipeline(Pipeline):
checks = 0
def _gate_blocks(self, node):
self.checks += 1
return super()._gate_blocks(node)
nodes = []
for i in range(100):
def relay(params, _index=i, **inputs):
return {f"v{_index + 1}": inputs[f"v{_index}"] + 1.0}
nodes.append(_node(f"n{i}", relay, requires=[f"v{i}"], provides=[f"v{i + 1}"]))
pipeline = CountingPipeline(nodes)
pipeline.run({"f.v0": 0.0})
assert pipeline.values()["f.v100"]["value"] == 100.0
assert pipeline.checks == len(nodes)
@pytest.mark.parametrize("disabled", [False, True])
def test_a_gate_opened_mid_wave_releases_its_waiting_candidate(disabled):
running, release, held, acted = (threading.Event() for _ in range(4))
class WatchingPipeline(Pipeline):
def _gate_blocks(self, node):
blocked = super()._gate_blocks(node)
if blocked:
held.set()
return blocked
def slow(params):
running.set()
assert release.wait(3)
pipeline = WatchingPipeline(
[
_node("slow", slow, flow="running"),
_node("action", lambda params: acted.set(), flow="paused"),
]
)
if disabled:
pipeline.set_disabled({"paused"})
else:
pipeline.pause("paused")
with ThreadPoolExecutor(max_workers=1) as driver:
future = driver.submit(pipeline.run)
try:
assert running.wait(1) and held.wait(1)
if disabled:
pipeline.set_disabled(set())
with nullcontext() if disabled else pipeline.stepping("paused"):
release.set()
future.result(2)
assert acted.is_set()
finally:
release.set()
+129
View File
@@ -0,0 +1,129 @@
"""Direct and portal browsers recover state without widening camera access."""
import asyncio
import json
from contextlib import suppress
from types import SimpleNamespace
import pytest
from fluksio.api import deps
from fluksio.api.routes import flows
from fluksio.cloud.connector import CloudConnector
from fluksio.flow import events
from fluksio.flow.events import QUEUE_SIZE, EventBus
@pytest.mark.parametrize("tunnel", [False, True])
@pytest.mark.parametrize("burst", [0, QUEUE_SIZE + 1])
def test_a_browser_catches_changes_during_its_snapshot(monkeypatch, tunnel, burst):
async def run():
bus = EventBus()
bus.bind(asyncio.get_running_loop())
monkeypatch.setattr(events, "event_bus", bus)
monkeypatch.setattr(flows, "event_bus", bus)
user = SimpleNamespace(id="viewer", is_superuser=False)
for module in (flows, deps):
monkeypatch.setattr(module, "user_from_token", lambda *a, **kw: user)
monkeypatch.setattr(
module, "hidden_from", lambda *a, **kw: frozenset({"f.camera"})
)
monkeypatch.setattr(flows, "panel_scope", lambda *a: None)
monkeypatch.setattr(flows, "socket_token", lambda *a: "test-token")
async def immediate(function, *args):
return function(*args)
monkeypatch.setattr(flows, "run_in_threadpool", immediate)
current = {"f.flag": {"value": 0}, "f.camera": {"value": "private"}}
controller = SimpleNamespace(
values=lambda: dict(current),
node_statuses=lambda: [],
issues=[],
paused_flows=lambda: [],
placer=None,
)
app = SimpleNamespace(state=SimpleNamespace(flow_controller=controller))
received = []
client_values = {}
caught_up = asyncio.Event()
drained = asyncio.Event()
class Socket:
headers = {}
async def accept(self):
pass
async def receive(self):
await asyncio.Future()
async def send(self, text):
frame = json.loads(text)
if frame["op"] == "ws_msg":
await self.send_text(frame["text"])
async def send_text(self, text):
payload = json.loads(text)
received.append(payload)
if payload["type"] == "snapshot":
assert "f.camera" not in payload["values"]
client_values.update(payload["values"])
if payload["values"]["f.flag"]["value"] == 1:
caught_up.set()
elif len(received) == 1:
# The state was read, but its frame is still sending.
# This value changes only once, then a burst loses its
# event and leaves only a snapshot able to recover it.
current["f.flag"] = {"value": 1}
bus.publish(
{"type": "message_value", "name": "f.flag", "value": 1}
)
for i in range(burst):
current["f.noise"] = {"value": i}
bus.publish(
{"type": "message_value", "name": "f.noise", "value": i}
)
# Recovery must forward retained critical events too.
bus.publish({"type": "node_error", "node": "f.example"})
await asyncio.sleep(0)
for event in payload.get("events", [payload]):
if event["type"] == "message_value":
client_values[event["name"]] = {"value": event["value"]}
if event.get("name") == "f.flag" and event.get("value") == 1:
caught_up.set()
if event["type"] == "node_error":
drained.set()
socket = Socket()
socket.app = app
if tunnel:
action = CloudConnector(app)._stream(
socket,
{"id": "review", "path": "/api/v1/flows/ws", "query": "token=test"},
)
else:
action = flows.flow_events(socket)
task = asyncio.create_task(action)
try:
await asyncio.wait_for(caught_up.wait(), timeout=2)
await asyncio.wait_for(drained.wait(), timeout=2)
finally:
task.cancel()
with suppress(asyncio.CancelledError):
await task
assert sum(item["type"] == "snapshot" for item in received) == (
2 if burst else 1
)
flattened = [
event for payload in received for event in payload.get("events", [payload])
]
assert sum(event["type"] == "node_error" for event in flattened) == 1
if burst:
# The client replaces values in arrival order. A snapshot ahead
# of the retained backlog temporarily rolls values backwards.
assert flattened[-1]["type"] == "snapshot"
assert client_values == {k: v for k, v in current.items() if k != "f.camera"}
assert not bus._subscribers
asyncio.run(run())
+112
View File
@@ -0,0 +1,112 @@
"""Deadlines retain worker capacity when ordinary work is blocked."""
import threading
import time
import pytest
from fluksio.flow import executor
from fluksio.flow.executor import ExecutionService
from fluksio.flow.messages import DType, MessageSpec
from fluksio.flow.nodes import Node
from fluksio.flow.pipeline import Pipeline
from fluksio.flow.queue import MemoryWorkQueue, WorkItem
def _node(name, function, requires=(), provides=()):
node = Node(
f=function,
requires=[MessageSpec(name=key, dtype=DType.FLOAT) for key in requires],
provides=[MessageSpec(name=key, dtype=DType.FLOAT) for key in provides],
name=name,
)
node.assign_flow("f", name)
return node
@pytest.mark.parametrize("reclaimed", [False, True])
def test_a_due_node_runs_while_ordinary_nodes_and_cascades_are_full(
monkeypatch, reclaimed
):
started, release, fired, recovered, touched = (threading.Event() for _ in range(5))
calls = []
class Queue(MemoryWorkQueue):
def reclaim_stale(self, min_idle_ms):
if not reclaimed or not started.is_set() or recovered.is_set():
return []
recovered.set()
return [
WorkItem(
kind="cascade",
node="f.source",
flow="f",
outputs={"f.raw": float(i)},
entry_id=f"recovered-{i}",
)
for i in range(3)
]
def touch(self, entry_ids):
if "recovered-0" in entry_ids:
touched.set()
def slow(raw, params):
calls.append(raw)
started.set()
assert release.wait(5)
source = _node("source", lambda params: None, provides=["raw"])
slow_node = _node("slow", slow, requires=["raw"])
timer = _node("timer", lambda params: None, provides=["stop"])
stop = _node("stop", lambda stop, params: fired.set(), requires=["stop"])
queue = Queue()
service = ExecutionService(queue, max_workers=1, max_cascades=1)
pipeline = Pipeline(
[source, slow_node, timer, stop], work_queue=queue, node_pool=service.node_pool
)
service.bind(pipeline)
monkeypatch.setattr(executor, "DELAYED_INTERVAL_S", 0.02)
monkeypatch.setattr(executor, "RECLAIM_INTERVAL_S", 0.02)
monkeypatch.setattr(executor, "TOUCH_INTERVAL_S", 0.02)
service.start()
try:
source.inject({"raw": 1.0})
assert started.wait(1)
if reclaimed:
assert touched.wait(1), "reclaimed work blocked the timer's heartbeat"
else:
for i in range(3):
source.inject({"raw": float(i)})
queue.add_delayed(
WorkItem(kind="cascade", node=timer.id, flow="f", outputs={"f.stop": 1.0}),
time.time(),
)
assert fired.wait(2), "the due node waited for an ordinary worker"
assert len(calls) == 1, "ordinary work used the timer reserve"
finally:
release.set()
service.stop()
def test_timers_can_use_all_idle_normal_workers():
"""A reserve must not cap an otherwise idle engine at two workers."""
service = ExecutionService(MemoryWorkQueue(), max_workers=3, max_cascades=3)
started = [threading.Event() for _ in range(3)]
release = threading.Event()
def block(index):
started[index].set()
assert release.wait(3)
futures = []
try:
for i in range(3):
futures.append(service._node_pools.due.submit(block, i))
assert all(event.wait(1) for event in started)
assert service._node_pools.available() == (0, executor.DUE_RESERVE)
finally:
release.set()
for future in futures:
future.result(1)
service.stop()
+55
View File
@@ -160,6 +160,61 @@ def test_a_flush_the_database_refused_is_written_by_the_next_one(
assert audit.detail == "held back" assert audit.detail == "held back"
def test_batched_rollups_add_and_redelivery_clears_a_finished_run(db: Session) -> None:
collector = MetricsCollector(EventBus())
ts = time.time()
for i in range(2):
collector.handle(
{"type": "cascade_started", "run": f"bulk-{i}", "flow": "bulk", "ts": ts}
)
collector.handle(
{
"type": "node_executed",
"flow": "bulk",
"node": f"bulk.n{i}",
"run": f"bulk-{i}",
"outputs": 1,
"duration_ms": 10.0,
"ts": ts,
}
)
collector.handle({"type": "cascade_finished", "run": "bulk-0", "ts": ts + 1})
# Write the same open/finished batch twice, as a retained run is upserted.
collector._write(collector._buckets, [], list(collector._runs.values()), False)
collector.handle(
{
"type": "cascade_started",
"run": "bulk-0",
"flow": "bulk",
"deliveries": 2,
"ts": ts + 2,
}
)
collector.handle({"type": "cascade_finished", "run": "bulk-1", "ts": ts + 3})
for bucket in collector._buckets.values():
bucket["duration_sum_ms"] = 4.0
bucket["duration_max_ms"] = 4.0
collector._write(collector._buckets, [], list(collector._runs.values()), False)
buckets = db.exec(select(MetricBucket).where(MetricBucket.flow == "bulk")).all()
assert len(buckets) == 2
assert all(
(b.executions, b.messages, b.duration_sum_ms, b.duration_max_ms)
== (2, 2, 14.0, 10.0)
for b in buckets
)
reopened = db.get(FlowRun, "bulk-0")
assert reopened is not None
assert (reopened.status, reopened.finished_at, reopened.deliveries) == (
"running",
None,
2,
)
finished = db.get(FlowRun, "bulk-1")
assert finished is not None and finished.finished_at is not None
assert finished.status == "ok"
def test_a_node_id_wider_than_the_column_still_records(db: Session) -> None: def test_a_node_id_wider_than_the_column_still_records(db: Session) -> None:
collector = MetricsCollector(EventBus()) collector = MetricsCollector(EventBus())
ts = datetime.now(UTC).replace(second=0, microsecond=0).timestamp() ts = datetime.now(UTC).replace(second=0, microsecond=0).timestamp()
+52
View File
@@ -1,7 +1,9 @@
import base64 import base64
import gzip
import hashlib import hashlib
from pathlib import Path from pathlib import Path
import pytest
from starlette.applications import Starlette from starlette.applications import Starlette
from starlette.responses import PlainTextResponse from starlette.responses import PlainTextResponse
from starlette.routing import Route from starlette.routing import Route
@@ -37,3 +39,53 @@ def test_the_dashboard_and_the_api_share_one_origin(tmp_path: Path) -> None:
assert client.get("/api/v1/ping").text == "api" assert client.get("/api/v1/ping").text == "api"
# Nothing outside the build directory is served, only the shell. # Nothing outside the build directory is served, only the shell.
assert THEME in client.get("/assets/../../etc/passwd").text assert THEME in client.get("/assets/../../etc/passwd").text
@pytest.mark.parametrize(
("accepted", "compressed"),
[
("gzip", True),
("br, gzip;q=0.8", True),
("gzip ; q=1, identity;q=0", True),
("*", True),
("gzip;q=0, *;q=1", False),
("gzip;q=0.5, identity;q=1", False),
("br", False),
("identity", False),
],
)
def test_precompressed_assets_keep_their_type_and_negotiate_encoding(
tmp_path: Path, accepted: str, compressed: bool
) -> None:
client = _client(tmp_path)
content = b"console.log('built once');\n" * 200
asset = tmp_path / "assets" / "app-1a2b.js"
asset.write_bytes(content)
packed = gzip.compress(content)
asset.with_name(asset.name + ".gz").write_bytes(packed)
response = client.get("/assets/app-1a2b.js", headers={"Accept-Encoding": accepted})
assert response.content == content
assert (response.headers.get("content-encoding") == "gzip") is compressed
assert "javascript" in response.headers["content-type"]
assert response.headers["vary"] == "Accept-Encoding"
assert "immutable" in response.headers["cache-control"]
expected = len(packed) if compressed else len(content)
assert int(response.headers["content-length"]) == expected
head = client.head("/assets/app-1a2b.js", headers={"Accept-Encoding": accepted})
assert head.content == b""
assert head.headers["content-length"] == str(expected)
def test_a_build_without_compressed_assets_still_serves_plain_files(
tmp_path: Path,
) -> None:
client = _client(tmp_path)
content = b"console.log('plain build');\n" * 200
(tmp_path / "assets" / "app-1a2b.js").write_bytes(content)
response = client.get("/assets/app-1a2b.js", headers={"Accept-Encoding": "gzip"})
assert response.content == content
assert "content-encoding" not in response.headers
+4
View File
@@ -80,6 +80,10 @@ make rebuild-frontend
This preserves the running stack's domain, overlays and LAN port. This preserves the running stack's domain, overlays and LAN port.
The frontend build writes gzip sidecars for compressible text assets. Keep
them beside the original files when deploying `dist/`. The API serves the
sidecar when the client accepts gzip and the original otherwise.
## Development services ## Development services
The development Compose file includes services for trying integrations: The development Compose file includes services for trying integrations:
+4
View File
@@ -151,6 +151,10 @@ The three concurrency limits must be at least `1`. They also have CLI flags:
engine logs a warning at startup and uses the value anyway. Resource flags are engine logs a warning at startup and uses the value anyway. Resource flags are
`--gpus`, `--ram` and `--worker-max-rss`. `--gpus`, `--ram` and `--worker-max-rss`.
Due timers can use up to two additional cascade threads and two additional
node-execution threads when the ordinary pools are busy. Declared resource
budgets still apply.
Nodes with declared resources are scheduled against the available budget. Nodes with declared resources are scheduled against the available budget.
Undeclared nodes use the shared pool and receive thread caps. Attached workers Undeclared nodes use the shared pool and receive thread caps. Attached workers
report their own inventory; the engine's resource settings describe only its report their own inventory; the engine's resource settings describe only its
+45 -30
View File
@@ -125,8 +125,8 @@ export const CURSOR: uPlot.Cursor = {
} as unknown as uPlot.Cursor.Bind, } as unknown as uPlot.Cursor.Bind,
drag: { drag: {
// Drag across the plot to read a stretch of it closer. `setScale` stays // Drag across the plot to read a stretch of it closer. `setScale` stays
// off because the chart owns its x range itself: `setData` runs on every // off because the chart owns its x range itself: `setData` would re-range
// render and would re-range the scales from the data, so a held window is // the scales from the data, so a held window is
// what tells it to leave them alone. Without that this only ever flashed a // what tells it to leave them alone. Without that this only ever flashed a
// selection box over a live chart, which is why it used to be off. // selection box over a live chart, which is why it used to be off.
x: true, x: true,
@@ -190,17 +190,35 @@ function tickLabels(ticks: number[]): string[] {
return ticks.map((value) => value.toFixed(decimals)) return ticks.map((value) => value.toFixed(decimals))
} }
/** The series joined onto one x axis, which is what uPlot draws. */ /** Numeric snapshots catch both rebuilt arrays and mutable caller buffers. */
function table(plots: HistoryPoint[][]): uPlot.AlignedData { export class ChartData {
return uPlot.join( private columns: uPlot.AlignedData[] = []
plots.map( private data: uPlot.AlignedData = [[]]
(plot) =>
[ update(plots: HistoryPoint[][]): uPlot.AlignedData {
plot.map((point) => point.ts), let changed = plots.length !== this.columns.length
plot.map((point) => point.value), this.columns.length = plots.length
] as uPlot.AlignedData, plots.forEach((plot, index) => {
), const previous = this.columns[index]
) if (
previous?.[0].length === plot.length &&
plot.every(
(point, i) =>
Object.is(point.ts, previous[0][i]) &&
Object.is(point.value, previous[1][i]),
)
) {
return
}
this.columns[index] = [
plot.map((point) => point.ts),
plot.map((point) => point.value),
]
changed = true
})
if (changed) this.data = plots.length ? uPlot.join(this.columns) : [[]]
return this.data
}
} }
/** /**
@@ -264,12 +282,16 @@ export function UplotChart({
const host = useRef<HTMLDivElement>(null) const host = useRef<HTMLDivElement>(null)
const legend = useRef<HTMLDivElement>(null) const legend = useRef<HTMLDivElement>(null)
const chart = useRef<uPlot | null>(null) const chart = useRef<uPlot | null>(null)
const prepared = useRef<ChartData | null>(null)
if (prepared.current === null) prepared.current = new ChartData()
// A dragged x window, held so the next reading does not wash it away. The // A dragged x window, held so the next reading does not wash it away. The
// ref is what the data effect reads; the state is only what draws the way // ref is what the data effect reads; the state is only what draws the way
// back out, and the two are set together. // back out, and the two are set together.
const zoomed = useRef(false) const zoomed = useRef(false)
const resetRange = useRef(false)
const [showReset, setShowReset] = useState(false) const [showReset, setShowReset] = useState(false)
const clearZoom = () => { const clearZoom = () => {
if (zoomed.current) resetRange.current = true
zoomed.current = false zoomed.current = false
setShowReset(false) setShowReset(false)
} }
@@ -432,10 +454,11 @@ export function UplotChart({
}, },
// Built with the readings it already has: uPlot's axes are only half // Built with the readings it already has: uPlot's axes are only half
// initialised while its scales have no range. // initialised while its scales have no range.
table(plots), prepared.current!.update(plots),
element, element,
) )
chart.current = plot chart.current = plot
resetRange.current = false
const observer = new ResizeObserver(() => { const observer = new ResizeObserver(() => {
plot.setSize({ plot.setSize({
@@ -455,27 +478,19 @@ export function UplotChart({
} }
}, [key, ready]) }, [key, ready])
// Every render, deliberately. `plots` is rebuilt on every render in both // Check contents every render: callers may rebuild arrays or mutate buffers.
// consumers, so no memo can hit, and any signature short of hashing shares // Only changed readings need joining and drawing. A zoom reset still ranges
// the blind spot a point count has: once a rolling window is full, a refetch // the current data, and a held window expires once all its points roll off.
// carrying different readings leaves the count where it was and never fires.
// Safe to run this often because `setData` is idempotent and re-ranges the
// scales *from the data* — the opposite of the `redraw(false)` below. That
// re-ranging is exactly what a dragged window has to be spared, so while one
// is held the data goes in and the scales stay where they were put. Clearing
// the window renders, which brings the next pass through here with the reset
// back on: that is what puts the whole range back.
//
// A live chart keeps a count of points rather than a span, so a held window
// scrolls off the left as readings arrive. Once its last point has gone it
// is let go, or the chart would sit empty until somebody reset it.
useEffect(() => { useEffect(() => {
const plot = chart.current const plot = chart.current
if (!plot || plots.length === 0) return if (!plot || plots.length === 0) return
const data = table(plots) const data = prepared.current!.update(plots)
const held = plot.scales.x.max const held = plot.scales.x.max
if (zoomed.current && held != null && held < data[0][0]) clearZoom() if (zoomed.current && held != null && held < data[0][0]) clearZoom()
plot.setData(data, !zoomed.current) if (data !== plot.data || resetRange.current) {
plot.setData(data, !zoomed.current)
resetRange.current = false
}
}) })
// The canvas cannot follow a CSS variable, so a theme swap is a redraw. The // The canvas cannot follow a CSS variable, so a theme swap is a redraw. The
@@ -0,0 +1,68 @@
/** Chart data stays current without joining unchanged series on every render. */
import assert from "node:assert/strict"
import uPlot from "uplot"
import { chartTails } from "../Dashboard/ChartWidget"
import { ChartData } from "./UplotChart"
const cache = new ChartData()
const plots = Array.from({ length: 5 }, (_, series) =>
Array.from({ length: 500 }, (_, i) => ({
ts: i + series / 10,
value: i * (series + 1),
})),
)
const join = uPlot.join
let joins = 0
uPlot.join = (...args) => {
joins++
return join(...args)
}
try {
const initial = cache.update(plots)
for (let render = 0; render < 100; render++) {
const rebuilt = plots.map((plot) => plot.map((point) => ({ ...point })))
assert.equal(cache.update(rebuilt), initial)
}
assert.equal(joins, 1)
// A correction in the middle of an existing mutable buffer is still data.
plots[0][200].value = -7
const corrected = cache.update(plots)
assert.notEqual(corrected, initial)
assert.equal(corrected[1][corrected[0].indexOf(200)], -7)
assert.equal(initial[1][initial[0].indexOf(200)], 200)
// A full window rolls forward without changing its point count.
plots[0].shift()
plots[0].push({ ts: 500, value: 12 })
const rolled = cache.update(plots)
assert.notEqual(rolled, corrected)
assert.equal(rolled[0][0], 0.1)
assert.equal(rolled[1][rolled[0].indexOf(500)], 12)
const reordered = cache.update([plots[1], plots[0]])
assert.equal(reordered.length, 3)
assert.equal(reordered[1][reordered[0].indexOf(200.1)], 400)
assert.deepEqual(cache.update([]), [[]])
} finally {
uPlot.join = join
}
let tails = chartTails({}, ["a"], [{ ts: 1, value: 10 }], 3)
assert.equal(chartTails(tails, ["a"], [{ ts: 1, value: 10 }], 3), tails)
tails = chartTails(tails, ["a"], [{ ts: 1, value: 11 }], 3)
assert.deepEqual(tails.a, [{ ts: 1, value: 11 }])
tails = chartTails(tails, ["a"], [{ ts: 2, value: 12 }], 3)
tails = chartTails(tails, ["a"], [{ ts: 3, value: 13 }], 3)
tails = chartTails(tails, ["a"], [{ ts: 3, value: 13 }], 2)
assert.deepEqual(tails.a, [
{ ts: 2, value: 12 },
{ ts: 3, value: 13 },
])
tails = chartTails(tails, ["b"], [undefined], 2)
assert.deepEqual(tails, { b: [] })
assert.equal(chartTails(tails, ["b"], [undefined], 2), tails)
assert.equal(chartTails(tails, ["b", "b"], [undefined, undefined], 2), tails)
console.log("chart data: ok (101 unchanged renders, one join)")
@@ -8,7 +8,11 @@ import {
type Range, type Range,
} from "@/components/Common/RangePicker" } from "@/components/Common/RangePicker"
import { MAX_SERIES, UplotChart } from "@/components/Common/UplotChart" import { MAX_SERIES, UplotChart } from "@/components/Common/UplotChart"
import { useLiveValue, useLiveValues } from "@/components/Flow/liveStore" import {
type LiveValue,
useLiveValue,
useLiveValues,
} from "@/components/Flow/liveStore"
import { import {
compareQueryOptions, compareQueryOptions,
NO_CURVE, NO_CURVE,
@@ -91,6 +95,36 @@ function merge(fetched: HistoryPoint[], tail: HistoryPoint[], cap: number) {
return [...fetched, ...tail.filter((point) => point.ts > since)].slice(-cap) return [...fetched, ...tail.filter((point) => point.ts > since)].slice(-cap)
} }
/** Advance the bounded live tails without scheduling a stale chart commit. */
export function chartTails(
previous: Record<string, HistoryPoint[]>,
names: string[],
live: (LiveValue | undefined)[],
cap: number,
): Record<string, HistoryPoint[]> {
const next: Record<string, HistoryPoint[]> = {}
let changed = false
names.forEach((name, index) => {
const kept = previous[name] ?? []
const point = live[index]
const last = kept[kept.length - 1]
let tail = kept
if (typeof point?.value === "number" && point.ts) {
if (last?.ts !== point.ts) {
tail = [...kept, { ts: point.ts, value: point.value }]
} else if (!Object.is(last.value, point.value)) {
tail = [...kept.slice(0, -1), { ts: point.ts, value: point.value }]
}
}
if (tail.length > cap) tail = tail.slice(-cap)
next[name] = tail
if (tail !== previous[name]) changed = true
})
return changed || Object.keys(previous).length !== Object.keys(next).length
? next
: previous
}
/** /**
* A chart, drawn from whichever source it was pointed at. * A chart, drawn from whichever source it was pointed at.
* *
@@ -275,34 +309,10 @@ function LiveChart({ widget }: WidgetProps) {
// What arrived over the socket since the history was fetched. Appending beats // What arrived over the socket since the history was fetched. Appending beats
// refetching the whole series every time a value lands. // refetching the whole series every time a value lands.
const [tails, setTails] = useState<Record<string, HistoryPoint[]>>({}) const [tails, setTails] = useState<Record<string, HistoryPoint[]>>({})
const key = names.join(" ") const nextTails = chartTails(tails, names, live, cap)
const stamps = live.map((value) => value?.ts ?? 0).join(",") // React retries this component before rendering its children. An effect
// biome-ignore lint/correctness/useExhaustiveDependencies: the timestamps are what makes a tail longer; `live` is a fresh array every render. // would commit the old chart first, then render and draw it a second time.
useEffect(() => { if (nextTails !== tails) setTails(nextTails)
setTails((previous) => {
const next: Record<string, HistoryPoint[]> = {}
let changed = false
names.forEach((name, index) => {
const point = live[index]
const kept = previous[name] ?? []
const ts = point?.ts
if (
typeof point?.value !== "number" ||
!ts ||
kept[kept.length - 1]?.ts === ts
) {
next[name] = kept
return
}
next[name] = [...kept, { ts, value: point.value }].slice(-cap)
changed = true
})
// Dropping a series is a change too, even without a new reading.
return changed || Object.keys(previous).length !== names.length
? next
: previous
})
}, [key, stamps, cap])
const plots = names.map((name, index) => const plots = names.map((name, index) =>
merge( merge(
+30 -1
View File
@@ -1,8 +1,36 @@
import { readdir, readFile, writeFile } from "node:fs/promises"
import path from "node:path" import path from "node:path"
import { gzipSync } from "node:zlib"
import tailwindcss from "@tailwindcss/vite" import tailwindcss from "@tailwindcss/vite"
import { tanstackRouter } from "@tanstack/router-plugin/vite" import { tanstackRouter } from "@tanstack/router-plugin/vite"
import react from "@vitejs/plugin-react-swc" import react from "@vitejs/plugin-react-swc"
import { defineConfig } from "vite" import { defineConfig, type Plugin } from "vite"
/** Compress once at build time, including text copied from public/. */
function precompress(): Plugin {
async function compress(directory: string): Promise<void> {
const entries = await readdir(directory, { withFileTypes: true })
await Promise.all(
entries.map(async (entry) => {
const file = path.join(directory, entry.name)
if (entry.isDirectory()) return compress(file)
if (!/\.(?:js|css|html|svg|json|webmanifest)$/.test(entry.name)) return
const original = await readFile(file)
if (original.length < 1024) return
const compressed = gzipSync(original, { level: 9 })
if (compressed.length < original.length)
await writeFile(`${file}.gz`, compressed)
}),
)
}
return {
name: "precompressed-assets",
apply: "build",
async writeBundle({ dir }) {
if (dir) await compress(dir)
},
}
}
// https://vitejs.dev/config/ // https://vitejs.dev/config/
export default defineConfig({ export default defineConfig({
@@ -39,5 +67,6 @@ export default defineConfig({
}), }),
react(), react(),
tailwindcss(), tailwindcss(),
precompress(),
], ],
}) })