Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s

This commit is contained in:
2026-10-29 14:24:17 +01:00
parent 20a0853f69
commit 75b3a93094
25 changed files with 1316 additions and 301 deletions
+24 -10
View File
@@ -1700,8 +1700,8 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
await websocket.send_text(orjson.dumps(payload).decode())
return list(payload.get("demand", []))
async def catch_up() -> bool:
"""Rescope a panel and send the snapshot again, demand included.
async def rescope() -> bool:
"""Recheck a panel and its demand before forwarding retained events.
False once a device whose credential stopped standing is closed with
1008, which sends it to check: otherwise its frames would still be
@@ -1722,12 +1722,18 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
device_def = _device_def(websocket.app, device)
wanted &= only
demand.want(viewer, wanted)
return True
async def catch_up() -> bool:
if not await rescope():
return False
await send_snapshot()
return True
told = await send_snapshot()
async with event_bus.subscribe() as queue:
# Listen before reading state: a value changing while the snapshot is
# sent must follow it on the socket, even if it never changes again.
told = await send_snapshot()
receiver = asyncio.create_task(_receive(websocket))
# Where a live view this socket is one end of reaches it. Its own
# outbox rather than the bus's queue, which drops when full.
@@ -1741,6 +1747,7 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
await websocket.send_text(demand_frame(ports))
live_view.sessions.attach(device, endpoint)
outbox = asyncio.create_task(endpoint.next())
catching_up = False
try:
while True:
sender = asyncio.create_task(queue.get())
@@ -1845,10 +1852,13 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
except asyncio.QueueEmpty:
break
# A panel this far behind lost events the bus dropped for it:
# its demand, a dashboard published or its own unpairing. It
# catches up once this batch is out, as after a dashboard.
resync = only is not None and event_bus.dropped(queue)
# Every client catches up after overflow. A lost event may be
# a value that never changes again, or a device's unpairing.
lost = event_bus.dropped(queue)
catching_up = catching_up or lost
if lost and not await rescope():
return
resync = False
if only is None:
fresh = hidden_from(user, websocket.app)
if fresh != hidden:
@@ -1883,8 +1893,12 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
for frame in media_frames(out, wanted, store):
await websocket.send_bytes(frame)
await _send(websocket, out)
if resync and not await catch_up():
return
# Send retained events before the recovery snapshot: values
# left in the queue would otherwise overwrite its newer state.
if resync or (catching_up and queue.empty()):
if not await catch_up():
return
catching_up = catching_up and not queue.empty()
except (WebSocketDisconnect, RuntimeError):
# A peer that goes away mid-send takes the RuntimeError route
# ("websocket.send after websocket.close") rather than the clean
+3 -5
View File
@@ -19,7 +19,6 @@ from fluksio.api.deps import (
get_current_user,
)
from fluksio.flow.devices import DeviceDef, DeviceStore
from fluksio.flow.messages import flow_of
from fluksio.flow.pipeline import ValueSource
from fluksio.flow.state import as_number
@@ -118,7 +117,7 @@ def _port_source(request: Request, name: str) -> ValueSource | None:
@router.post("/{name}", response_model=MessageValue)
async def publish_message(
def publish_message(
name: str, body: PublishRequest, controller: FlowControllerDep, request: Request
) -> Any:
"""Put a value into the graph, as a dashboard control does.
@@ -133,7 +132,7 @@ async def publish_message(
detail=body.source_detail,
)
try:
await run_in_threadpool(controller.publish_message, name, body.value, source)
controller.publish_message(name, body.value, source)
except KeyError:
raise HTTPException(
status_code=404, detail=f"No flow declares a message named '{name}'"
@@ -141,8 +140,7 @@ async def publish_message(
except TypeError as exc:
raise HTTPException(status_code=422, detail=str(exc))
values = controller.values(flow_of(name))
current = values.get(name, {})
current = controller.value(name)
return MessageValue(name=name, value=current.get("value"), ts=current.get("ts"))
+42 -32
View File
@@ -722,16 +722,6 @@ class CloudConnector:
# The cameras a person may not see, looked at again per event.
hidden = hidden_from(user, self._app, device=only is not None)
controller = getattr(self._app.state, "flow_controller", None)
told: list[str] = []
if controller is not None:
# A device here sends its frames over HTTP: the tunnel carries
# text only.
snapshot = snapshot_payload(controller, only, device, "http", hidden)
told = list(snapshot.get("demand", []))
await socket.send(
_dump({"op": "ws_msg", "id": stream_id, "text": _dump(snapshot)})
)
# The media this client's tiles are drawing count as demand on the
# devices sending them, as a direct socket's do. The bytes themselves
# are fetched one by one through the portal.
@@ -743,6 +733,20 @@ class CloudConnector:
outbox: asyncio.Task[str] | None = None
sender: asyncio.Task[dict[str, Any]] | None = None
async def rescope() -> bool:
nonlocal only, wanted
if only is not None:
if device and revoked(token, self._app):
await socket.send(
_dump({"op": "ws_close", "id": stream_id, "code": 1008})
)
return False
# A deleted panel scopes to nothing, never to every message.
only = panel_scope(token, self._app) or set()
wanted &= only
demand.want(viewer, wanted)
return True
def heard(text: str) -> None:
nonlocal wanted
asked = wanted_names(text, only, hidden)
@@ -768,6 +772,19 @@ class CloudConnector:
live_view.sessions.attach(device, endpoint)
try:
async with event_bus.subscribe() as queue:
# Subscribe before the snapshot so changes during its send
# cannot fall between the snapshot and the live stream.
told: list[str] = []
if controller is not None:
snapshot = snapshot_payload(
controller, only, device, "http", hidden
)
told = list(snapshot.get("demand", []))
await socket.send(
_dump(
{"op": "ws_msg", "id": stream_id, "text": _dump(snapshot)}
)
)
outbox = asyncio.create_task(endpoint.next())
if device and (ports := demand.ports(device)) != told:
await socket.send(
@@ -779,6 +796,7 @@ class CloudConnector:
}
)
)
catching_up = False
while True:
sender = asyncio.create_task(queue.get())
done, _ = await asyncio.wait(
@@ -796,11 +814,13 @@ class CloudConnector:
if sender not in done:
continue
event = sender.result()
# Behind far enough to lose events, a device catches up
# as after a dashboard: the lost one may have been its
# demand or its unpairing. A person whose cameras opened
# or closed is sent what may be seen now.
resync = only is not None and event_bus.dropped(queue)
# A browser needs the same recovery as a device after
# overflow: the lost value may never be published again.
lost = event_bus.dropped(queue)
catching_up = catching_up or lost
if lost and not await rescope():
return
resync = False
if only is None:
fresh = hidden_from(user, self._app)
if fresh != hidden:
@@ -824,24 +844,10 @@ class CloudConnector:
{"op": "ws_msg", "id": stream_id, "text": _dump(event)}
)
)
if not resync:
if not resync and not (catching_up and queue.empty()):
continue
if only is not None:
if device and revoked(token, self._app):
# Unpaired or deleted: 1008 sends it to check.
await socket.send(
_dump({"op": "ws_close", "id": stream_id, "code": 1008})
)
return
# What this screen shows may have just changed under
# it. Rescope, then resend the snapshot so a dashboard
# it has only now been assigned draws values instead of
# blanks. ``or set()`` is the point: a panel that was
# deleted scopes to nothing, and None would widen this
# socket to everything on the bus.
only = panel_scope(token, self._app) or set()
wanted &= only
demand.want(viewer, wanted)
if not await rescope():
return
if controller is not None:
catch_up = snapshot_payload(
controller, only, device, "http", hidden
@@ -855,6 +861,10 @@ class CloudConnector:
}
)
)
# The snapshot follows the retained backlog. If sending
# it queued more events, finish those before recovering
# again so an older event cannot leave the client stale.
catching_up = catching_up and not queue.empty()
if kind == "dashboard_changed" and only is not None:
# After the snapshot, as the device socket sends it.
await socket.send(
+3
View File
@@ -1545,6 +1545,9 @@ class FlowController:
return _short_error(exc)
return None
def value(self, name: str) -> dict[str, Any]:
return self.pipeline.value(name) if self.pipeline else {}
def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]:
return self.pipeline.values(flow) if self.pipeline else {}
+112 -26
View File
@@ -1,9 +1,10 @@
"""Event bus bridging the engine's worker threads to async subscribers.
Nodes execute in a thread pool; websocket clients live on the event loop.
Publishers are therefore thread-safe and never block: a subscriber that cannot
keep up loses its oldest queued events rather than stalling the engine, and
can ask whether it did (``dropped``) to catch up another way.
Publishers are therefore thread-safe and never wait for subscribers. Both
staging and subscriber queues are bounded, retaining errors and control events
ahead of routine traffic. Subscribers can ask whether they lost events
(``dropped``) to catch up another way.
"""
from __future__ import annotations
@@ -19,24 +20,88 @@ from typing import Any
logger = logging.getLogger(__name__)
QUEUE_SIZE = 256
DISPATCH_BUDGET = 1024
LOG_HISTORY = 400
#: How many of a message's last artifact digests are remembered.
RECENT_DIGESTS = 8
def _critical(event: dict[str, Any]) -> bool:
"""Keep failures, lifecycle changes and unknown events ahead of telemetry."""
kind = event.get("type")
if kind == "node_log":
return event.get("level") == "error"
return kind not in {
"message_value",
"node_started",
"node_executed",
"work_latency",
"run_metric",
}
class _EventBuffer:
"""Two FIFO lanes allow constant-time eviction without reordering events."""
def __init__(self) -> None:
self._ordinary: deque[tuple[int, dict[str, Any]]] = deque()
self._critical: deque[tuple[int, dict[str, Any]]] = deque()
self._sequence = 0
def __len__(self) -> int:
return len(self._ordinary) + len(self._critical)
def append(self, event: dict[str, Any]) -> None:
lane = self._critical if _critical(event) else self._ordinary
lane.append((self._sequence, event))
self._sequence += 1
def popleft(self) -> dict[str, Any]:
lane = self._critical
if self._ordinary and (not lane or self._ordinary[0][0] < lane[0][0]):
lane = self._ordinary
return lane.popleft()[1]
def make_room(self, event: dict[str, Any]) -> bool:
"""Evict the oldest ordinary event, or keep a full critical backlog."""
if self._ordinary:
self._ordinary.popleft()
elif _critical(event):
self._critical.popleft()
else:
return False
return True
class _EventQueue(asyncio.Queue[dict[str, Any]]):
def _init(self, maxsize: int) -> None:
self._queue = _EventBuffer()
def offer(self, event: dict[str, Any]) -> bool:
"""Enqueue without waiting and report any loss, including rejection."""
lost = self.full()
if not lost or self._queue.make_room(event):
self.put_nowait(event)
return lost
class EventBus:
"""Fan-out of engine events to any number of async subscribers."""
def __init__(self) -> None:
self._loop: asyncio.AbstractEventLoop | None = None
self._subscribers: set[asyncio.Queue[dict[str, Any]]] = set()
self._subscribers: set[_EventQueue] = set()
# Events published since the last flush was scheduled. A cascade
# publishes 13-16 of them and each used to cost its own
# `call_soon_threadsafe`; they now cost one between two turns of the
# loop, which is what a wave is.
self._pending: list[dict[str, Any]] = []
self._pending = _EventBuffer()
self._pending_dropped = False
self._pending_lock = threading.Lock()
self._flush_scheduled = False
self._batch: list[dict[str, Any]] = []
self._batch_dropped = False
self._targets: deque[_EventQueue] = deque()
# Kept whether or not anyone is listening, so opening the log panel
# shows what just happened rather than an empty box.
self.recent_logs: deque[dict[str, Any]] = deque(maxlen=LOG_HISTORY)
@@ -82,7 +147,12 @@ class EventBus:
if loop is None or not self._subscribers:
return
with self._pending_lock:
self._pending.append(event)
room = True
if len(self._pending) == QUEUE_SIZE:
self._pending_dropped = True
room = self._pending.make_room(event)
if room:
self._pending.append(event)
if self._flush_scheduled:
# A flush is already on its way and has not run yet, so it
# will find this event too.
@@ -93,33 +163,49 @@ class EventBus:
except RuntimeError:
# Loop already closed — shutting down.
with self._pending_lock:
self._pending.clear()
self._pending = _EventBuffer()
self._pending_dropped = False
self._flush_scheduled = False
def _dispatch(self) -> None:
"""Hand everything published since the last turn to every subscriber."""
with self._pending_lock:
batch = self._pending
self._pending = []
self._flush_scheduled = False
if not batch:
return
for queue in self._subscribers:
for event in batch:
if queue.full():
# Drop the oldest so a slow client never blocks the engine.
try:
queue.get_nowait()
except asyncio.QueueEmpty:
pass
"""Deliver bounded work, yielding between batches or subscriber groups."""
if not self._targets:
with self._pending_lock:
self._batch = [
self._pending.popleft() for _ in range(len(self._pending))
]
self._batch_dropped = self._pending_dropped
self._pending_dropped = False
self._targets = deque(self._subscribers)
if self._batch:
# A subscriber receives the whole bounded batch before it resumes,
# so its catch-up snapshot cannot precede the rest of that batch.
count = max(1, DISPATCH_BUDGET // len(self._batch))
for _ in range(min(count, len(self._targets))):
queue = self._targets.popleft()
if queue not in self._subscribers:
continue
if self._batch_dropped:
self._dropped.add(queue)
queue.put_nowait(event)
for event in self._batch:
if queue.offer(event):
self._dropped.add(queue)
else:
self._targets.clear()
if not self._targets:
self._batch = []
with self._pending_lock:
if not self._pending:
self._flush_scheduled = False
return
if self._loop is not None:
self._loop.call_soon(self._dispatch)
def dropped(self, queue: asyncio.Queue[dict[str, Any]]) -> bool:
"""Whether this subscriber lost events since it last asked.
A device's socket that did catches up with a snapshot: what it lost
may have been its demand, or the event that tells it it was unpaired.
This includes staging overflow before fan-out. A socket catches up
with a snapshot, since a lost value might never change again.
"""
if queue in self._dropped:
self._dropped.discard(queue)
@@ -129,7 +215,7 @@ class EventBus:
@asynccontextmanager
async def subscribe(self) -> AsyncIterator[asyncio.Queue[dict[str, Any]]]:
"""Yield a queue receiving every event published while subscribed."""
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=QUEUE_SIZE)
queue = _EventQueue(maxsize=QUEUE_SIZE)
self._subscribers.add(queue)
try:
yield queue
+123 -24
View File
@@ -17,8 +17,10 @@ from __future__ import annotations
import logging
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING, Any
from collections import deque
from collections.abc import Callable
from concurrent.futures import Executor, Future, ThreadPoolExecutor
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
from fluksio.flow.queue import MAX_DELIVERIES, WorkItem, WorkQueue
@@ -41,9 +43,9 @@ TOUCH_INTERVAL_S = 20.0
# resolution of a delay: a delayed item is waited for exactly, so what a timer
# fires late by is a wake-up and a promotion rather than up to a whole second.
DELAYED_INTERVAL_S = 1.0
# Cascade slots a promoted timer may use past `max_cascades`. A due item was
# Extra cascade and node workers reserved for promoted timers. A due item was
# already waited for, so making it queue behind whatever long node happens to
# hold the pool is the one lateness the sleeping timer thread cannot remove.
# hold either pool is lateness the sleeping timer thread cannot remove.
DUE_RESERVE = 2
# How long the saturated engine waits on the due lane before going back to
# check whether a cascade slot has freed.
@@ -67,6 +69,73 @@ BACKLOG_INTERVAL_S = 5.0
BACKLOG_DEGRADED = 50
BACKLOG_STRIKES = 3
_P = ParamSpec("_P")
_T = TypeVar("_T")
class _PoolLane(Executor):
def __init__(self, pool: _ReservedPool, due: bool) -> None:
self._pool = pool
self._due = due
def submit(
self, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs
) -> Future[_T]:
return self._pool.submit(self._due, fn, *args, **kwargs)
class _ReservedPool:
"""Due work uses idle normal workers, or its own bounded reserve."""
def __init__(
self, workers: int, prefix: str, on_done: Callable[[], None] | None = None
) -> None:
self._limits = (workers, DUE_RESERVE)
self._pools = (
ThreadPoolExecutor(max_workers=workers, thread_name_prefix=prefix),
ThreadPoolExecutor(
max_workers=DUE_RESERVE, thread_name_prefix=f"{prefix}-due"
),
)
self._pending = [0, 0]
self._lock = threading.Lock()
self._on_done = on_done
self.normal = _PoolLane(self, False)
self.due = _PoolLane(self, True)
def available(self) -> tuple[int, int]:
with self._lock:
return (
max(0, self._limits[0] - self._pending[0]),
max(0, self._limits[1] - self._pending[1]),
)
def _done(self, lane: int) -> None:
with self._lock:
self._pending[lane] -= 1
if self._on_done is not None:
self._on_done()
def submit(
self, due: bool, fn: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs
) -> Future[_T]:
with self._lock:
# Count queued work too: an idle thread with ordinary work already
# waiting is not capacity a timer can use without waiting behind it.
lane = int(due and self._pending[0] >= self._limits[0])
self._pending[lane] += 1
try:
future = self._pools[lane].submit(fn, *args, **kwargs)
except RuntimeError:
self._done(lane)
raise
future.add_done_callback(lambda _: self._done(lane))
return future
def shutdown(self) -> None:
for pool in self._pools:
pool.shutdown(wait=False)
class ExecutionService:
"""Owns the engine's worker threads and the queue they read from."""
@@ -93,12 +162,15 @@ class ExecutionService:
self._inflight_lock = threading.Condition()
# Entry ids claimed and still running, under _inflight_lock.
self._active: set[str] = set()
self.node_pool = ThreadPoolExecutor(
max_workers=MAX_WORKERS if max_workers is None else max_workers,
thread_name_prefix="node",
# At most one reclaim batch waits here. The timer thread must keep
# promoting and touching work while the consumer waits for capacity.
self._reclaimed: deque[WorkItem] = deque()
self._node_pools = _ReservedPool(
MAX_WORKERS if max_workers is None else max_workers, "node"
)
self._cascade_pool = ThreadPoolExecutor(
max_workers=self.max_cascades, thread_name_prefix="cascade"
self.node_pool = self._node_pools.normal
self._cascade_pools = _ReservedPool(
self.max_cascades, "cascade", self._capacity_changed
)
self._consumer: threading.Thread | None = None
self._timers: threading.Thread | None = None
@@ -132,14 +204,18 @@ class ExecutionService:
thread.join(timeout=5)
self._consumer = None
self._timers = None
self._cascade_pool.shutdown(wait=False)
self.node_pool.shutdown(wait=False)
self._cascade_pools.shutdown()
self._node_pools.shutdown()
self.queue.close()
def bind(self, pipeline: Pipeline) -> None:
"""Point the service at the pipeline it should execute against."""
self._pipeline = pipeline
def _capacity_changed(self) -> None:
with self._inflight_lock:
self._inflight_lock.notify_all()
def pause_intake(self) -> None:
"""Stop claiming, and wait for what is already claimed to finish.
@@ -182,6 +258,18 @@ class ExecutionService:
free, due_only = self._await_capacity()
if not free:
continue
with self._inflight_lock:
recovered = [
item
for item in self._reclaimed
if not due_only or item.not_before > 0
][:free]
for item in recovered:
self._reclaimed.remove(item)
if recovered:
for item in recovered:
self._dispatch(item)
continue
try:
items = self.queue.claim(
free,
@@ -262,21 +350,23 @@ class ExecutionService:
if now - last_reclaim < RECLAIM_INTERVAL_S:
continue
with self._inflight_lock:
if self._reclaimed or not self._intake.is_set():
continue
last_reclaim = now
try:
for item in self.queue.reclaim_stale(RECLAIM_IDLE_MS):
recovered = self.queue.reclaim_stale(RECLAIM_IDLE_MS)
for item in recovered:
logger.info(
"Reclaimed work for '%s' (delivery %d)",
item.node,
item.deliveries,
)
# Through the same gate the main loop uses: a reclaim can
# return sixty-odd entries at once, and dispatching them
# all would push `_inflight` far past `max_cascades` —
# exactly the overcommit the gate exists to prevent.
if not self._await_capacity()[0]:
break
self._dispatch(item)
with self._inflight_lock:
self._reclaimed.extend(recovered)
self._active.update(
item.entry_id for item in recovered if item.entry_id
)
except Exception as exc:
logger.error("Could not reclaim stale work: %s", exc)
@@ -331,7 +421,9 @@ class ExecutionService:
"""
with self._inflight_lock:
while not self._stop.is_set():
free = self.max_cascades - self._inflight
if not self._intake.is_set():
return 0, False
free, _ = self._cascade_pools.available()
if free > 0:
return free, False
# Only once a slot has genuinely failed to free: the due lane
@@ -340,7 +432,7 @@ class ExecutionService:
# read every time the pool filled up.
if self._inflight_lock.wait(0.5):
continue
reserve = self.max_cascades + DUE_RESERVE - self._inflight
_, reserve = self._cascade_pools.available()
if reserve > 0:
return reserve, True
return 0, False
@@ -351,7 +443,12 @@ class ExecutionService:
if item.entry_id:
self._active.add(item.entry_id)
try:
self._cascade_pool.submit(self._handle, item)
pool = (
self._cascade_pools.due
if item.not_before > 0
else self._cascade_pools.normal
)
pool.submit(self._handle, item)
except RuntimeError:
# Pool already shutting down.
self._done(item)
@@ -386,7 +483,7 @@ class ExecutionService:
return item.node
def _handle(self, item: WorkItem) -> None:
handled = True
handled = False
try:
handled = self._run_item(item)
except Exception:
@@ -473,11 +570,12 @@ class ExecutionService:
replay = item.deliveries > 1
emission = item.kind == "emission"
node_pool = self._node_pools.due if item.not_before > 0 else self.node_pool
try:
if item.kind == "flush":
# A rate-limit window ended; nothing to replay, only to let out.
# A run of its own, so what it sets off is recorded somewhere.
pipeline.flush(node, item.entry_id)
pipeline.flush(node, item.entry_id, node_pool=node_pool)
return True
# An emission's values went into state when the node produced them;
# this item carries them so its readers get the chunk that caused
@@ -498,6 +596,7 @@ class ExecutionService:
# Neither can say what changed, so neither filters on it.
changed=None if replay or not item.outputs else published,
overrides=item.outputs if emission else None,
node_pool=node_pool,
)
finally:
# Paired, or a cascade that raised — state backend gone, say — is a
+23 -13
View File
@@ -394,13 +394,8 @@ class MetricsCollector:
prune: bool,
) -> None:
with Session(engine) as session:
for (flow, node, minute), agg in buckets.items():
statement = insert(MetricBucket).values(
flow=flow,
node=node,
bucket=datetime.fromtimestamp(minute, UTC),
**agg,
)
if buckets:
statement = insert(MetricBucket)
# The same minute is written several times, so the counters add
# and the maxima take whichever is larger. Columns are read by
# subscript: `excluded.items` is the collection's own method.
@@ -418,21 +413,36 @@ class MetricsCollector:
name: func.max(col(getattr(MetricBucket, name)), new[name])
for name in MAXIMA
},
)
),
[
{
"flow": flow,
"node": node,
"bucket": datetime.fromtimestamp(minute, UTC),
**agg,
}
for (flow, node, minute), agg in buckets.items()
],
)
for run in runs:
values = {k: v for k, v in run.items() if k != "started_ts"}
statement = insert(FlowRun).values(**values)
if runs:
values = [
{k: v for k, v in run.items() if k != "started_ts"} for run in runs
]
statement = insert(FlowRun)
session.execute(
statement.on_conflict_do_update(
index_elements=["id"],
set_={
key: statement.excluded[key]
for key in values
for key in values[0]
if key != "id"
},
)
),
values,
# A redelivery clears finished_at. Include nulls in the
# batch so open and finished runs share the same statement.
execution_options={"render_nulls": True},
)
session.add_all(pending)
+88 -98
View File
@@ -21,10 +21,11 @@ import time
import uuid
from collections import deque
from collections.abc import Callable, Iterator
from concurrent.futures import Future, ThreadPoolExecutor, wait
from concurrent.futures import Executor, Future, ThreadPoolExecutor
from contextlib import contextmanager
from datetime import UTC, datetime
from functools import lru_cache
from queue import SimpleQueue
from typing import Any, Literal, Protocol
from pydantic import BaseModel, model_validator
@@ -275,6 +276,7 @@ class Pipeline:
"_paused",
"_stepping",
"_gate_lock",
"_gate_revision",
"_graph_lock",
"_queue",
"_node_pool",
@@ -294,7 +296,7 @@ class Pipeline:
initial_values: dict[str, Any] | None = None,
disabled_flows: set[str] | None = None,
work_queue: WorkQueue | None = None,
node_pool: ThreadPoolExecutor | None = None,
node_pool: Executor | None = None,
observer: Callable[[NodeOutcome], None] | None = None,
emission_observer: Callable[[str, dict[str, Any]], None] | None = None,
run_cache: RunCacheLookup | None = None,
@@ -307,6 +309,7 @@ class Pipeline:
self._paused: set[str] = set()
self._stepping: set[str] = set()
self._gate_lock = threading.Lock()
self._gate_revision = 0
# Held only while the graph maps are read or swapped together, never
# across an await, a node call or any I/O. Reentrant because the lazy
# `edges` build re-enters through `_graph`. It is never nested with
@@ -448,6 +451,7 @@ class Pipeline:
with self._gate_lock:
self._paused.discard(flow)
self._stepping.discard(flow)
self._gate_revision += 1
def remove_flow(self, flow: str) -> None:
"""Take a deleted flow out of the graph."""
@@ -742,7 +746,9 @@ class Pipeline:
self._state[self._flush_key(node.id)] = at
self.defer(node, {}, at - now, kind="flush", parent=entry_id)
def flush(self, node: Node, entry_id: str = "") -> None:
def flush(
self, node: Node, entry_id: str = "", *, node_pool: Executor | None = None
) -> None:
"""Let through what this node's rate limits held back.
Runs when a window ends: the last value a limited output kept back is
@@ -766,7 +772,9 @@ class Pipeline:
# nothing came out and there is nothing downstream to wake.
published = self.apply_outputs(node, held, entry_id)
if published:
self.run_downstream(node, entry_id=entry_id, changed=published)
self.run_downstream(
node, entry_id=entry_id, changed=published, node_pool=node_pool
)
if any(spec.interval > 0 for spec in node.requires.values()):
# A wake-up that was held back, not a value that just changed —
# so this one deliberately walks everything reachable.
@@ -775,6 +783,7 @@ class Pipeline:
self._state,
check_ready=True,
entry_id=entry_id,
node_pool=node_pool,
)
def _increment_message_versions(self, outputs: dict[str, Any]) -> None:
@@ -1252,7 +1261,9 @@ class Pipeline:
def set_disabled(self, flows: set[str]) -> None:
"""Which flows are stopped. Swapped whole, so a reader never sees half."""
self._disabled = frozenset(flows)
with self._gate_lock:
self._disabled = frozenset(flows)
self._gate_revision += 1
def paused_flows(self) -> list[str]:
with self._gate_lock:
@@ -1262,11 +1273,13 @@ class Pipeline:
"""Hold this flow's nodes. Values still arrive; nothing acts on them."""
with self._gate_lock:
self._paused.add(flow)
self._gate_revision += 1
self._publish({"type": "flow_paused", "flow": flow, "paused": True})
def resume(self, flow: str) -> None:
with self._gate_lock:
self._paused.discard(flow)
self._gate_revision += 1
self._publish({"type": "flow_paused", "flow": flow, "paused": False})
# Whatever was held back is free to run now.
self._execute_parallel(self.flow_nodes(flow), self._state, check_ready=True)
@@ -1280,11 +1293,13 @@ class Pipeline:
"""
with self._gate_lock:
self._stepping.add(flow)
self._gate_revision += 1
try:
yield
finally:
with self._gate_lock:
self._stepping.discard(flow)
self._gate_revision += 1
def _gate_blocks(self, node: Node) -> bool:
"""Is this node's flow held back from executing?"""
@@ -1311,6 +1326,7 @@ class Pipeline:
replay: bool = False,
changed: set[str] | None = None,
overrides: dict[str, Any] | None = None,
node_pool: Executor | None = None,
) -> StateBackend:
"""Execute nodes concurrently, scheduling each as its inputs arrive.
@@ -1344,9 +1360,11 @@ class Pipeline:
# Copied: the caller's set is not this wave's to grow.
fresh = set(changed) if changed is not None else None
submitted: set[Node] = set()
skipped: set[Node] = set()
node_futures: dict[Node, Future[dict[str, Any] | None]] = {}
ready_nodes = deque(n for n, count in in_degree.items() if count == 0)
blocked: set[Node] = set()
gate_revision = self._gate_revision
node_futures: dict[Future[dict[str, Any] | None], Node] = {}
finished: SimpleQueue[Future[dict[str, Any] | None]] = SimpleQueue()
def untouched(n: Node) -> bool:
"""Did this wave publish nothing this node waits on?"""
@@ -1356,108 +1374,72 @@ class Pipeline:
)
def complete(n: Node) -> None:
"""Count a node as done without running it, freeing its consumers."""
skipped.add(n)
"""Release consumers when their last dependency finishes or skips."""
for consumer in edges[n]:
if consumer in target_nodes:
in_degree[consumer] -= 1
if in_degree[consumer] == 0:
ready_nodes.append(consumer)
def submit_ready(executor: ThreadPoolExecutor) -> None:
# To a fixpoint: completing a node without running it frees its
# consumers, and one this pass has already walked past would
# otherwise be stranded — there is no future outstanding to bring
# the drain loop back round for it.
progressed = True
while progressed:
progressed = False
for n in target_nodes:
if n in submitted or n in skipped:
continue
# Gate before the readiness check, so a held-back node does
# not spend the synchronous claim it needs once it may run.
if self._gate_blocks(n) or in_degree[n] != 0:
continue
if n.passive:
# Something outside publishes for it; a run is not it.
def submit_ready(executor: Executor) -> None:
nonlocal gate_revision
with self._gate_lock:
revision = self._gate_revision
if blocked and revision != gate_revision:
# A flow can resume while another branch of this wave runs.
# Revisit only its held candidates, and only if a gate moved.
ready_nodes.extend(blocked)
blocked.clear()
gate_revision = revision
while ready_nodes:
n = ready_nodes.popleft()
# Gate before readiness, so a pause spends no synchronous claim.
if self._gate_blocks(n):
blocked.add(n)
continue
if n.passive:
complete(n)
continue
if fresh is not None and untouched(n):
complete(n)
continue
ready_inputs: dict[str, Any] | None = None
if check_ready:
ready, ready_inputs = self._is_node_ready(n, state, entry_id)
if not ready:
# Its old output remains usable by a consumer whose
# other producer did publish during this wave.
complete(n)
progressed = True
continue
if fresh is not None and untouched(n):
# Ahead of the readiness check, so an unchanged node
# spends neither a delivery stamp nor a synchronous
# claim on a run it is not going to make.
complete(n)
progressed = True
continue
ready_inputs: dict[str, Any] | None = None
if check_ready:
# The values are kept whether or not the answer is yes:
# a node that may run reads them below instead of
# asking state for the same keys again.
ready, ready_inputs = self._is_node_ready(n, state, entry_id)
if not ready:
# Completed, not merely passed over: a later trigger
# may still make it ready, but not running is not
# the same as blocking. Whatever it last published
# is in state, and a consumer reading that alongside
# something this wave *did* publish has every right
# to run on the pair.
#
# Passing over it held that consumer for the whole
# wave, and a node that can never be ready — an
# input nothing has ever published — took its
# consumers out for good. On this house
# `plugs.pump_run` waits on a watering pulse that
# only exists at 02:00, and it sits upstream of
# `dmx.switches`: every boiler, plug and appliance
# command reached its own message and stopped there,
# reaching the DMX universe only in the occasional
# wave the pump node happened not to be part of.
complete(n)
progressed = True
continue
if replay and entry_id and self._already_done(entry_id, n):
# Its side effect happened on an earlier delivery; its
# outputs are still in state, so downstream carries on.
complete(n)
progressed = True
continue
submitted.add(n)
node_futures[n] = executor.submit(
self._execute_node, n, state, entry_id, overrides, ready_inputs
)
if replay and entry_id and self._already_done(entry_id, n):
complete(n)
continue
future = executor.submit(
self._execute_node, n, state, entry_id, overrides, ready_inputs
)
node_futures[future] = n
future.add_done_callback(finished.put)
def drain(executor: ThreadPoolExecutor) -> None:
def drain(executor: Executor) -> None:
submit_ready(executor)
while node_futures:
done, _ = wait(node_futures.values(), return_when="FIRST_COMPLETED")
completed = [n for n, f in node_futures.items() if f in done]
# A completion queue also avoids scanning every outstanding
# future whenever one node finishes.
future = finished.get()
n = node_futures.pop(future)
result = future.result()
if result is not None and fresh is not None:
fresh.update(result)
# Silent, failed and throttled producers still release their
# consumers. Fresh inputs decide whether those consumers run.
complete(n)
submit_ready(executor)
for n in completed:
result = node_futures.pop(n).result()
# Before the decrement: a consumer freed by this node is
# judged on what it just published.
if result is not None and fresh is not None:
fresh.update(result)
# Decremented whatever it returned. A node that published
# nothing — rate limited, unchanged, or failed — has not
# *blocked* its consumers; it has merely given them nothing
# to read, and `untouched` above is the accurate test of
# that. Holding the count instead made a quiet producer
# veto a noisy sibling: `dmx.switches` reads both boilers
# through report-by-exception nodes, so the kitchen one
# staying silent — which is almost always — kept the main
# boiler's command from ever reaching the DMX universe.
for consumer in edges[n]:
if consumer in target_nodes:
in_degree[consumer] -= 1
submit_ready(executor)
if self._node_pool is not None:
pool = node_pool if node_pool is not None else self._node_pool
if pool is not None:
# The execution service owns a long-lived pool; building one per
# wave is what used to spawn threads without bound under load.
drain(self._node_pool)
drain(pool)
else:
with ThreadPoolExecutor(max_workers=self._max_workers) as executor:
drain(executor)
@@ -1519,6 +1501,7 @@ class Pipeline:
replay: bool = False,
changed: set[str] | None = None,
overrides: dict[str, Any] | None = None,
node_pool: Executor | None = None,
) -> StateBackend:
"""Run everything downstream of a node that has just published.
@@ -1552,6 +1535,7 @@ class Pipeline:
replay=replay,
changed=changed,
overrides=overrides,
node_pool=node_pool,
)
def trigger(
@@ -1796,6 +1780,12 @@ class Pipeline:
# Losing the value outright would be worse than running it here.
self._run_here(node, outputs, cause="external", delivered=kind == "emission")
def value(self, name: str) -> dict[str, Any]:
"""One message and its timestamp, read together without listing state."""
stamp = self._timestamp_key(name)
found = self._state.get_multi([name, stamp])
return {"value": found.get(name), "ts": found.get(stamp)}
def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]:
"""Last value and timestamp of every message, optionally one flow's.
+25 -29
View File
@@ -53,6 +53,23 @@ STALE_CONSUMER_IDLE_MS = 3_600_000
# count, since redelivery happens seconds after the claim and never later.
DONE_MARKERS = 10_000
# A promotion is one Redis operation, including when the caller loses its
# connection before hearing the result. Write first: Lua does not roll back a
# command error, so a refused XADD must leave its timer in the delayed set.
_MOVE_DUE = """
local due = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[1], 'LIMIT', 0, 100)
for _, raw in ipairs(due) do
local fields = {}
for key, value in pairs(cjson.decode(raw)) do
fields[#fields + 1] = key
fields[#fields + 1] = value
end
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[2], '*', unpack(fields))
redis.call('ZREM', KEYS[1], raw)
end
return #due
"""
@dataclass
class WorkItem:
@@ -446,6 +463,7 @@ class RedisWorkQueue(WorkQueue):
# finding them with a keyspace scan walked every state and idempotency
# key in the database, twice per health poll.
self._parked_flows_key = f"{namespace}:__parked_flows__"
self._move_due = self._redis.register_script(_MOVE_DUE)
self._ensure_group()
self._drop_stale_consumers()
@@ -619,36 +637,14 @@ class RedisWorkQueue(WorkQueue):
]
def move_due(self, now: float) -> int:
due = cast(
list[str],
self._redis.zrangebyscore(self._delayed_key, "-inf", now, start=0, num=100),
"""Move up to 100 timers into the due lane atomically."""
return cast(
int,
self._move_due(
keys=[self._delayed_key, self._due_stream],
args=[now, STREAM_MAXLEN],
),
)
if not due:
return 0
# Whoever removes it owns it: a second engine gets 0 for that member and
# leaves it alone. One round trip for the batch rather than one each,
# which matters because this runs every second.
pipe = self._redis.pipeline()
for raw in due:
pipe.zrem(self._delayed_key, raw)
claimed = cast(list[int], pipe.execute())
pipe = self._redis.pipeline()
moved = 0
for raw, owned in zip(due, claimed, strict=True):
if owned:
# Into the due lane, which `claim` reads first: what has waited
# out a deadline should not then queue behind a backlog.
pipe.xadd(
self._due_stream,
cast(Any, orjson.loads(raw)),
maxlen=STREAM_MAXLEN,
approximate=True,
)
moved += 1
if moved:
pipe.execute()
return moved
def dead_letter(self, item: WorkItem, reason: str) -> None:
fields = item.to_fields()
+28 -3
View File
@@ -15,7 +15,7 @@ import mimetypes
import re
from pathlib import Path
from starlette.middleware.gzip import GZipMiddleware
from starlette.datastructures import Headers
from starlette.responses import FileResponse
from starlette.types import ASGIApp, Receive, Scope, Send
@@ -26,6 +26,24 @@ API_PREFIXES = ("/api/", "/hooks/", "/mcp", "/.well-known/", "/docs", "/redoc")
mimetypes.add_type("application/manifest+json", ".webmanifest")
def _accepts_gzip(header: str) -> bool:
"""An explicit refusal overrides a wildcard, including on a cached asset."""
qualities: dict[str, float] = {}
for entry in header.lower().split(","):
encoding, *parameters = entry.strip().split(";")
quality = 1.0
for parameter in parameters:
key, _, value = parameter.strip().partition("=")
if key == "q":
try:
quality = float(value)
except ValueError:
quality = 0.0
qualities[encoding.strip()] = quality if 0.0 <= quality <= 1.0 else 0.0
gzip = qualities.get("gzip", qualities.get("*", 0.0))
return gzip > 0 and gzip >= qualities.get("identity", 0.0)
def content_security_policy(index_html: str) -> str:
"""The policy for the dashboard, admitting its inline scripts by hash.
@@ -84,5 +102,12 @@ class Dashboard:
# assets, so a stale copy pins the browser to the previous build.
file = self.index
headers = {**self.headers, "Cache-Control": "no-cache"}
response = FileResponse(file, headers=headers)
await GZipMiddleware(response, minimum_size=1024)(scope, receive, send)
media_type = mimetypes.guess_type(file.name)[0] or "text/plain"
compressed = file.with_name(file.name + ".gz").resolve()
if compressed.is_relative_to(self.dist) and compressed.is_file():
headers["Vary"] = "Accept-Encoding"
if _accepts_gzip(Headers(scope=scope).get("accept-encoding", "")):
file = compressed
headers["Content-Encoding"] = "gzip"
response = FileResponse(file, headers=headers, media_type=media_type)
await response(scope, receive, send)