Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s

This commit is contained in:
2026-10-29 14:24:17 +01:00
parent 20a0853f69
commit 75b3a93094
25 changed files with 1316 additions and 301 deletions
+88 -98
View File
@@ -21,10 +21,11 @@ import time
import uuid
from collections import deque
from collections.abc import Callable, Iterator
from concurrent.futures import Future, ThreadPoolExecutor, wait
from concurrent.futures import Executor, Future, ThreadPoolExecutor
from contextlib import contextmanager
from datetime import UTC, datetime
from functools import lru_cache
from queue import SimpleQueue
from typing import Any, Literal, Protocol
from pydantic import BaseModel, model_validator
@@ -275,6 +276,7 @@ class Pipeline:
"_paused",
"_stepping",
"_gate_lock",
"_gate_revision",
"_graph_lock",
"_queue",
"_node_pool",
@@ -294,7 +296,7 @@ class Pipeline:
initial_values: dict[str, Any] | None = None,
disabled_flows: set[str] | None = None,
work_queue: WorkQueue | None = None,
node_pool: ThreadPoolExecutor | None = None,
node_pool: Executor | None = None,
observer: Callable[[NodeOutcome], None] | None = None,
emission_observer: Callable[[str, dict[str, Any]], None] | None = None,
run_cache: RunCacheLookup | None = None,
@@ -307,6 +309,7 @@ class Pipeline:
self._paused: set[str] = set()
self._stepping: set[str] = set()
self._gate_lock = threading.Lock()
self._gate_revision = 0
# Held only while the graph maps are read or swapped together, never
# across an await, a node call or any I/O. Reentrant because the lazy
# `edges` build re-enters through `_graph`. It is never nested with
@@ -448,6 +451,7 @@ class Pipeline:
with self._gate_lock:
self._paused.discard(flow)
self._stepping.discard(flow)
self._gate_revision += 1
def remove_flow(self, flow: str) -> None:
"""Take a deleted flow out of the graph."""
@@ -742,7 +746,9 @@ class Pipeline:
self._state[self._flush_key(node.id)] = at
self.defer(node, {}, at - now, kind="flush", parent=entry_id)
def flush(self, node: Node, entry_id: str = "") -> None:
def flush(
self, node: Node, entry_id: str = "", *, node_pool: Executor | None = None
) -> None:
"""Let through what this node's rate limits held back.
Runs when a window ends: the last value a limited output kept back is
@@ -766,7 +772,9 @@ class Pipeline:
# nothing came out and there is nothing downstream to wake.
published = self.apply_outputs(node, held, entry_id)
if published:
self.run_downstream(node, entry_id=entry_id, changed=published)
self.run_downstream(
node, entry_id=entry_id, changed=published, node_pool=node_pool
)
if any(spec.interval > 0 for spec in node.requires.values()):
# A wake-up that was held back, not a value that just changed —
# so this one deliberately walks everything reachable.
@@ -775,6 +783,7 @@ class Pipeline:
self._state,
check_ready=True,
entry_id=entry_id,
node_pool=node_pool,
)
def _increment_message_versions(self, outputs: dict[str, Any]) -> None:
@@ -1252,7 +1261,9 @@ class Pipeline:
def set_disabled(self, flows: set[str]) -> None:
"""Which flows are stopped. Swapped whole, so a reader never sees half."""
self._disabled = frozenset(flows)
with self._gate_lock:
self._disabled = frozenset(flows)
self._gate_revision += 1
def paused_flows(self) -> list[str]:
with self._gate_lock:
@@ -1262,11 +1273,13 @@ class Pipeline:
"""Hold this flow's nodes. Values still arrive; nothing acts on them."""
with self._gate_lock:
self._paused.add(flow)
self._gate_revision += 1
self._publish({"type": "flow_paused", "flow": flow, "paused": True})
def resume(self, flow: str) -> None:
with self._gate_lock:
self._paused.discard(flow)
self._gate_revision += 1
self._publish({"type": "flow_paused", "flow": flow, "paused": False})
# Whatever was held back is free to run now.
self._execute_parallel(self.flow_nodes(flow), self._state, check_ready=True)
@@ -1280,11 +1293,13 @@ class Pipeline:
"""
with self._gate_lock:
self._stepping.add(flow)
self._gate_revision += 1
try:
yield
finally:
with self._gate_lock:
self._stepping.discard(flow)
self._gate_revision += 1
def _gate_blocks(self, node: Node) -> bool:
"""Is this node's flow held back from executing?"""
@@ -1311,6 +1326,7 @@ class Pipeline:
replay: bool = False,
changed: set[str] | None = None,
overrides: dict[str, Any] | None = None,
node_pool: Executor | None = None,
) -> StateBackend:
"""Execute nodes concurrently, scheduling each as its inputs arrive.
@@ -1344,9 +1360,11 @@ class Pipeline:
# Copied: the caller's set is not this wave's to grow.
fresh = set(changed) if changed is not None else None
submitted: set[Node] = set()
skipped: set[Node] = set()
node_futures: dict[Node, Future[dict[str, Any] | None]] = {}
ready_nodes = deque(n for n, count in in_degree.items() if count == 0)
blocked: set[Node] = set()
gate_revision = self._gate_revision
node_futures: dict[Future[dict[str, Any] | None], Node] = {}
finished: SimpleQueue[Future[dict[str, Any] | None]] = SimpleQueue()
def untouched(n: Node) -> bool:
"""Did this wave publish nothing this node waits on?"""
@@ -1356,108 +1374,72 @@ class Pipeline:
)
def complete(n: Node) -> None:
"""Count a node as done without running it, freeing its consumers."""
skipped.add(n)
"""Release consumers when their last dependency finishes or skips."""
for consumer in edges[n]:
if consumer in target_nodes:
in_degree[consumer] -= 1
if in_degree[consumer] == 0:
ready_nodes.append(consumer)
def submit_ready(executor: ThreadPoolExecutor) -> None:
# To a fixpoint: completing a node without running it frees its
# consumers, and one this pass has already walked past would
# otherwise be stranded — there is no future outstanding to bring
# the drain loop back round for it.
progressed = True
while progressed:
progressed = False
for n in target_nodes:
if n in submitted or n in skipped:
continue
# Gate before the readiness check, so a held-back node does
# not spend the synchronous claim it needs once it may run.
if self._gate_blocks(n) or in_degree[n] != 0:
continue
if n.passive:
# Something outside publishes for it; a run is not it.
def submit_ready(executor: Executor) -> None:
nonlocal gate_revision
with self._gate_lock:
revision = self._gate_revision
if blocked and revision != gate_revision:
# A flow can resume while another branch of this wave runs.
# Revisit only its held candidates, and only if a gate moved.
ready_nodes.extend(blocked)
blocked.clear()
gate_revision = revision
while ready_nodes:
n = ready_nodes.popleft()
# Gate before readiness, so a pause spends no synchronous claim.
if self._gate_blocks(n):
blocked.add(n)
continue
if n.passive:
complete(n)
continue
if fresh is not None and untouched(n):
complete(n)
continue
ready_inputs: dict[str, Any] | None = None
if check_ready:
ready, ready_inputs = self._is_node_ready(n, state, entry_id)
if not ready:
# Its old output remains usable by a consumer whose
# other producer did publish during this wave.
complete(n)
progressed = True
continue
if fresh is not None and untouched(n):
# Ahead of the readiness check, so an unchanged node
# spends neither a delivery stamp nor a synchronous
# claim on a run it is not going to make.
complete(n)
progressed = True
continue
ready_inputs: dict[str, Any] | None = None
if check_ready:
# The values are kept whether or not the answer is yes:
# a node that may run reads them below instead of
# asking state for the same keys again.
ready, ready_inputs = self._is_node_ready(n, state, entry_id)
if not ready:
# Completed, not merely passed over: a later trigger
# may still make it ready, but not running is not
# the same as blocking. Whatever it last published
# is in state, and a consumer reading that alongside
# something this wave *did* publish has every right
# to run on the pair.
#
# Passing over it held that consumer for the whole
# wave, and a node that can never be ready — an
# input nothing has ever published — took its
# consumers out for good. On this house
# `plugs.pump_run` waits on a watering pulse that
# only exists at 02:00, and it sits upstream of
# `dmx.switches`: every boiler, plug and appliance
# command reached its own message and stopped there,
# reaching the DMX universe only in the occasional
# wave the pump node happened not to be part of.
complete(n)
progressed = True
continue
if replay and entry_id and self._already_done(entry_id, n):
# Its side effect happened on an earlier delivery; its
# outputs are still in state, so downstream carries on.
complete(n)
progressed = True
continue
submitted.add(n)
node_futures[n] = executor.submit(
self._execute_node, n, state, entry_id, overrides, ready_inputs
)
if replay and entry_id and self._already_done(entry_id, n):
complete(n)
continue
future = executor.submit(
self._execute_node, n, state, entry_id, overrides, ready_inputs
)
node_futures[future] = n
future.add_done_callback(finished.put)
def drain(executor: ThreadPoolExecutor) -> None:
def drain(executor: Executor) -> None:
submit_ready(executor)
while node_futures:
done, _ = wait(node_futures.values(), return_when="FIRST_COMPLETED")
completed = [n for n, f in node_futures.items() if f in done]
# A completion queue also avoids scanning every outstanding
# future whenever one node finishes.
future = finished.get()
n = node_futures.pop(future)
result = future.result()
if result is not None and fresh is not None:
fresh.update(result)
# Silent, failed and throttled producers still release their
# consumers. Fresh inputs decide whether those consumers run.
complete(n)
submit_ready(executor)
for n in completed:
result = node_futures.pop(n).result()
# Before the decrement: a consumer freed by this node is
# judged on what it just published.
if result is not None and fresh is not None:
fresh.update(result)
# Decremented whatever it returned. A node that published
# nothing — rate limited, unchanged, or failed — has not
# *blocked* its consumers; it has merely given them nothing
# to read, and `untouched` above is the accurate test of
# that. Holding the count instead made a quiet producer
# veto a noisy sibling: `dmx.switches` reads both boilers
# through report-by-exception nodes, so the kitchen one
# staying silent — which is almost always — kept the main
# boiler's command from ever reaching the DMX universe.
for consumer in edges[n]:
if consumer in target_nodes:
in_degree[consumer] -= 1
submit_ready(executor)
if self._node_pool is not None:
pool = node_pool if node_pool is not None else self._node_pool
if pool is not None:
# The execution service owns a long-lived pool; building one per
# wave is what used to spawn threads without bound under load.
drain(self._node_pool)
drain(pool)
else:
with ThreadPoolExecutor(max_workers=self._max_workers) as executor:
drain(executor)
@@ -1519,6 +1501,7 @@ class Pipeline:
replay: bool = False,
changed: set[str] | None = None,
overrides: dict[str, Any] | None = None,
node_pool: Executor | None = None,
) -> StateBackend:
"""Run everything downstream of a node that has just published.
@@ -1552,6 +1535,7 @@ class Pipeline:
replay=replay,
changed=changed,
overrides=overrides,
node_pool=node_pool,
)
def trigger(
@@ -1796,6 +1780,12 @@ class Pipeline:
# Losing the value outright would be worse than running it here.
self._run_here(node, outputs, cause="external", delivered=kind == "emission")
def value(self, name: str) -> dict[str, Any]:
"""One message and its timestamp, read together without listing state."""
stamp = self._timestamp_key(name)
found = self._state.get_multi([name, stamp])
return {"value": found.get(name), "ts": found.get(stamp)}
def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]:
"""Last value and timestamp of every message, optionally one flow's.