Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
This commit is contained in:
@@ -21,10 +21,11 @@ import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from collections.abc import Callable, Iterator
|
||||
from concurrent.futures import Future, ThreadPoolExecutor, wait
|
||||
from concurrent.futures import Executor, Future, ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from functools import lru_cache
|
||||
from queue import SimpleQueue
|
||||
from typing import Any, Literal, Protocol
|
||||
|
||||
from pydantic import BaseModel, model_validator
|
||||
@@ -275,6 +276,7 @@ class Pipeline:
|
||||
"_paused",
|
||||
"_stepping",
|
||||
"_gate_lock",
|
||||
"_gate_revision",
|
||||
"_graph_lock",
|
||||
"_queue",
|
||||
"_node_pool",
|
||||
@@ -294,7 +296,7 @@ class Pipeline:
|
||||
initial_values: dict[str, Any] | None = None,
|
||||
disabled_flows: set[str] | None = None,
|
||||
work_queue: WorkQueue | None = None,
|
||||
node_pool: ThreadPoolExecutor | None = None,
|
||||
node_pool: Executor | None = None,
|
||||
observer: Callable[[NodeOutcome], None] | None = None,
|
||||
emission_observer: Callable[[str, dict[str, Any]], None] | None = None,
|
||||
run_cache: RunCacheLookup | None = None,
|
||||
@@ -307,6 +309,7 @@ class Pipeline:
|
||||
self._paused: set[str] = set()
|
||||
self._stepping: set[str] = set()
|
||||
self._gate_lock = threading.Lock()
|
||||
self._gate_revision = 0
|
||||
# Held only while the graph maps are read or swapped together, never
|
||||
# across an await, a node call or any I/O. Reentrant because the lazy
|
||||
# `edges` build re-enters through `_graph`. It is never nested with
|
||||
@@ -448,6 +451,7 @@ class Pipeline:
|
||||
with self._gate_lock:
|
||||
self._paused.discard(flow)
|
||||
self._stepping.discard(flow)
|
||||
self._gate_revision += 1
|
||||
|
||||
def remove_flow(self, flow: str) -> None:
|
||||
"""Take a deleted flow out of the graph."""
|
||||
@@ -742,7 +746,9 @@ class Pipeline:
|
||||
self._state[self._flush_key(node.id)] = at
|
||||
self.defer(node, {}, at - now, kind="flush", parent=entry_id)
|
||||
|
||||
def flush(self, node: Node, entry_id: str = "") -> None:
|
||||
def flush(
|
||||
self, node: Node, entry_id: str = "", *, node_pool: Executor | None = None
|
||||
) -> None:
|
||||
"""Let through what this node's rate limits held back.
|
||||
|
||||
Runs when a window ends: the last value a limited output kept back is
|
||||
@@ -766,7 +772,9 @@ class Pipeline:
|
||||
# nothing came out and there is nothing downstream to wake.
|
||||
published = self.apply_outputs(node, held, entry_id)
|
||||
if published:
|
||||
self.run_downstream(node, entry_id=entry_id, changed=published)
|
||||
self.run_downstream(
|
||||
node, entry_id=entry_id, changed=published, node_pool=node_pool
|
||||
)
|
||||
if any(spec.interval > 0 for spec in node.requires.values()):
|
||||
# A wake-up that was held back, not a value that just changed —
|
||||
# so this one deliberately walks everything reachable.
|
||||
@@ -775,6 +783,7 @@ class Pipeline:
|
||||
self._state,
|
||||
check_ready=True,
|
||||
entry_id=entry_id,
|
||||
node_pool=node_pool,
|
||||
)
|
||||
|
||||
def _increment_message_versions(self, outputs: dict[str, Any]) -> None:
|
||||
@@ -1252,7 +1261,9 @@ class Pipeline:
|
||||
|
||||
def set_disabled(self, flows: set[str]) -> None:
|
||||
"""Which flows are stopped. Swapped whole, so a reader never sees half."""
|
||||
self._disabled = frozenset(flows)
|
||||
with self._gate_lock:
|
||||
self._disabled = frozenset(flows)
|
||||
self._gate_revision += 1
|
||||
|
||||
def paused_flows(self) -> list[str]:
|
||||
with self._gate_lock:
|
||||
@@ -1262,11 +1273,13 @@ class Pipeline:
|
||||
"""Hold this flow's nodes. Values still arrive; nothing acts on them."""
|
||||
with self._gate_lock:
|
||||
self._paused.add(flow)
|
||||
self._gate_revision += 1
|
||||
self._publish({"type": "flow_paused", "flow": flow, "paused": True})
|
||||
|
||||
def resume(self, flow: str) -> None:
|
||||
with self._gate_lock:
|
||||
self._paused.discard(flow)
|
||||
self._gate_revision += 1
|
||||
self._publish({"type": "flow_paused", "flow": flow, "paused": False})
|
||||
# Whatever was held back is free to run now.
|
||||
self._execute_parallel(self.flow_nodes(flow), self._state, check_ready=True)
|
||||
@@ -1280,11 +1293,13 @@ class Pipeline:
|
||||
"""
|
||||
with self._gate_lock:
|
||||
self._stepping.add(flow)
|
||||
self._gate_revision += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with self._gate_lock:
|
||||
self._stepping.discard(flow)
|
||||
self._gate_revision += 1
|
||||
|
||||
def _gate_blocks(self, node: Node) -> bool:
|
||||
"""Is this node's flow held back from executing?"""
|
||||
@@ -1311,6 +1326,7 @@ class Pipeline:
|
||||
replay: bool = False,
|
||||
changed: set[str] | None = None,
|
||||
overrides: dict[str, Any] | None = None,
|
||||
node_pool: Executor | None = None,
|
||||
) -> StateBackend:
|
||||
"""Execute nodes concurrently, scheduling each as its inputs arrive.
|
||||
|
||||
@@ -1344,9 +1360,11 @@ class Pipeline:
|
||||
# Copied: the caller's set is not this wave's to grow.
|
||||
fresh = set(changed) if changed is not None else None
|
||||
|
||||
submitted: set[Node] = set()
|
||||
skipped: set[Node] = set()
|
||||
node_futures: dict[Node, Future[dict[str, Any] | None]] = {}
|
||||
ready_nodes = deque(n for n, count in in_degree.items() if count == 0)
|
||||
blocked: set[Node] = set()
|
||||
gate_revision = self._gate_revision
|
||||
node_futures: dict[Future[dict[str, Any] | None], Node] = {}
|
||||
finished: SimpleQueue[Future[dict[str, Any] | None]] = SimpleQueue()
|
||||
|
||||
def untouched(n: Node) -> bool:
|
||||
"""Did this wave publish nothing this node waits on?"""
|
||||
@@ -1356,108 +1374,72 @@ class Pipeline:
|
||||
)
|
||||
|
||||
def complete(n: Node) -> None:
|
||||
"""Count a node as done without running it, freeing its consumers."""
|
||||
skipped.add(n)
|
||||
"""Release consumers when their last dependency finishes or skips."""
|
||||
for consumer in edges[n]:
|
||||
if consumer in target_nodes:
|
||||
in_degree[consumer] -= 1
|
||||
if in_degree[consumer] == 0:
|
||||
ready_nodes.append(consumer)
|
||||
|
||||
def submit_ready(executor: ThreadPoolExecutor) -> None:
|
||||
# To a fixpoint: completing a node without running it frees its
|
||||
# consumers, and one this pass has already walked past would
|
||||
# otherwise be stranded — there is no future outstanding to bring
|
||||
# the drain loop back round for it.
|
||||
progressed = True
|
||||
while progressed:
|
||||
progressed = False
|
||||
for n in target_nodes:
|
||||
if n in submitted or n in skipped:
|
||||
continue
|
||||
# Gate before the readiness check, so a held-back node does
|
||||
# not spend the synchronous claim it needs once it may run.
|
||||
if self._gate_blocks(n) or in_degree[n] != 0:
|
||||
continue
|
||||
if n.passive:
|
||||
# Something outside publishes for it; a run is not it.
|
||||
def submit_ready(executor: Executor) -> None:
|
||||
nonlocal gate_revision
|
||||
with self._gate_lock:
|
||||
revision = self._gate_revision
|
||||
if blocked and revision != gate_revision:
|
||||
# A flow can resume while another branch of this wave runs.
|
||||
# Revisit only its held candidates, and only if a gate moved.
|
||||
ready_nodes.extend(blocked)
|
||||
blocked.clear()
|
||||
gate_revision = revision
|
||||
while ready_nodes:
|
||||
n = ready_nodes.popleft()
|
||||
# Gate before readiness, so a pause spends no synchronous claim.
|
||||
if self._gate_blocks(n):
|
||||
blocked.add(n)
|
||||
continue
|
||||
if n.passive:
|
||||
complete(n)
|
||||
continue
|
||||
if fresh is not None and untouched(n):
|
||||
complete(n)
|
||||
continue
|
||||
ready_inputs: dict[str, Any] | None = None
|
||||
if check_ready:
|
||||
ready, ready_inputs = self._is_node_ready(n, state, entry_id)
|
||||
if not ready:
|
||||
# Its old output remains usable by a consumer whose
|
||||
# other producer did publish during this wave.
|
||||
complete(n)
|
||||
progressed = True
|
||||
continue
|
||||
if fresh is not None and untouched(n):
|
||||
# Ahead of the readiness check, so an unchanged node
|
||||
# spends neither a delivery stamp nor a synchronous
|
||||
# claim on a run it is not going to make.
|
||||
complete(n)
|
||||
progressed = True
|
||||
continue
|
||||
ready_inputs: dict[str, Any] | None = None
|
||||
if check_ready:
|
||||
# The values are kept whether or not the answer is yes:
|
||||
# a node that may run reads them below instead of
|
||||
# asking state for the same keys again.
|
||||
ready, ready_inputs = self._is_node_ready(n, state, entry_id)
|
||||
if not ready:
|
||||
# Completed, not merely passed over: a later trigger
|
||||
# may still make it ready, but not running is not
|
||||
# the same as blocking. Whatever it last published
|
||||
# is in state, and a consumer reading that alongside
|
||||
# something this wave *did* publish has every right
|
||||
# to run on the pair.
|
||||
#
|
||||
# Passing over it held that consumer for the whole
|
||||
# wave, and a node that can never be ready — an
|
||||
# input nothing has ever published — took its
|
||||
# consumers out for good. On this house
|
||||
# `plugs.pump_run` waits on a watering pulse that
|
||||
# only exists at 02:00, and it sits upstream of
|
||||
# `dmx.switches`: every boiler, plug and appliance
|
||||
# command reached its own message and stopped there,
|
||||
# reaching the DMX universe only in the occasional
|
||||
# wave the pump node happened not to be part of.
|
||||
complete(n)
|
||||
progressed = True
|
||||
continue
|
||||
if replay and entry_id and self._already_done(entry_id, n):
|
||||
# Its side effect happened on an earlier delivery; its
|
||||
# outputs are still in state, so downstream carries on.
|
||||
complete(n)
|
||||
progressed = True
|
||||
continue
|
||||
submitted.add(n)
|
||||
node_futures[n] = executor.submit(
|
||||
self._execute_node, n, state, entry_id, overrides, ready_inputs
|
||||
)
|
||||
if replay and entry_id and self._already_done(entry_id, n):
|
||||
complete(n)
|
||||
continue
|
||||
future = executor.submit(
|
||||
self._execute_node, n, state, entry_id, overrides, ready_inputs
|
||||
)
|
||||
node_futures[future] = n
|
||||
future.add_done_callback(finished.put)
|
||||
|
||||
def drain(executor: ThreadPoolExecutor) -> None:
|
||||
def drain(executor: Executor) -> None:
|
||||
submit_ready(executor)
|
||||
|
||||
while node_futures:
|
||||
done, _ = wait(node_futures.values(), return_when="FIRST_COMPLETED")
|
||||
completed = [n for n, f in node_futures.items() if f in done]
|
||||
# A completion queue also avoids scanning every outstanding
|
||||
# future whenever one node finishes.
|
||||
future = finished.get()
|
||||
n = node_futures.pop(future)
|
||||
result = future.result()
|
||||
if result is not None and fresh is not None:
|
||||
fresh.update(result)
|
||||
# Silent, failed and throttled producers still release their
|
||||
# consumers. Fresh inputs decide whether those consumers run.
|
||||
complete(n)
|
||||
submit_ready(executor)
|
||||
|
||||
for n in completed:
|
||||
result = node_futures.pop(n).result()
|
||||
# Before the decrement: a consumer freed by this node is
|
||||
# judged on what it just published.
|
||||
if result is not None and fresh is not None:
|
||||
fresh.update(result)
|
||||
# Decremented whatever it returned. A node that published
|
||||
# nothing — rate limited, unchanged, or failed — has not
|
||||
# *blocked* its consumers; it has merely given them nothing
|
||||
# to read, and `untouched` above is the accurate test of
|
||||
# that. Holding the count instead made a quiet producer
|
||||
# veto a noisy sibling: `dmx.switches` reads both boilers
|
||||
# through report-by-exception nodes, so the kitchen one
|
||||
# staying silent — which is almost always — kept the main
|
||||
# boiler's command from ever reaching the DMX universe.
|
||||
for consumer in edges[n]:
|
||||
if consumer in target_nodes:
|
||||
in_degree[consumer] -= 1
|
||||
submit_ready(executor)
|
||||
|
||||
if self._node_pool is not None:
|
||||
pool = node_pool if node_pool is not None else self._node_pool
|
||||
if pool is not None:
|
||||
# The execution service owns a long-lived pool; building one per
|
||||
# wave is what used to spawn threads without bound under load.
|
||||
drain(self._node_pool)
|
||||
drain(pool)
|
||||
else:
|
||||
with ThreadPoolExecutor(max_workers=self._max_workers) as executor:
|
||||
drain(executor)
|
||||
@@ -1519,6 +1501,7 @@ class Pipeline:
|
||||
replay: bool = False,
|
||||
changed: set[str] | None = None,
|
||||
overrides: dict[str, Any] | None = None,
|
||||
node_pool: Executor | None = None,
|
||||
) -> StateBackend:
|
||||
"""Run everything downstream of a node that has just published.
|
||||
|
||||
@@ -1552,6 +1535,7 @@ class Pipeline:
|
||||
replay=replay,
|
||||
changed=changed,
|
||||
overrides=overrides,
|
||||
node_pool=node_pool,
|
||||
)
|
||||
|
||||
def trigger(
|
||||
@@ -1796,6 +1780,12 @@ class Pipeline:
|
||||
# Losing the value outright would be worse than running it here.
|
||||
self._run_here(node, outputs, cause="external", delivered=kind == "emission")
|
||||
|
||||
def value(self, name: str) -> dict[str, Any]:
|
||||
"""One message and its timestamp, read together without listing state."""
|
||||
stamp = self._timestamp_key(name)
|
||||
found = self._state.get_multi([name, stamp])
|
||||
return {"value": found.get(name), "ts": found.get(stamp)}
|
||||
|
||||
def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]:
|
||||
"""Last value and timestamp of every message, optionally one flow's.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user