Engine holds for a rename: the pipeline refuses publish and inline cascades with EngineHeld once those running have finished, the execution service stops claiming and its timers once claimed cascades have finished; both wait without a limit, report what is busy and can be aborted. REST answers 503 with Retry-After, a device frame busy. Redis adds no longer wait for each other, only a queue hold waits for them

This commit is contained in:
2026-11-02 20:21:17 +01:00
parent 46b57de4ec
commit badcf94491
10 changed files with 687 additions and 156 deletions
+136 -65
View File
@@ -19,7 +19,7 @@ import logging
import threading
import time
import uuid
from collections import deque
from collections import Counter, deque
from collections.abc import Callable, Iterator
from concurrent.futures import Executor, Future, ThreadPoolExecutor
from contextlib import contextmanager
@@ -33,6 +33,7 @@ from pydantic import BaseModel, model_validator
from fluksio.flow import logs
from fluksio.flow.artifacts import is_reference
from fluksio.flow.events import EventBus
from fluksio.flow.executor import wait_out
from fluksio.flow.messages import flow_of, requalify
from fluksio.flow.namespace import (
DELIVERED,
@@ -265,6 +266,14 @@ def _derive(
return produces, dependencies, consumes, by_id
class EngineHeld(RuntimeError):
"""A value arrived while a rename holds the engine.
Answered as a 503 with ``Retry-After``: nothing is wrong with the value,
and the same publish works once the rename is done.
"""
class Pipeline:
"""Directed graph of nodes with automatic dependency resolution."""
@@ -286,6 +295,9 @@ class Pipeline:
"_gate_lock",
"_gate_revision",
"_graph_lock",
"_holding",
"_inline",
"_inline_lock",
"_queue",
"_node_pool",
"history_limits",
@@ -324,6 +336,12 @@ class Pipeline:
# `_gate_lock` or `state.lock()` in either order, which is what keeps
# three locks from needing an ordering rule.
self._graph_lock = threading.RLock()
# Set while a rename holds the engine (`hold`). What runs outside the
# queue, by node id or message name, is counted so a hold can wait
# it out.
self._holding = False
self._inline: Counter[str] = Counter()
self._inline_lock = threading.Condition()
# An empty state backend is falsy, so this cannot be ``state or ...``:
# that would quietly hand the pipeline a second, private state and
# leave everyone reading the shared one seeing nothing.
@@ -1325,6 +1343,49 @@ class Pipeline:
with self._gate_lock:
return flow in self._stepping
def hold(
self,
abort: threading.Event,
report: Callable[[list[str]], None] | None = None,
) -> bool:
"""Refuse `publish` and every cascade outside the queue, once those
already running have finished. For a rename, which nothing may write
past.
Waits without a limit and tells ``report`` the nodes and messages
still running inline each time they change. Setting ``abort`` gives
up and lets them through again: False. True means held until
`release`.
"""
with self._inline_lock:
self._holding = True
if wait_out(self._inline_lock, lambda: sorted(self._inline), abort, report):
return True
self.release()
return False
def release(self) -> None:
"""End a `hold`."""
with self._inline_lock:
self._holding = False
@contextmanager
def _inline_run(self, names: list[str]) -> Iterator[None]:
"""Count a run outside the queue, which a hold waits out.
:raises EngineHeld: While held.
"""
with self._inline_lock:
if self._holding:
raise EngineHeld("A rename holds the engine; try again shortly")
self._inline.update(names)
try:
yield
finally:
with self._inline_lock:
self._inline -= Counter(names)
self._inline_lock.notify_all()
def _execute_parallel(
self,
nodes_subset: set[Node] | None,
@@ -1625,44 +1686,45 @@ class Pipeline:
already in state, so they are handed to their readers rather than
written again.
"""
run_id = f"{MANUAL_RUN_PREFIX}{uuid.uuid4().hex[:12]}"
self._publish(
{
"type": "cascade_started",
"run": run_id,
"flow": node.flow,
"node": node.id,
"cause": cause,
"deliveries": 1,
"ts": time.time(),
}
)
try:
published = (
set(outputs or {})
if delivered
else self.apply_outputs(node, outputs, run_id)
)
state = self.run_downstream(
node,
entry_id=run_id,
# No payload means the value is already in state and this is a
# wake-up, which has nothing to name as changed.
changed=published if outputs else None,
overrides=outputs if delivered else None,
)
finally:
# Paired, or a cascade that raised leaves the run open until the
# abandoned sweep ten minutes later.
with self._inline_run([node.id]):
run_id = f"{MANUAL_RUN_PREFIX}{uuid.uuid4().hex[:12]}"
self._publish(
{
"type": "cascade_finished",
"type": "cascade_started",
"run": run_id,
"flow": node.flow,
"node": node.id,
"cause": cause,
"deliveries": 1,
"ts": time.time(),
}
)
return state
try:
published = (
set(outputs or {})
if delivered
else self.apply_outputs(node, outputs, run_id)
)
state = self.run_downstream(
node,
entry_id=run_id,
# No payload means the value is already in state and this is a
# wake-up, which has nothing to name as changed.
changed=published if outputs else None,
overrides=outputs if delivered else None,
)
finally:
# Paired, or a cascade that raised leaves the run open until the
# abandoned sweep ten minutes later.
self._publish(
{
"type": "cascade_finished",
"run": run_id,
"flow": node.flow,
"ts": time.time(),
}
)
return state
def publish(
self, values: dict[str, Any], source: ValueSource | None = None
@@ -1676,45 +1738,48 @@ class Pipeline:
``source`` says what did, so the canvas can show the value arriving
from outside instead of blaming whichever node happens to be drawn as
a producer of that message.
:raises EngineHeld: While a rename holds the engine.
"""
if not values:
return
origin = source or ValueSource(kind="api", label="API")
ts = time.time()
self._state.record(
values,
{self._timestamp_key(name): ts for name in values},
[self._version_key(name) for name in values],
ts,
self.history_limits,
)
source_dump = origin.model_dump()
for name, value in values.items():
self._publish(
{
"type": "message_value",
"flow": flow_of(name),
"name": name,
"value": value,
"ts": ts,
"source": source_dump,
}
with self._inline_run(list(values)):
ts = time.time()
self._state.record(
values,
{self._timestamp_key(name): ts for name in values},
[self._version_key(name) for name in values],
ts,
self.history_limits,
)
source_dump = origin.model_dump()
for name, value in values.items():
self._publish(
{
"type": "message_value",
"flow": flow_of(name),
"name": name,
"value": value,
"ts": ts,
"source": source_dump,
}
)
targets: set[Node] = set()
for name in values:
for consumer in self.consumes.get(name, ()):
targets.add(consumer)
targets.update(self._get_downstream(consumer))
if targets:
# Only what these values reach, as in a cascade: a node further
# down whose producer published nothing would otherwise run again
# on what it already read, once per microphone chunk.
self._execute_parallel(
targets, self._state, check_ready=True, changed=set(values)
)
targets: set[Node] = set()
for name in values:
for consumer in self.consumes.get(name, ()):
targets.add(consumer)
targets.update(self._get_downstream(consumer))
if targets:
# Only what these values reach, as in a cascade: a node further
# down whose producer published nothing would otherwise run again
# on what it already read, once per microphone chunk.
self._execute_parallel(
targets, self._state, check_ready=True, changed=set(values)
)
def defer(
self,
@@ -1785,8 +1850,14 @@ class Pipeline:
"ts": time.time(),
}
)
# Losing the value outright would be worse than running it here.
self._run_here(node, outputs, cause="external", delivered=kind == "emission")
# Losing the value outright would be worse than running it here,
# unless a rename holds the engine, which nothing runs past.
try:
self._run_here(
node, outputs, cause="external", delivered=kind == "emission"
)
except EngineHeld:
logger.warning("Dropped work for '%s' while the engine is held", node.id)
def value(self, name: str) -> dict[str, Any]:
"""One message and its timestamp, read together without listing state."""