Engine holds for a rename: the pipeline refuses publish and inline cascades with EngineHeld once those running have finished, the execution service stops claiming and its timers once claimed cascades have finished; both wait without a limit, report what is busy and can be aborted. REST answers 503 with Retry-After, a device frame busy. Redis adds no longer wait for each other, only a queue hold waits for them
This commit is contained in:
@@ -19,7 +19,7 @@ import logging
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from collections import Counter, deque
|
||||
from collections.abc import Callable, Iterator
|
||||
from concurrent.futures import Executor, Future, ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
@@ -33,6 +33,7 @@ from pydantic import BaseModel, model_validator
|
||||
from fluksio.flow import logs
|
||||
from fluksio.flow.artifacts import is_reference
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.executor import wait_out
|
||||
from fluksio.flow.messages import flow_of, requalify
|
||||
from fluksio.flow.namespace import (
|
||||
DELIVERED,
|
||||
@@ -265,6 +266,14 @@ def _derive(
|
||||
return produces, dependencies, consumes, by_id
|
||||
|
||||
|
||||
class EngineHeld(RuntimeError):
|
||||
"""A value arrived while a rename holds the engine.
|
||||
|
||||
Answered as a 503 with ``Retry-After``: nothing is wrong with the value,
|
||||
and the same publish works once the rename is done.
|
||||
"""
|
||||
|
||||
|
||||
class Pipeline:
|
||||
"""Directed graph of nodes with automatic dependency resolution."""
|
||||
|
||||
@@ -286,6 +295,9 @@ class Pipeline:
|
||||
"_gate_lock",
|
||||
"_gate_revision",
|
||||
"_graph_lock",
|
||||
"_holding",
|
||||
"_inline",
|
||||
"_inline_lock",
|
||||
"_queue",
|
||||
"_node_pool",
|
||||
"history_limits",
|
||||
@@ -324,6 +336,12 @@ class Pipeline:
|
||||
# `_gate_lock` or `state.lock()` in either order, which is what keeps
|
||||
# three locks from needing an ordering rule.
|
||||
self._graph_lock = threading.RLock()
|
||||
# Set while a rename holds the engine (`hold`). What runs outside the
|
||||
# queue, by node id or message name, is counted so a hold can wait
|
||||
# it out.
|
||||
self._holding = False
|
||||
self._inline: Counter[str] = Counter()
|
||||
self._inline_lock = threading.Condition()
|
||||
# An empty state backend is falsy, so this cannot be ``state or ...``:
|
||||
# that would quietly hand the pipeline a second, private state and
|
||||
# leave everyone reading the shared one seeing nothing.
|
||||
@@ -1325,6 +1343,49 @@ class Pipeline:
|
||||
with self._gate_lock:
|
||||
return flow in self._stepping
|
||||
|
||||
def hold(
|
||||
self,
|
||||
abort: threading.Event,
|
||||
report: Callable[[list[str]], None] | None = None,
|
||||
) -> bool:
|
||||
"""Refuse `publish` and every cascade outside the queue, once those
|
||||
already running have finished. For a rename, which nothing may write
|
||||
past.
|
||||
|
||||
Waits without a limit and tells ``report`` the nodes and messages
|
||||
still running inline each time they change. Setting ``abort`` gives
|
||||
up and lets them through again: False. True means held until
|
||||
`release`.
|
||||
"""
|
||||
with self._inline_lock:
|
||||
self._holding = True
|
||||
if wait_out(self._inline_lock, lambda: sorted(self._inline), abort, report):
|
||||
return True
|
||||
self.release()
|
||||
return False
|
||||
|
||||
def release(self) -> None:
|
||||
"""End a `hold`."""
|
||||
with self._inline_lock:
|
||||
self._holding = False
|
||||
|
||||
@contextmanager
|
||||
def _inline_run(self, names: list[str]) -> Iterator[None]:
|
||||
"""Count a run outside the queue, which a hold waits out.
|
||||
|
||||
:raises EngineHeld: While held.
|
||||
"""
|
||||
with self._inline_lock:
|
||||
if self._holding:
|
||||
raise EngineHeld("A rename holds the engine; try again shortly")
|
||||
self._inline.update(names)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with self._inline_lock:
|
||||
self._inline -= Counter(names)
|
||||
self._inline_lock.notify_all()
|
||||
|
||||
def _execute_parallel(
|
||||
self,
|
||||
nodes_subset: set[Node] | None,
|
||||
@@ -1625,44 +1686,45 @@ class Pipeline:
|
||||
already in state, so they are handed to their readers rather than
|
||||
written again.
|
||||
"""
|
||||
run_id = f"{MANUAL_RUN_PREFIX}{uuid.uuid4().hex[:12]}"
|
||||
self._publish(
|
||||
{
|
||||
"type": "cascade_started",
|
||||
"run": run_id,
|
||||
"flow": node.flow,
|
||||
"node": node.id,
|
||||
"cause": cause,
|
||||
"deliveries": 1,
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
try:
|
||||
published = (
|
||||
set(outputs or {})
|
||||
if delivered
|
||||
else self.apply_outputs(node, outputs, run_id)
|
||||
)
|
||||
state = self.run_downstream(
|
||||
node,
|
||||
entry_id=run_id,
|
||||
# No payload means the value is already in state and this is a
|
||||
# wake-up, which has nothing to name as changed.
|
||||
changed=published if outputs else None,
|
||||
overrides=outputs if delivered else None,
|
||||
)
|
||||
finally:
|
||||
# Paired, or a cascade that raised leaves the run open until the
|
||||
# abandoned sweep ten minutes later.
|
||||
with self._inline_run([node.id]):
|
||||
run_id = f"{MANUAL_RUN_PREFIX}{uuid.uuid4().hex[:12]}"
|
||||
self._publish(
|
||||
{
|
||||
"type": "cascade_finished",
|
||||
"type": "cascade_started",
|
||||
"run": run_id,
|
||||
"flow": node.flow,
|
||||
"node": node.id,
|
||||
"cause": cause,
|
||||
"deliveries": 1,
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
return state
|
||||
try:
|
||||
published = (
|
||||
set(outputs or {})
|
||||
if delivered
|
||||
else self.apply_outputs(node, outputs, run_id)
|
||||
)
|
||||
state = self.run_downstream(
|
||||
node,
|
||||
entry_id=run_id,
|
||||
# No payload means the value is already in state and this is a
|
||||
# wake-up, which has nothing to name as changed.
|
||||
changed=published if outputs else None,
|
||||
overrides=outputs if delivered else None,
|
||||
)
|
||||
finally:
|
||||
# Paired, or a cascade that raised leaves the run open until the
|
||||
# abandoned sweep ten minutes later.
|
||||
self._publish(
|
||||
{
|
||||
"type": "cascade_finished",
|
||||
"run": run_id,
|
||||
"flow": node.flow,
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
return state
|
||||
|
||||
def publish(
|
||||
self, values: dict[str, Any], source: ValueSource | None = None
|
||||
@@ -1676,45 +1738,48 @@ class Pipeline:
|
||||
``source`` says what did, so the canvas can show the value arriving
|
||||
from outside instead of blaming whichever node happens to be drawn as
|
||||
a producer of that message.
|
||||
|
||||
:raises EngineHeld: While a rename holds the engine.
|
||||
"""
|
||||
if not values:
|
||||
return
|
||||
|
||||
origin = source or ValueSource(kind="api", label="API")
|
||||
|
||||
ts = time.time()
|
||||
self._state.record(
|
||||
values,
|
||||
{self._timestamp_key(name): ts for name in values},
|
||||
[self._version_key(name) for name in values],
|
||||
ts,
|
||||
self.history_limits,
|
||||
)
|
||||
source_dump = origin.model_dump()
|
||||
for name, value in values.items():
|
||||
self._publish(
|
||||
{
|
||||
"type": "message_value",
|
||||
"flow": flow_of(name),
|
||||
"name": name,
|
||||
"value": value,
|
||||
"ts": ts,
|
||||
"source": source_dump,
|
||||
}
|
||||
with self._inline_run(list(values)):
|
||||
ts = time.time()
|
||||
self._state.record(
|
||||
values,
|
||||
{self._timestamp_key(name): ts for name in values},
|
||||
[self._version_key(name) for name in values],
|
||||
ts,
|
||||
self.history_limits,
|
||||
)
|
||||
source_dump = origin.model_dump()
|
||||
for name, value in values.items():
|
||||
self._publish(
|
||||
{
|
||||
"type": "message_value",
|
||||
"flow": flow_of(name),
|
||||
"name": name,
|
||||
"value": value,
|
||||
"ts": ts,
|
||||
"source": source_dump,
|
||||
}
|
||||
)
|
||||
|
||||
targets: set[Node] = set()
|
||||
for name in values:
|
||||
for consumer in self.consumes.get(name, ()):
|
||||
targets.add(consumer)
|
||||
targets.update(self._get_downstream(consumer))
|
||||
if targets:
|
||||
# Only what these values reach, as in a cascade: a node further
|
||||
# down whose producer published nothing would otherwise run again
|
||||
# on what it already read, once per microphone chunk.
|
||||
self._execute_parallel(
|
||||
targets, self._state, check_ready=True, changed=set(values)
|
||||
)
|
||||
targets: set[Node] = set()
|
||||
for name in values:
|
||||
for consumer in self.consumes.get(name, ()):
|
||||
targets.add(consumer)
|
||||
targets.update(self._get_downstream(consumer))
|
||||
if targets:
|
||||
# Only what these values reach, as in a cascade: a node further
|
||||
# down whose producer published nothing would otherwise run again
|
||||
# on what it already read, once per microphone chunk.
|
||||
self._execute_parallel(
|
||||
targets, self._state, check_ready=True, changed=set(values)
|
||||
)
|
||||
|
||||
def defer(
|
||||
self,
|
||||
@@ -1785,8 +1850,14 @@ class Pipeline:
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
# Losing the value outright would be worse than running it here.
|
||||
self._run_here(node, outputs, cause="external", delivered=kind == "emission")
|
||||
# Losing the value outright would be worse than running it here,
|
||||
# unless a rename holds the engine, which nothing runs past.
|
||||
try:
|
||||
self._run_here(
|
||||
node, outputs, cause="external", delivered=kind == "emission"
|
||||
)
|
||||
except EngineHeld:
|
||||
logger.warning("Dropped work for '%s' while the engine is held", node.id)
|
||||
|
||||
def value(self, name: str) -> dict[str, Any]:
|
||||
"""One message and its timestamp, read together without listing state."""
|
||||
|
||||
Reference in New Issue
Block a user