Stopping a live flow ends the calls it has running, on workers and in the local pool, and deleting one now ends its local calls too; what a call returns after its flow was stopped or deleted is dropped by a per-flow count the node compares, so a flow started again never takes an old call's answer. A live remote call's id carries a suffix, so two calls of one node no longer share it, and a call is registered before it queues for a slot, so ending its flow or run reaches it and it is never sent; a run's cancel goes through the call's own thread as well

This commit is contained in:
2026-11-11 12:03:43 +01:00
parent cd8831cfd3
commit 7132d26f36
8 changed files with 195 additions and 75 deletions
+26 -5
View File
@@ -308,6 +308,7 @@ class Pipeline:
"emission_observer",
"run_cache",
"run_id",
"_ended",
)
def __init__(
@@ -373,6 +374,9 @@ class Pipeline:
# How deep to keep each message's series; a chart asking for more
# than the default puts its message in here. Swapped, never mutated.
self.history_limits: dict[str, int] = {}
# How often each flow had what it was running ended (`end_running`).
# Swapped, never mutated.
self._ended: dict[str, int] = {}
(
self.produces,
@@ -493,6 +497,22 @@ class Pipeline:
self.replace_flow(flow, [])
for node in gone:
node.bind(None)
self.end_running(flow)
def end_running(self, flow: str) -> None:
"""Drop what this flow's nodes return from what they are running now.
For a flow that was stopped or deleted. Started again, its nodes run
as before, and a call from before still answering is not taken for
theirs.
"""
with self._gate_lock:
self._ended = {**self._ended, flow: self._ended.get(flow, 0) + 1}
def ended(self, flow: str) -> int:
"""How often *flow* had what it was running ended: a node compares it
before and after it runs."""
return self._ended.get(flow, 0)
def _seed(self, initial_values: dict[str, Any] | None) -> None:
"""Give messages a starting value, without overwriting one already there.
@@ -1093,6 +1113,7 @@ class Pipeline:
"""
started = time.perf_counter()
collected = logs.Collector()
ended = self.ended(node.flow)
try:
# One MGET. The lock this used to be read under bought nothing a
# single bulk read does not, and it was the engine's one global
@@ -1127,8 +1148,8 @@ class Pipeline:
result = node.execute(inputs)
finally:
ARRIVED.reset(token)
if node._pipeline is not self:
# Its flow was deleted while it ran (`remove_flow`).
if self.ended(node.flow) != ended:
# Its flow was stopped or deleted while it ran (`end_running`).
return None
self.publish_log(node, collected, "")
@@ -1191,9 +1212,9 @@ class Pipeline:
)
return result or {}
except Exception as exc:
if node._pipeline is not self:
# Deleted while it ran, and most likely failing because the
# call was ended for that: nobody is left to tell.
if self.ended(node.flow) != ended:
# Stopped or deleted while it ran, and most likely failing
# because the call was ended for that.
return None
# One failing node must not take the rest of the graph down.
error = self.publish_error(node, exc, collected, entry_id)