Stopping a live flow ends the calls it has running, on workers and in the local pool, and deleting one now ends its local calls too; what a call returns after its flow was stopped or deleted is dropped by a per-flow count the node compares, so a flow started again never takes an old call's answer. A live remote call's id carries a suffix, so two calls of one node no longer share it, and a call is registered before it queues for a slot, so ending its flow or run reaches it and it is never sent; a run's cancel goes through the call's own thread as well
This commit is contained in:
@@ -308,6 +308,7 @@ class Pipeline:
|
||||
"emission_observer",
|
||||
"run_cache",
|
||||
"run_id",
|
||||
"_ended",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -373,6 +374,9 @@ class Pipeline:
|
||||
# How deep to keep each message's series; a chart asking for more
|
||||
# than the default puts its message in here. Swapped, never mutated.
|
||||
self.history_limits: dict[str, int] = {}
|
||||
# How often each flow had what it was running ended (`end_running`).
|
||||
# Swapped, never mutated.
|
||||
self._ended: dict[str, int] = {}
|
||||
|
||||
(
|
||||
self.produces,
|
||||
@@ -493,6 +497,22 @@ class Pipeline:
|
||||
self.replace_flow(flow, [])
|
||||
for node in gone:
|
||||
node.bind(None)
|
||||
self.end_running(flow)
|
||||
|
||||
def end_running(self, flow: str) -> None:
|
||||
"""Drop what this flow's nodes return from what they are running now.
|
||||
|
||||
For a flow that was stopped or deleted. Started again, its nodes run
|
||||
as before, and a call from before still answering is not taken for
|
||||
theirs.
|
||||
"""
|
||||
with self._gate_lock:
|
||||
self._ended = {**self._ended, flow: self._ended.get(flow, 0) + 1}
|
||||
|
||||
def ended(self, flow: str) -> int:
|
||||
"""How often *flow* had what it was running ended: a node compares it
|
||||
before and after it runs."""
|
||||
return self._ended.get(flow, 0)
|
||||
|
||||
def _seed(self, initial_values: dict[str, Any] | None) -> None:
|
||||
"""Give messages a starting value, without overwriting one already there.
|
||||
@@ -1093,6 +1113,7 @@ class Pipeline:
|
||||
"""
|
||||
started = time.perf_counter()
|
||||
collected = logs.Collector()
|
||||
ended = self.ended(node.flow)
|
||||
try:
|
||||
# One MGET. The lock this used to be read under bought nothing a
|
||||
# single bulk read does not, and it was the engine's one global
|
||||
@@ -1127,8 +1148,8 @@ class Pipeline:
|
||||
result = node.execute(inputs)
|
||||
finally:
|
||||
ARRIVED.reset(token)
|
||||
if node._pipeline is not self:
|
||||
# Its flow was deleted while it ran (`remove_flow`).
|
||||
if self.ended(node.flow) != ended:
|
||||
# Its flow was stopped or deleted while it ran (`end_running`).
|
||||
return None
|
||||
self.publish_log(node, collected, "")
|
||||
|
||||
@@ -1191,9 +1212,9 @@ class Pipeline:
|
||||
)
|
||||
return result or {}
|
||||
except Exception as exc:
|
||||
if node._pipeline is not self:
|
||||
# Deleted while it ran, and most likely failing because the
|
||||
# call was ended for that: nobody is left to tell.
|
||||
if self.ended(node.flow) != ended:
|
||||
# Stopped or deleted while it ran, and most likely failing
|
||||
# because the call was ended for that.
|
||||
return None
|
||||
# One failing node must not take the rest of the graph down.
|
||||
error = self.publish_error(node, exc, collected, entry_id)
|
||||
|
||||
Reference in New Issue
Block a user