Stopping a live flow ends the calls it has running, on workers and in the local pool, and deleting one now ends its local calls too; what a call returns after its flow was stopped or deleted is dropped by a per-flow count the node compares, so a flow started again never takes an old call's answer. A live remote call's id carries a suffix, so two calls of one node no longer share it, and a call is registered before it queues for a slot, so ending its flow or run reaches it and it is never sent; a run's cancel goes through the call's own thread as well

This commit is contained in:
2026-11-11 12:03:43 +01:00
parent cd8831cfd3
commit 7132d26f36
8 changed files with 195 additions and 75 deletions
+23 -4
View File
@@ -699,7 +699,9 @@ class FlowController:
else:
self.disabled = self.disabled | {flow}
self.pipeline.set_disabled(self.disabled)
self.pipeline.end_running(flow)
await self._teardown(flow)
await self._end_calls(flow)
finally:
self._lock.release()
@@ -862,10 +864,7 @@ class FlowController:
if built is None:
if name:
pipeline.remove_flow(name)
# A call still out on a worker would keep its slot and its
# process there, for a result that is now dropped.
if self.remote is not None:
self.remote.cancel_flow(name)
await self._end_calls(name)
self.loaded = {
node_id: entry
for node_id, entry in self.loaded.items()
@@ -1142,6 +1141,26 @@ class FlowController:
else:
await self.supervisor.cancel_flow(flow)
async def _end_calls(self, flow: str) -> None:
"""End the calls a stopped or deleted flow has running, here and on
workers. A call left out there would keep its slot and its process for
a result that is dropped (`Pipeline.end_running`)."""
if self.remote is not None:
self.remote.cancel_flow(flow)
pool = self.workers
if pool is None:
return
nodes = [
node_id for node_id, entry in self.loaded.items() if entry.flow == flow
]
def cancel() -> None:
for node_id in nodes:
pool.cancel(node_id)
# Off the loop: a killed worker is waited for.
await run_in_threadpool(cancel)
async def _activate(self, flow: str | None = None) -> None:
"""Start subscriptions, schedules and webhooks of the new pipeline.
+7 -2
View File
@@ -464,11 +464,12 @@ class Node:
# A source node asked to inject nothing produces its own data.
if not outputs and not self.requires:
ended = self._pipeline.ended(self.flow)
collected = logs.Collector()
with logs.capture(collected):
outputs = self.f(params=self.params) or {}
if self._pipeline is None:
# Its flow was deleted while it ran (`Pipeline.remove_flow`).
if self._pipeline is None or self._pipeline.ended(self.flow) != ended:
# Its flow was stopped or deleted while it ran.
return None
self._pipeline.publish_log(self, collected, "")
@@ -490,7 +491,11 @@ class Node:
:returns: Node outputs if successful, or pipeline execution results if bound.
:rtype: dict | None
"""
ended = self._pipeline.ended(self.flow) if self._pipeline is not None else 0
outputs = self.execute(inputs)
if self._pipeline is None:
return outputs
if self._pipeline.ended(self.flow) != ended:
# Its flow was stopped while it ran.
return None
return self._pipeline.trigger(self, outputs, durable=durable)
+26 -5
View File
@@ -308,6 +308,7 @@ class Pipeline:
"emission_observer",
"run_cache",
"run_id",
"_ended",
)
def __init__(
@@ -373,6 +374,9 @@ class Pipeline:
# How deep to keep each message's series; a chart asking for more
# than the default puts its message in here. Swapped, never mutated.
self.history_limits: dict[str, int] = {}
# How often each flow had what it was running ended (`end_running`).
# Swapped, never mutated.
self._ended: dict[str, int] = {}
(
self.produces,
@@ -493,6 +497,22 @@ class Pipeline:
self.replace_flow(flow, [])
for node in gone:
node.bind(None)
self.end_running(flow)
def end_running(self, flow: str) -> None:
"""Drop what this flow's nodes return from what they are running now.
For a flow that was stopped or deleted. Started again, its nodes run
as before, and a call from before still answering is not taken for
theirs.
"""
with self._gate_lock:
self._ended = {**self._ended, flow: self._ended.get(flow, 0) + 1}
def ended(self, flow: str) -> int:
"""How often *flow* had what it was running ended: a node compares it
before and after it runs."""
return self._ended.get(flow, 0)
def _seed(self, initial_values: dict[str, Any] | None) -> None:
"""Give messages a starting value, without overwriting one already there.
@@ -1093,6 +1113,7 @@ class Pipeline:
"""
started = time.perf_counter()
collected = logs.Collector()
ended = self.ended(node.flow)
try:
# One MGET. The lock this used to be read under bought nothing a
# single bulk read does not, and it was the engine's one global
@@ -1127,8 +1148,8 @@ class Pipeline:
result = node.execute(inputs)
finally:
ARRIVED.reset(token)
if node._pipeline is not self:
# Its flow was deleted while it ran (`remove_flow`).
if self.ended(node.flow) != ended:
# Its flow was stopped or deleted while it ran (`end_running`).
return None
self.publish_log(node, collected, "")
@@ -1191,9 +1212,9 @@ class Pipeline:
)
return result or {}
except Exception as exc:
if node._pipeline is not self:
# Deleted while it ran, and most likely failing because the
# call was ended for that: nobody is left to tell.
if self.ended(node.flow) != ended:
# Stopped or deleted while it ran, and most likely failing
# because the call was ended for that.
return None
# One failing node must not take the rest of the graph down.
error = self.publish_error(node, exc, collected, entry_id)
+30 -20
View File
@@ -27,13 +27,19 @@ import queue
import sys
import threading
import time
import uuid
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from fluksio.flow.nodes.base import ARRIVED, NodeOutputError
from fluksio.flow.resources import ResourceAccountant
from fluksio.flow.workers import NodeTimeout, RemoteError, _remote_class
from fluksio.flow.workers import (
NodeCancelled,
NodeTimeout,
RemoteError,
_remote_class,
)
logger = logging.getLogger(__name__)
@@ -189,13 +195,19 @@ class RemoteWorker:
call_id = str(payload["call_id"])
if inbox is None:
inbox = queue.Queue()
# Blocking here is the backpressure, exactly as taking a slot is in the
# local pool.
if slot:
self._slots.acquire()
# Registered before it queues for a slot, so ending its flow or its run
# reaches a call still waiting for one.
with self._lock:
self._pending[call_id] = inbox
try:
# Blocking here is the backpressure, exactly as taking a slot is in
# the local pool.
if slot:
self._slots.acquire()
if not inbox.empty():
# Ended, or the worker gone, while it waited: nothing went out,
# so there is nothing on the worker to cancel.
raise NodeCancelled(f"ended before '{self.name}' was sent the call")
future = asyncio.run_coroutine_threadsafe(self._send(payload), self._loop)
future.result(timeout=SEND_TIMEOUT_S)
@@ -271,12 +283,9 @@ class RemoteWorker:
with self._lock:
return len(self._pending)
def calls_of(self, run_id: str) -> list[str]:
with self._lock:
return [call for call in self._pending if call.startswith(f"{run_id}:")]
def end_calls(self, prefix: str) -> None:
"""End every call whose id starts with *prefix*, each once it is sent."""
def end_calls(self, prefix: str) -> int:
"""End every call whose id starts with *prefix*, each once it is sent,
and one still waiting for a slot before it is."""
with self._lock:
inboxes = [
inbox
@@ -285,6 +294,7 @@ class RemoteWorker:
]
for inbox in inboxes:
inbox.put(self.END)
return len(inboxes)
def cancel(self, call_id: str) -> None:
"""Ask the worker to kill what it is running for this call."""
@@ -423,7 +433,11 @@ class RemoteWorkerHub:
"""
payload = {
"op": "run",
"call_id": f"{run_id}:{node_id}" if run_id else node_id,
# A live node can run in two cascades at once, so its calls are
# told apart; a run runs each node once.
"call_id": (
f"{run_id}:{node_id}" if run_id else f"{node_id}:{uuid.uuid4().hex[:8]}"
),
"flow": flow,
"node": node,
"source": source,
@@ -529,18 +543,14 @@ class RemoteWorkerHub:
def cancel_run(self, run_id: str) -> int:
"""Kill whatever this run has executing on any attached worker."""
stopped = 0
for worker in self.workers():
for call in worker.calls_of(run_id):
worker.cancel(call)
stopped += 1
return stopped
return sum(worker.end_calls(f"{run_id}:") for worker in self.workers())
def cancel_flow(self, flow: str) -> None:
"""End what a live flow has running on any attached worker.
A live call goes by its node's id, ``flow.node``. A run's carries the
run's id first and a compile its op, so neither is touched.
A live call goes by its node's id and a suffix, ``flow.node:…``. A
run's carries the run's id first and a compile its op, so neither is
touched.
"""
for worker in self.workers():
worker.end_calls(f"{flow}.")