Stopping a live flow ends the calls it has running, on workers and in the local pool, and deleting one now ends its local calls too; what a call returns after its flow was stopped or deleted is dropped by a per-flow count the node compares, so a flow started again never takes an old call's answer. A live remote call's id carries a suffix, so two calls of one node no longer share it, and a call is registered before it queues for a slot, so ending its flow or run reaches it and it is never sent; a run's cancel goes through the call's own thread as well
This commit is contained in:
@@ -699,7 +699,9 @@ class FlowController:
|
||||
else:
|
||||
self.disabled = self.disabled | {flow}
|
||||
self.pipeline.set_disabled(self.disabled)
|
||||
self.pipeline.end_running(flow)
|
||||
await self._teardown(flow)
|
||||
await self._end_calls(flow)
|
||||
finally:
|
||||
self._lock.release()
|
||||
|
||||
@@ -862,10 +864,7 @@ class FlowController:
|
||||
if built is None:
|
||||
if name:
|
||||
pipeline.remove_flow(name)
|
||||
# A call still out on a worker would keep its slot and its
|
||||
# process there, for a result that is now dropped.
|
||||
if self.remote is not None:
|
||||
self.remote.cancel_flow(name)
|
||||
await self._end_calls(name)
|
||||
self.loaded = {
|
||||
node_id: entry
|
||||
for node_id, entry in self.loaded.items()
|
||||
@@ -1142,6 +1141,26 @@ class FlowController:
|
||||
else:
|
||||
await self.supervisor.cancel_flow(flow)
|
||||
|
||||
async def _end_calls(self, flow: str) -> None:
|
||||
"""End the calls a stopped or deleted flow has running, here and on
|
||||
workers. A call left out there would keep its slot and its process for
|
||||
a result that is dropped (`Pipeline.end_running`)."""
|
||||
if self.remote is not None:
|
||||
self.remote.cancel_flow(flow)
|
||||
pool = self.workers
|
||||
if pool is None:
|
||||
return
|
||||
nodes = [
|
||||
node_id for node_id, entry in self.loaded.items() if entry.flow == flow
|
||||
]
|
||||
|
||||
def cancel() -> None:
|
||||
for node_id in nodes:
|
||||
pool.cancel(node_id)
|
||||
|
||||
# Off the loop: a killed worker is waited for.
|
||||
await run_in_threadpool(cancel)
|
||||
|
||||
async def _activate(self, flow: str | None = None) -> None:
|
||||
"""Start subscriptions, schedules and webhooks of the new pipeline.
|
||||
|
||||
|
||||
@@ -464,11 +464,12 @@ class Node:
|
||||
|
||||
# A source node asked to inject nothing produces its own data.
|
||||
if not outputs and not self.requires:
|
||||
ended = self._pipeline.ended(self.flow)
|
||||
collected = logs.Collector()
|
||||
with logs.capture(collected):
|
||||
outputs = self.f(params=self.params) or {}
|
||||
if self._pipeline is None:
|
||||
# Its flow was deleted while it ran (`Pipeline.remove_flow`).
|
||||
if self._pipeline is None or self._pipeline.ended(self.flow) != ended:
|
||||
# Its flow was stopped or deleted while it ran.
|
||||
return None
|
||||
self._pipeline.publish_log(self, collected, "")
|
||||
|
||||
@@ -490,7 +491,11 @@ class Node:
|
||||
:returns: Node outputs if successful, or pipeline execution results if bound.
|
||||
:rtype: dict | None
|
||||
"""
|
||||
ended = self._pipeline.ended(self.flow) if self._pipeline is not None else 0
|
||||
outputs = self.execute(inputs)
|
||||
if self._pipeline is None:
|
||||
return outputs
|
||||
if self._pipeline.ended(self.flow) != ended:
|
||||
# Its flow was stopped while it ran.
|
||||
return None
|
||||
return self._pipeline.trigger(self, outputs, durable=durable)
|
||||
|
||||
@@ -308,6 +308,7 @@ class Pipeline:
|
||||
"emission_observer",
|
||||
"run_cache",
|
||||
"run_id",
|
||||
"_ended",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -373,6 +374,9 @@ class Pipeline:
|
||||
# How deep to keep each message's series; a chart asking for more
|
||||
# than the default puts its message in here. Swapped, never mutated.
|
||||
self.history_limits: dict[str, int] = {}
|
||||
# How often each flow had what it was running ended (`end_running`).
|
||||
# Swapped, never mutated.
|
||||
self._ended: dict[str, int] = {}
|
||||
|
||||
(
|
||||
self.produces,
|
||||
@@ -493,6 +497,22 @@ class Pipeline:
|
||||
self.replace_flow(flow, [])
|
||||
for node in gone:
|
||||
node.bind(None)
|
||||
self.end_running(flow)
|
||||
|
||||
def end_running(self, flow: str) -> None:
|
||||
"""Drop what this flow's nodes return from what they are running now.
|
||||
|
||||
For a flow that was stopped or deleted. Started again, its nodes run
|
||||
as before, and a call from before still answering is not taken for
|
||||
theirs.
|
||||
"""
|
||||
with self._gate_lock:
|
||||
self._ended = {**self._ended, flow: self._ended.get(flow, 0) + 1}
|
||||
|
||||
def ended(self, flow: str) -> int:
|
||||
"""How often *flow* had what it was running ended: a node compares it
|
||||
before and after it runs."""
|
||||
return self._ended.get(flow, 0)
|
||||
|
||||
def _seed(self, initial_values: dict[str, Any] | None) -> None:
|
||||
"""Give messages a starting value, without overwriting one already there.
|
||||
@@ -1093,6 +1113,7 @@ class Pipeline:
|
||||
"""
|
||||
started = time.perf_counter()
|
||||
collected = logs.Collector()
|
||||
ended = self.ended(node.flow)
|
||||
try:
|
||||
# One MGET. The lock this used to be read under bought nothing a
|
||||
# single bulk read does not, and it was the engine's one global
|
||||
@@ -1127,8 +1148,8 @@ class Pipeline:
|
||||
result = node.execute(inputs)
|
||||
finally:
|
||||
ARRIVED.reset(token)
|
||||
if node._pipeline is not self:
|
||||
# Its flow was deleted while it ran (`remove_flow`).
|
||||
if self.ended(node.flow) != ended:
|
||||
# Its flow was stopped or deleted while it ran (`end_running`).
|
||||
return None
|
||||
self.publish_log(node, collected, "")
|
||||
|
||||
@@ -1191,9 +1212,9 @@ class Pipeline:
|
||||
)
|
||||
return result or {}
|
||||
except Exception as exc:
|
||||
if node._pipeline is not self:
|
||||
# Deleted while it ran, and most likely failing because the
|
||||
# call was ended for that: nobody is left to tell.
|
||||
if self.ended(node.flow) != ended:
|
||||
# Stopped or deleted while it ran, and most likely failing
|
||||
# because the call was ended for that.
|
||||
return None
|
||||
# One failing node must not take the rest of the graph down.
|
||||
error = self.publish_error(node, exc, collected, entry_id)
|
||||
|
||||
@@ -27,13 +27,19 @@ import queue
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from fluksio.flow.nodes.base import ARRIVED, NodeOutputError
|
||||
from fluksio.flow.resources import ResourceAccountant
|
||||
from fluksio.flow.workers import NodeTimeout, RemoteError, _remote_class
|
||||
from fluksio.flow.workers import (
|
||||
NodeCancelled,
|
||||
NodeTimeout,
|
||||
RemoteError,
|
||||
_remote_class,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -189,13 +195,19 @@ class RemoteWorker:
|
||||
call_id = str(payload["call_id"])
|
||||
if inbox is None:
|
||||
inbox = queue.Queue()
|
||||
# Blocking here is the backpressure, exactly as taking a slot is in the
|
||||
# local pool.
|
||||
if slot:
|
||||
self._slots.acquire()
|
||||
# Registered before it queues for a slot, so ending its flow or its run
|
||||
# reaches a call still waiting for one.
|
||||
with self._lock:
|
||||
self._pending[call_id] = inbox
|
||||
try:
|
||||
# Blocking here is the backpressure, exactly as taking a slot is in
|
||||
# the local pool.
|
||||
if slot:
|
||||
self._slots.acquire()
|
||||
if not inbox.empty():
|
||||
# Ended, or the worker gone, while it waited: nothing went out,
|
||||
# so there is nothing on the worker to cancel.
|
||||
raise NodeCancelled(f"ended before '{self.name}' was sent the call")
|
||||
future = asyncio.run_coroutine_threadsafe(self._send(payload), self._loop)
|
||||
future.result(timeout=SEND_TIMEOUT_S)
|
||||
|
||||
@@ -271,12 +283,9 @@ class RemoteWorker:
|
||||
with self._lock:
|
||||
return len(self._pending)
|
||||
|
||||
def calls_of(self, run_id: str) -> list[str]:
|
||||
with self._lock:
|
||||
return [call for call in self._pending if call.startswith(f"{run_id}:")]
|
||||
|
||||
def end_calls(self, prefix: str) -> None:
|
||||
"""End every call whose id starts with *prefix*, each once it is sent."""
|
||||
def end_calls(self, prefix: str) -> int:
|
||||
"""End every call whose id starts with *prefix*, each once it is sent,
|
||||
and one still waiting for a slot before it is."""
|
||||
with self._lock:
|
||||
inboxes = [
|
||||
inbox
|
||||
@@ -285,6 +294,7 @@ class RemoteWorker:
|
||||
]
|
||||
for inbox in inboxes:
|
||||
inbox.put(self.END)
|
||||
return len(inboxes)
|
||||
|
||||
def cancel(self, call_id: str) -> None:
|
||||
"""Ask the worker to kill what it is running for this call."""
|
||||
@@ -423,7 +433,11 @@ class RemoteWorkerHub:
|
||||
"""
|
||||
payload = {
|
||||
"op": "run",
|
||||
"call_id": f"{run_id}:{node_id}" if run_id else node_id,
|
||||
# A live node can run in two cascades at once, so its calls are
|
||||
# told apart; a run runs each node once.
|
||||
"call_id": (
|
||||
f"{run_id}:{node_id}" if run_id else f"{node_id}:{uuid.uuid4().hex[:8]}"
|
||||
),
|
||||
"flow": flow,
|
||||
"node": node,
|
||||
"source": source,
|
||||
@@ -529,18 +543,14 @@ class RemoteWorkerHub:
|
||||
|
||||
def cancel_run(self, run_id: str) -> int:
|
||||
"""Kill whatever this run has executing on any attached worker."""
|
||||
stopped = 0
|
||||
for worker in self.workers():
|
||||
for call in worker.calls_of(run_id):
|
||||
worker.cancel(call)
|
||||
stopped += 1
|
||||
return stopped
|
||||
return sum(worker.end_calls(f"{run_id}:") for worker in self.workers())
|
||||
|
||||
def cancel_flow(self, flow: str) -> None:
|
||||
"""End what a live flow has running on any attached worker.
|
||||
|
||||
A live call goes by its node's id, ``flow.node``. A run's carries the
|
||||
run's id first and a compile its op, so neither is touched.
|
||||
A live call goes by its node's id and a suffix, ``flow.node:…``. A
|
||||
run's carries the run's id first and a compile its op, so neither is
|
||||
touched.
|
||||
"""
|
||||
for worker in self.workers():
|
||||
worker.end_calls(f"{flow}.")
|
||||
|
||||
Reference in New Issue
Block a user