Stopping a live flow ends the calls it has running, on workers and in the local pool, and deleting one now ends its local calls too; what a call returns after its flow was stopped or deleted is dropped by a per-flow count the node compares, so a flow started again never takes an old call's answer. A live remote call's id carries a suffix, so two calls of one node no longer share it, and a call is registered before it queues for a slot, so ending its flow or run reaches it and it is never sent; a run's cancel goes through the call's own thread as well

This commit is contained in:
2026-11-11 12:03:43 +01:00
parent cd8831cfd3
commit 7132d26f36
8 changed files with 195 additions and 75 deletions
+30 -20
View File
@@ -27,13 +27,19 @@ import queue
import sys
import threading
import time
import uuid
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from fluksio.flow.nodes.base import ARRIVED, NodeOutputError
from fluksio.flow.resources import ResourceAccountant
from fluksio.flow.workers import NodeTimeout, RemoteError, _remote_class
from fluksio.flow.workers import (
NodeCancelled,
NodeTimeout,
RemoteError,
_remote_class,
)
logger = logging.getLogger(__name__)
@@ -189,13 +195,19 @@ class RemoteWorker:
call_id = str(payload["call_id"])
if inbox is None:
inbox = queue.Queue()
# Blocking here is the backpressure, exactly as taking a slot is in the
# local pool.
if slot:
self._slots.acquire()
# Registered before it queues for a slot, so ending its flow or its run
# reaches a call still waiting for one.
with self._lock:
self._pending[call_id] = inbox
try:
# Blocking here is the backpressure, exactly as taking a slot is in
# the local pool.
if slot:
self._slots.acquire()
if not inbox.empty():
# Ended, or the worker gone, while it waited: nothing went out,
# so there is nothing on the worker to cancel.
raise NodeCancelled(f"ended before '{self.name}' was sent the call")
future = asyncio.run_coroutine_threadsafe(self._send(payload), self._loop)
future.result(timeout=SEND_TIMEOUT_S)
@@ -271,12 +283,9 @@ class RemoteWorker:
with self._lock:
return len(self._pending)
def calls_of(self, run_id: str) -> list[str]:
with self._lock:
return [call for call in self._pending if call.startswith(f"{run_id}:")]
def end_calls(self, prefix: str) -> None:
"""End every call whose id starts with *prefix*, each once it is sent."""
def end_calls(self, prefix: str) -> int:
"""End every call whose id starts with *prefix*, each once it is sent,
and one still waiting for a slot before it is."""
with self._lock:
inboxes = [
inbox
@@ -285,6 +294,7 @@ class RemoteWorker:
]
for inbox in inboxes:
inbox.put(self.END)
return len(inboxes)
def cancel(self, call_id: str) -> None:
"""Ask the worker to kill what it is running for this call."""
@@ -423,7 +433,11 @@ class RemoteWorkerHub:
"""
payload = {
"op": "run",
"call_id": f"{run_id}:{node_id}" if run_id else node_id,
# A live node can run in two cascades at once, so its calls are
# told apart; a run runs each node once.
"call_id": (
f"{run_id}:{node_id}" if run_id else f"{node_id}:{uuid.uuid4().hex[:8]}"
),
"flow": flow,
"node": node,
"source": source,
@@ -529,18 +543,14 @@ class RemoteWorkerHub:
def cancel_run(self, run_id: str) -> int:
"""Kill whatever this run has executing on any attached worker."""
stopped = 0
for worker in self.workers():
for call in worker.calls_of(run_id):
worker.cancel(call)
stopped += 1
return stopped
return sum(worker.end_calls(f"{run_id}:") for worker in self.workers())
def cancel_flow(self, flow: str) -> None:
"""End what a live flow has running on any attached worker.
A live call goes by its node's id, ``flow.node``. A run's carries the
run's id first and a compile its op, so neither is touched.
A live call goes by its node's id and a suffix, ``flow.node:…``. A
run's carries the run's id first and a compile its op, so neither is
touched.
"""
for worker in self.workers():
worker.end_calls(f"{flow}.")