Stopping a live flow ends the calls it has running, on workers and in the local pool, and deleting one now ends its local calls too; what a call returns after its flow was stopped or deleted is dropped by a per-flow count the node compares, so a flow started again never takes an old call's answer. A live remote call's id carries a suffix, so two calls of one node no longer share it, and a call is registered before it queues for a slot, so ending its flow or run reaches it and it is never sent; a run's cancel goes through the call's own thread as well
This commit is contained in:
@@ -27,13 +27,19 @@ import queue
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from fluksio.flow.nodes.base import ARRIVED, NodeOutputError
|
||||
from fluksio.flow.resources import ResourceAccountant
|
||||
from fluksio.flow.workers import NodeTimeout, RemoteError, _remote_class
|
||||
from fluksio.flow.workers import (
|
||||
NodeCancelled,
|
||||
NodeTimeout,
|
||||
RemoteError,
|
||||
_remote_class,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -189,13 +195,19 @@ class RemoteWorker:
|
||||
call_id = str(payload["call_id"])
|
||||
if inbox is None:
|
||||
inbox = queue.Queue()
|
||||
# Blocking here is the backpressure, exactly as taking a slot is in the
|
||||
# local pool.
|
||||
if slot:
|
||||
self._slots.acquire()
|
||||
# Registered before it queues for a slot, so ending its flow or its run
|
||||
# reaches a call still waiting for one.
|
||||
with self._lock:
|
||||
self._pending[call_id] = inbox
|
||||
try:
|
||||
# Blocking here is the backpressure, exactly as taking a slot is in
|
||||
# the local pool.
|
||||
if slot:
|
||||
self._slots.acquire()
|
||||
if not inbox.empty():
|
||||
# Ended, or the worker gone, while it waited: nothing went out,
|
||||
# so there is nothing on the worker to cancel.
|
||||
raise NodeCancelled(f"ended before '{self.name}' was sent the call")
|
||||
future = asyncio.run_coroutine_threadsafe(self._send(payload), self._loop)
|
||||
future.result(timeout=SEND_TIMEOUT_S)
|
||||
|
||||
@@ -271,12 +283,9 @@ class RemoteWorker:
|
||||
with self._lock:
|
||||
return len(self._pending)
|
||||
|
||||
def calls_of(self, run_id: str) -> list[str]:
|
||||
with self._lock:
|
||||
return [call for call in self._pending if call.startswith(f"{run_id}:")]
|
||||
|
||||
def end_calls(self, prefix: str) -> None:
|
||||
"""End every call whose id starts with *prefix*, each once it is sent."""
|
||||
def end_calls(self, prefix: str) -> int:
|
||||
"""End every call whose id starts with *prefix*, each once it is sent,
|
||||
and one still waiting for a slot before it is."""
|
||||
with self._lock:
|
||||
inboxes = [
|
||||
inbox
|
||||
@@ -285,6 +294,7 @@ class RemoteWorker:
|
||||
]
|
||||
for inbox in inboxes:
|
||||
inbox.put(self.END)
|
||||
return len(inboxes)
|
||||
|
||||
def cancel(self, call_id: str) -> None:
|
||||
"""Ask the worker to kill what it is running for this call."""
|
||||
@@ -423,7 +433,11 @@ class RemoteWorkerHub:
|
||||
"""
|
||||
payload = {
|
||||
"op": "run",
|
||||
"call_id": f"{run_id}:{node_id}" if run_id else node_id,
|
||||
# A live node can run in two cascades at once, so its calls are
|
||||
# told apart; a run runs each node once.
|
||||
"call_id": (
|
||||
f"{run_id}:{node_id}" if run_id else f"{node_id}:{uuid.uuid4().hex[:8]}"
|
||||
),
|
||||
"flow": flow,
|
||||
"node": node,
|
||||
"source": source,
|
||||
@@ -529,18 +543,14 @@ class RemoteWorkerHub:
|
||||
|
||||
def cancel_run(self, run_id: str) -> int:
|
||||
"""Kill whatever this run has executing on any attached worker."""
|
||||
stopped = 0
|
||||
for worker in self.workers():
|
||||
for call in worker.calls_of(run_id):
|
||||
worker.cancel(call)
|
||||
stopped += 1
|
||||
return stopped
|
||||
return sum(worker.end_calls(f"{run_id}:") for worker in self.workers())
|
||||
|
||||
def cancel_flow(self, flow: str) -> None:
|
||||
"""End what a live flow has running on any attached worker.
|
||||
|
||||
A live call goes by its node's id, ``flow.node``. A run's carries the
|
||||
run's id first and a compile its op, so neither is touched.
|
||||
A live call goes by its node's id and a suffix, ``flow.node:…``. A
|
||||
run's carries the run's id first and a compile its op, so neither is
|
||||
touched.
|
||||
"""
|
||||
for worker in self.workers():
|
||||
worker.end_calls(f"{flow}.")
|
||||
|
||||
Reference in New Issue
Block a user