Deleting a live flow ends the calls it has out on workers and drops what a removed node still returns, so a node that never returns gives its slot and process back and a late answer no longer recreates the flow's values; a cancel goes through the call's own inbox and is sent after the call's frame, so a service stopped right after starting no longer cancels before its serve
This commit is contained in:
@@ -94,6 +94,11 @@ class WorkerInventory:
|
||||
class RemoteWorker:
|
||||
"""One attached worker, and the calls it has in flight."""
|
||||
|
||||
#: Put in a call's inbox to end the call. The thread waiting there sends
|
||||
#: the cancel, and only once the call's own frame is out, so a cancel never
|
||||
#: reaches the worker ahead of the call it ends, which it would ignore.
|
||||
END: dict[str, Any] = {}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
@@ -169,16 +174,21 @@ class RemoteWorker:
|
||||
timeout: float,
|
||||
on_event: Callable[[dict[str, Any]], None] | None = None,
|
||||
slot: bool = True,
|
||||
inbox: queue.Queue[dict[str, Any] | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Send a call and wait for its answer, on a thread rather than the loop.
|
||||
|
||||
``slot`` off is for what must not queue behind running calls, nor hold
|
||||
one: a compile, and a service, which would hold one for good.
|
||||
|
||||
``inbox`` is the caller's own, for one that may end the call before
|
||||
this thread has even sent it: `END` put there is read after the frame.
|
||||
"""
|
||||
if self._gone:
|
||||
raise RemoteError(f"worker '{self.name}' is no longer attached")
|
||||
call_id = str(payload["call_id"])
|
||||
inbox: queue.Queue[dict[str, Any] | None] = queue.Queue()
|
||||
if inbox is None:
|
||||
inbox = queue.Queue()
|
||||
# Blocking here is the backpressure, exactly as taking a slot is in the
|
||||
# local pool.
|
||||
if slot:
|
||||
@@ -213,6 +223,11 @@ class RemoteWorker:
|
||||
) from None
|
||||
if message is None:
|
||||
raise RemoteError(f"worker '{self.name}' went away mid-call")
|
||||
if message is self.END:
|
||||
# The worker answers once the call is killed, and only
|
||||
# that answer gives the slot back.
|
||||
self.cancel(call_id)
|
||||
continue
|
||||
kind = message.get("event")
|
||||
if kind == "heartbeat":
|
||||
continue
|
||||
@@ -260,6 +275,17 @@ class RemoteWorker:
|
||||
with self._lock:
|
||||
return [call for call in self._pending if call.startswith(f"{run_id}:")]
|
||||
|
||||
def end_calls(self, prefix: str) -> None:
|
||||
"""End every call whose id starts with *prefix*, each once it is sent."""
|
||||
with self._lock:
|
||||
inboxes = [
|
||||
inbox
|
||||
for call, inbox in self._pending.items()
|
||||
if call.startswith(prefix)
|
||||
]
|
||||
for inbox in inboxes:
|
||||
inbox.put(self.END)
|
||||
|
||||
def cancel(self, call_id: str) -> None:
|
||||
"""Ask the worker to kill what it is running for this call."""
|
||||
# A closed loop refuses the coroutine and leaves it never awaited, so it
|
||||
@@ -509,3 +535,12 @@ class RemoteWorkerHub:
|
||||
worker.cancel(call)
|
||||
stopped += 1
|
||||
return stopped
|
||||
|
||||
def cancel_flow(self, flow: str) -> None:
|
||||
"""End what a live flow has running on any attached worker.
|
||||
|
||||
A live call goes by its node's id, ``flow.node``. A run's carries the
|
||||
run's id first and a compile its op, so neither is touched.
|
||||
"""
|
||||
for worker in self.workers():
|
||||
worker.end_calls(f"{flow}.")
|
||||
|
||||
Reference in New Issue
Block a user