Deleting a live flow ends the calls it has out on workers and drops what a removed node still returns, so a node that never returns gives its slot and process back and a late answer no longer recreates the flow's values; a cancel goes through the call's own inbox and is sent after the call's frame, so a service stopped right after starting no longer cancels before its serve

This commit is contained in:
2026-11-11 09:40:59 +01:00
parent 138c632576
commit a3d532f3a5
8 changed files with 187 additions and 12 deletions
+36 -1
View File
@@ -94,6 +94,11 @@ class WorkerInventory:
class RemoteWorker:
"""One attached worker, and the calls it has in flight."""
#: Put in a call's inbox to end the call. The thread waiting there sends
#: the cancel, and only once the call's own frame is out, so a cancel never
#: reaches the worker ahead of the call it ends, which it would ignore.
END: dict[str, Any] = {}
def __init__(
self,
name: str,
@@ -169,16 +174,21 @@ class RemoteWorker:
timeout: float,
on_event: Callable[[dict[str, Any]], None] | None = None,
slot: bool = True,
inbox: queue.Queue[dict[str, Any] | None] | None = None,
) -> dict[str, Any]:
"""Send a call and wait for its answer, on a thread rather than the loop.
``slot`` off is for what must not queue behind running calls, nor hold
one: a compile, and a service, which would hold one for good.
``inbox`` is the caller's own, for one that may end the call before
this thread has even sent it: `END` put there is read after the frame.
"""
if self._gone:
raise RemoteError(f"worker '{self.name}' is no longer attached")
call_id = str(payload["call_id"])
inbox: queue.Queue[dict[str, Any] | None] = queue.Queue()
if inbox is None:
inbox = queue.Queue()
# Blocking here is the backpressure, exactly as taking a slot is in the
# local pool.
if slot:
@@ -213,6 +223,11 @@ class RemoteWorker:
) from None
if message is None:
raise RemoteError(f"worker '{self.name}' went away mid-call")
if message is self.END:
# The worker answers once the call is killed, and only
# that answer gives the slot back.
self.cancel(call_id)
continue
kind = message.get("event")
if kind == "heartbeat":
continue
@@ -260,6 +275,17 @@ class RemoteWorker:
with self._lock:
return [call for call in self._pending if call.startswith(f"{run_id}:")]
def end_calls(self, prefix: str) -> None:
"""End every call whose id starts with *prefix*, each once it is sent."""
with self._lock:
inboxes = [
inbox
for call, inbox in self._pending.items()
if call.startswith(prefix)
]
for inbox in inboxes:
inbox.put(self.END)
def cancel(self, call_id: str) -> None:
"""Ask the worker to kill what it is running for this call."""
# A closed loop refuses the coroutine and leaves it never awaited, so it
@@ -509,3 +535,12 @@ class RemoteWorkerHub:
worker.cancel(call)
stopped += 1
return stopped
def cancel_flow(self, flow: str) -> None:
"""End what a live flow has running on any attached worker.
A live call goes by its node's id, ``flow.node``. A run's carries the
run's id first and a compile its op, so neither is touched.
"""
for worker in self.workers():
worker.end_calls(f"{flow}.")