Deleting a live flow ends the calls it has out on workers and drops what a removed node still returns, so a node that never returns gives its slot and process back and a late answer no longer recreates the flow's values; a cancel goes through the call's own inbox and is sent after the call's frame, so a service stopped right after starting no longer cancels before its serve
This commit is contained in:
@@ -862,6 +862,10 @@ class FlowController:
|
||||
if built is None:
|
||||
if name:
|
||||
pipeline.remove_flow(name)
|
||||
# A call still out on a worker would keep its slot and its
|
||||
# process there, for a result that is now dropped.
|
||||
if self.remote is not None:
|
||||
self.remote.cancel_flow(name)
|
||||
self.loaded = {
|
||||
node_id: entry
|
||||
for node_id, entry in self.loaded.items()
|
||||
|
||||
@@ -264,15 +264,16 @@ class Node:
|
||||
if self._on_health is not None:
|
||||
self._on_health(self, status, detail)
|
||||
|
||||
def bind(self, pipeline: Pipeline) -> None:
|
||||
def bind(self, pipeline: Pipeline | None) -> None:
|
||||
"""
|
||||
Bind this node to a pipeline for external triggering.
|
||||
|
||||
Once bound, the node can trigger downstream execution when called.
|
||||
This is typically done automatically during pipeline construction.
|
||||
|
||||
:param pipeline: The pipeline to bind this node to.
|
||||
:type pipeline: Pipeline
|
||||
:param pipeline: The pipeline to bind this node to, or None once its
|
||||
flow has been taken out of the graph.
|
||||
:type pipeline: Pipeline | None
|
||||
"""
|
||||
self._pipeline = pipeline
|
||||
|
||||
@@ -466,6 +467,9 @@ class Node:
|
||||
collected = logs.Collector()
|
||||
with logs.capture(collected):
|
||||
outputs = self.f(params=self.params) or {}
|
||||
if self._pipeline is None:
|
||||
# Its flow was deleted while it ran (`Pipeline.remove_flow`).
|
||||
return None
|
||||
self._pipeline.publish_log(self, collected, "")
|
||||
|
||||
return self._pipeline.trigger(self, self._to_messages(outputs), durable=durable)
|
||||
|
||||
@@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import queue
|
||||
import shlex
|
||||
import uuid
|
||||
from collections.abc import Iterable
|
||||
@@ -169,9 +170,12 @@ class ServiceNode(Node):
|
||||
"env": self.cfg.env,
|
||||
"weights": self.cfg.weights,
|
||||
}
|
||||
# On a thread, as every call is, and holding no slot.
|
||||
# On a thread, as every call is, and holding no slot. Its inbox is made
|
||||
# here, so a stop that comes before the thread has sent the serve can
|
||||
# still end it, after the serve rather than ahead of it.
|
||||
inbox: queue.Queue[dict[str, Any] | None] = queue.Queue()
|
||||
call = asyncio.ensure_future(
|
||||
asyncio.to_thread(worker.request, payload, 0, slot=False)
|
||||
asyncio.to_thread(worker.request, payload, 0, slot=False, inbox=inbox)
|
||||
)
|
||||
# Retrieved however it ends, so one that outlives a stop is not
|
||||
# reported at exit.
|
||||
@@ -197,7 +201,7 @@ class ServiceNode(Node):
|
||||
# The flow stopped, or this side failed. Ending the call kills
|
||||
# the server's process group on the worker, and its answer says
|
||||
# that is done, so the next start finds the port free.
|
||||
await asyncio.to_thread(worker.cancel, call_id)
|
||||
inbox.put(worker.END)
|
||||
await asyncio.wait({call}, timeout=STOP_WAIT_S)
|
||||
await asyncio.to_thread(self._withdraw)
|
||||
|
||||
|
||||
@@ -483,8 +483,16 @@ class Pipeline:
|
||||
self._gate_revision += 1
|
||||
|
||||
def remove_flow(self, flow: str) -> None:
|
||||
"""Take a deleted flow out of the graph."""
|
||||
"""Take a deleted flow out of the graph.
|
||||
|
||||
Its nodes are unbound as well. One still running has nowhere left to
|
||||
put what it returns, and writing it anyway would bring the deleted
|
||||
flow's values back.
|
||||
"""
|
||||
gone = self.flow_nodes(flow)
|
||||
self.replace_flow(flow, [])
|
||||
for node in gone:
|
||||
node.bind(None)
|
||||
|
||||
def _seed(self, initial_values: dict[str, Any] | None) -> None:
|
||||
"""Give messages a starting value, without overwriting one already there.
|
||||
@@ -1119,6 +1127,9 @@ class Pipeline:
|
||||
result = node.execute(inputs)
|
||||
finally:
|
||||
ARRIVED.reset(token)
|
||||
if node._pipeline is not self:
|
||||
# Its flow was deleted while it ran (`remove_flow`).
|
||||
return None
|
||||
self.publish_log(node, collected, "")
|
||||
|
||||
if (
|
||||
@@ -1180,6 +1191,10 @@ class Pipeline:
|
||||
)
|
||||
return result or {}
|
||||
except Exception as exc:
|
||||
if node._pipeline is not self:
|
||||
# Deleted while it ran, and most likely failing because the
|
||||
# call was ended for that: nobody is left to tell.
|
||||
return None
|
||||
# One failing node must not take the rest of the graph down.
|
||||
error = self.publish_error(node, exc, collected, entry_id)
|
||||
if self.observer is not None:
|
||||
|
||||
@@ -94,6 +94,11 @@ class WorkerInventory:
|
||||
class RemoteWorker:
|
||||
"""One attached worker, and the calls it has in flight."""
|
||||
|
||||
#: Put in a call's inbox to end the call. The thread waiting there sends
|
||||
#: the cancel, and only once the call's own frame is out, so a cancel never
|
||||
#: reaches the worker ahead of the call it ends, which it would ignore.
|
||||
END: dict[str, Any] = {}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
@@ -169,16 +174,21 @@ class RemoteWorker:
|
||||
timeout: float,
|
||||
on_event: Callable[[dict[str, Any]], None] | None = None,
|
||||
slot: bool = True,
|
||||
inbox: queue.Queue[dict[str, Any] | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Send a call and wait for its answer, on a thread rather than the loop.
|
||||
|
||||
``slot`` off is for what must not queue behind running calls, nor hold
|
||||
one: a compile, and a service, which would hold one for good.
|
||||
|
||||
``inbox`` is the caller's own, for one that may end the call before
|
||||
this thread has even sent it: `END` put there is read after the frame.
|
||||
"""
|
||||
if self._gone:
|
||||
raise RemoteError(f"worker '{self.name}' is no longer attached")
|
||||
call_id = str(payload["call_id"])
|
||||
inbox: queue.Queue[dict[str, Any] | None] = queue.Queue()
|
||||
if inbox is None:
|
||||
inbox = queue.Queue()
|
||||
# Blocking here is the backpressure, exactly as taking a slot is in the
|
||||
# local pool.
|
||||
if slot:
|
||||
@@ -213,6 +223,11 @@ class RemoteWorker:
|
||||
) from None
|
||||
if message is None:
|
||||
raise RemoteError(f"worker '{self.name}' went away mid-call")
|
||||
if message is self.END:
|
||||
# The worker answers once the call is killed, and only
|
||||
# that answer gives the slot back.
|
||||
self.cancel(call_id)
|
||||
continue
|
||||
kind = message.get("event")
|
||||
if kind == "heartbeat":
|
||||
continue
|
||||
@@ -260,6 +275,17 @@ class RemoteWorker:
|
||||
with self._lock:
|
||||
return [call for call in self._pending if call.startswith(f"{run_id}:")]
|
||||
|
||||
def end_calls(self, prefix: str) -> None:
|
||||
"""End every call whose id starts with *prefix*, each once it is sent."""
|
||||
with self._lock:
|
||||
inboxes = [
|
||||
inbox
|
||||
for call, inbox in self._pending.items()
|
||||
if call.startswith(prefix)
|
||||
]
|
||||
for inbox in inboxes:
|
||||
inbox.put(self.END)
|
||||
|
||||
def cancel(self, call_id: str) -> None:
|
||||
"""Ask the worker to kill what it is running for this call."""
|
||||
# A closed loop refuses the coroutine and leaves it never awaited, so it
|
||||
@@ -509,3 +535,12 @@ class RemoteWorkerHub:
|
||||
worker.cancel(call)
|
||||
stopped += 1
|
||||
return stopped
|
||||
|
||||
def cancel_flow(self, flow: str) -> None:
|
||||
"""End what a live flow has running on any attached worker.
|
||||
|
||||
A live call goes by its node's id, ``flow.node``. A run's carries the
|
||||
run's id first and a compile its op, so neither is touched.
|
||||
"""
|
||||
for worker in self.workers():
|
||||
worker.end_calls(f"{flow}.")
|
||||
|
||||
Reference in New Issue
Block a user