Stopping a live flow ends the calls it has running, on workers and in the local pool, and deleting one now ends its local calls too; what a call returns after its flow was stopped or deleted is dropped by a per-flow count the node compares, so a flow started again never takes an old call's answer. A live remote call's id carries a suffix, so two calls of one node no longer share it, and a call is registered before it queues for a slot, so ending its flow or run reaches it and it is never sent; a run's cancel goes through the call's own thread as well

This commit is contained in:
2026-11-11 12:03:43 +01:00
parent cd8831cfd3
commit 7132d26f36
8 changed files with 195 additions and 75 deletions
+97 -34
View File
@@ -244,49 +244,65 @@ def test_cancelling_a_run_reaches_only_that_run(loop):
call_in_thread(lambda: call("run-b"))
assert socket.arrived.wait(5)
socket.arrived.clear()
assert hub.cancel_run("run-a") == 1
# Sent by the call's own thread, once its frame is out.
assert socket.arrived.wait(5)
cancels = [frame for frame in socket.sent if frame.get("op") == "cancel"]
assert [frame["call_id"] for frame in cancels] == ["run-a:flow.node"]
class Loading(FakeSocket):
"""A worker that loads whatever it is sent and leaves each call running."""
worker: RemoteWorker
async def send_json(self, payload: dict) -> None:
await super().send_json(payload)
if payload["op"] == "compile":
self.worker.deliver({"call_id": payload["call_id"], "ok": True})
def live_voice(loop, tmp_path, pool: PythonWorkerPool, local: str = ""):
"""A live flow 'voice' whose node 'gpu' runs on a worker and, given
*local*, a node 'cpu' running that source in *pool*. Built and started."""
store = FlowStore(tmp_path / "flows")
nodes = [
NodeDef(
id="gpu",
worker="gpu",
provides=[MessageSpec(name="reply", dtype=DType.STR)],
)
]
if local:
nodes.append(
NodeDef(id="cpu", provides=[MessageSpec(name="local", dtype=DType.STR)])
)
store.write_node_source("voice", "cpu", local)
store.write_flow(FlowDef(name="voice", nodes=nodes))
store.write_node_source("voice", "gpu", "def process():\n return None\n")
socket = Loading()
socket.worker = RemoteWorker(
name="gpu1", labels=["gpu"], send=socket.send_json, loop=loop, max_parallel=2
)
hub = RemoteWorkerHub()
hub.attach(socket.worker)
controller = FlowController(store, workers=pool, remote=hub)
asyncio.run(controller.reload())
socket.arrived.clear()
return controller, store, socket
def test_deleting_a_flow_ends_its_call_and_drops_a_late_answer(loop, tmp_path):
"""A node that never returns kept its worker slot past its flow's delete,
and an answer that came anyway put the deleted flow's value back."""
store = FlowStore(tmp_path / "flows")
store.write_flow(
FlowDef(
name="voice",
nodes=[
NodeDef(
id="gpu",
worker="gpu",
provides=[MessageSpec(name="reply", dtype=DType.STR)],
)
],
)
)
store.write_node_source("voice", "gpu", "def process():\n return None\n")
pool = PythonWorkerPool(python=sys.executable, size=1)
controller, store, socket = live_voice(loop, tmp_path, pool)
worker = socket.worker
class Agent(FakeSocket):
"""Loads whatever it is sent, and leaves a call running until told."""
async def send_json(self, payload: dict) -> None:
await super().send_json(payload)
if payload["op"] == "compile":
worker.deliver({"call_id": payload["call_id"], "ok": True})
socket = Agent()
worker = RemoteWorker(name="gpu1", labels=["gpu"], send=socket.send_json, loop=loop)
hub = RemoteWorkerHub()
hub.attach(worker)
controller = FlowController(
store, workers=PythonWorkerPool(python=sys.executable, size=1), remote=hub
)
asyncio.run(controller.reload())
socket.arrived.clear()
running = call_in_thread(lambda: controller.run_flow("voice"))
assert socket.arrived.wait(5)
call = socket.sent[-1]["call_id"]
assert socket.sent[-1]["op"] == "run"
socket.arrived.clear()
@@ -294,15 +310,62 @@ def test_deleting_a_flow_ends_its_call_and_drops_a_late_answer(loop, tmp_path):
asyncio.run(controller.reload_flow("voice"))
controller.forget_flow(key)
assert socket.arrived.wait(5)
assert socket.sent[-1] == {"op": "cancel", "call_id": "voice.gpu"}
assert socket.sent[-1] == {"op": "cancel", "call_id": call}
worker.deliver({"call_id": "voice.gpu", "ok": True, "result": {"reply": "late"}})
worker.deliver({"call_id": call, "ok": True, "result": {"reply": "late"}})
running.join(timeout=5)
assert not running.is_alive()
assert worker.in_flight == 0
assert "voice.reply" not in controller.state
def test_stopping_a_flow_ends_its_calls_here_and_on_workers(loop, tmp_path):
"""A stopped flow's calls ran on, here and on the worker, and what they
returned was written to its state, or, once it was started again, taken
for a new call's: a live call's id was its node's."""
pool = PythonWorkerPool(python=sys.executable, size=1)
pool.start()
local = "import time\n\n\ndef process():\n time.sleep(60)\n return None\n"
controller, _, socket = live_voice(loop, tmp_path, pool, local=local)
worker = socket.worker
try:
running = call_in_thread(lambda: controller.run_flow("voice"))
assert socket.arrived.wait(5)
old = socket.sent[-1]["call_id"]
deadline = time.monotonic() + 10
while ("", "voice.cpu") not in pool._running:
assert time.monotonic() < deadline, "the local node never started"
time.sleep(0.01)
socket.arrived.clear()
asyncio.run(controller.set_enabled("voice", "voice", False))
assert socket.arrived.wait(5)
assert socket.sent[-1] == {"op": "cancel", "call_id": old}
deadline = time.monotonic() + 5
while ("", "voice.cpu") in pool._running:
assert time.monotonic() < deadline, "the local node was left running"
time.sleep(0.01)
# Started again before the worker has answered the old call.
asyncio.run(controller.set_enabled("voice", "voice", True))
socket.arrived.clear()
gpu = controller.get_node("voice.gpu")
assert gpu is not None
again = call_in_thread(lambda: controller.pipeline.run({}, nodes={gpu}))
assert socket.arrived.wait(5)
new = socket.sent[-1]["call_id"]
worker.deliver({"call_id": old, "ok": True, "result": {"reply": "old"}})
running.join(timeout=5)
assert not running.is_alive()
assert "voice.reply" not in controller.state
worker.deliver({"call_id": new, "ok": True, "result": {"reply": "new"}})
again.join(timeout=5)
assert controller.state.get("voice.reply") == "new"
finally:
pool.stop()
def test_a_service_and_a_compile_leave_the_only_slot_to_calls(loop):
"""A service holding the slot for good would leave the worker blocked."""
hub = RemoteWorkerHub()
+3 -2
View File
@@ -180,10 +180,11 @@ def test_a_service_that_never_gets_healthy_reads_down(monkeypatch):
asyncio.run(scenario())
def test_a_stop_right_after_the_start_ends_the_serve_after_it(monkeypatch):
def test_a_stop_right_after_the_start_leaves_no_server_running(monkeypatch):
"""The stop's cancel went out on a thread of its own, so with the serve's
thread slow to start it reached the worker first. A worker ignores a cancel
for a call it does not know, and ran that server until its socket dropped.
Ended before its thread has sent it, the serve now never goes out.
"""
monkeypatch.setattr(remote, "SILENCE_S", 1.0)
monkeypatch.setattr(service, "STOP_WAIT_S", 1.0)
@@ -228,4 +229,4 @@ def test_a_stop_right_after_the_start_ends_the_serve_after_it(monkeypatch):
await node.stop()
asyncio.run(scenario())
assert [frame["op"] for frame in agent.sent] == ["serve", "cancel"]
assert agent.sent == []