Stopping a live flow ends the calls it has running, on workers and in the local pool, and deleting one now ends its local calls too; what a call returns after its flow was stopped or deleted is dropped by a per-flow count the node compares, so a flow started again never takes an old call's answer. A live remote call's id carries a suffix, so two calls of one node no longer share it, and a call is registered before it queues for a slot, so ending its flow or run reaches it and it is never sent; a run's cancel goes through the call's own thread as well
This commit is contained in:
@@ -244,49 +244,65 @@ def test_cancelling_a_run_reaches_only_that_run(loop):
|
||||
call_in_thread(lambda: call("run-b"))
|
||||
assert socket.arrived.wait(5)
|
||||
|
||||
socket.arrived.clear()
|
||||
assert hub.cancel_run("run-a") == 1
|
||||
# Sent by the call's own thread, once its frame is out.
|
||||
assert socket.arrived.wait(5)
|
||||
cancels = [frame for frame in socket.sent if frame.get("op") == "cancel"]
|
||||
assert [frame["call_id"] for frame in cancels] == ["run-a:flow.node"]
|
||||
|
||||
|
||||
class Loading(FakeSocket):
|
||||
"""A worker that loads whatever it is sent and leaves each call running."""
|
||||
|
||||
worker: RemoteWorker
|
||||
|
||||
async def send_json(self, payload: dict) -> None:
|
||||
await super().send_json(payload)
|
||||
if payload["op"] == "compile":
|
||||
self.worker.deliver({"call_id": payload["call_id"], "ok": True})
|
||||
|
||||
|
||||
def live_voice(loop, tmp_path, pool: PythonWorkerPool, local: str = ""):
|
||||
"""A live flow 'voice' whose node 'gpu' runs on a worker and, given
|
||||
*local*, a node 'cpu' running that source in *pool*. Built and started."""
|
||||
store = FlowStore(tmp_path / "flows")
|
||||
nodes = [
|
||||
NodeDef(
|
||||
id="gpu",
|
||||
worker="gpu",
|
||||
provides=[MessageSpec(name="reply", dtype=DType.STR)],
|
||||
)
|
||||
]
|
||||
if local:
|
||||
nodes.append(
|
||||
NodeDef(id="cpu", provides=[MessageSpec(name="local", dtype=DType.STR)])
|
||||
)
|
||||
store.write_node_source("voice", "cpu", local)
|
||||
store.write_flow(FlowDef(name="voice", nodes=nodes))
|
||||
store.write_node_source("voice", "gpu", "def process():\n return None\n")
|
||||
socket = Loading()
|
||||
socket.worker = RemoteWorker(
|
||||
name="gpu1", labels=["gpu"], send=socket.send_json, loop=loop, max_parallel=2
|
||||
)
|
||||
hub = RemoteWorkerHub()
|
||||
hub.attach(socket.worker)
|
||||
controller = FlowController(store, workers=pool, remote=hub)
|
||||
asyncio.run(controller.reload())
|
||||
socket.arrived.clear()
|
||||
return controller, store, socket
|
||||
|
||||
|
||||
def test_deleting_a_flow_ends_its_call_and_drops_a_late_answer(loop, tmp_path):
|
||||
"""A node that never returns kept its worker slot past its flow's delete,
|
||||
and an answer that came anyway put the deleted flow's value back."""
|
||||
store = FlowStore(tmp_path / "flows")
|
||||
store.write_flow(
|
||||
FlowDef(
|
||||
name="voice",
|
||||
nodes=[
|
||||
NodeDef(
|
||||
id="gpu",
|
||||
worker="gpu",
|
||||
provides=[MessageSpec(name="reply", dtype=DType.STR)],
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
store.write_node_source("voice", "gpu", "def process():\n return None\n")
|
||||
pool = PythonWorkerPool(python=sys.executable, size=1)
|
||||
controller, store, socket = live_voice(loop, tmp_path, pool)
|
||||
worker = socket.worker
|
||||
|
||||
class Agent(FakeSocket):
|
||||
"""Loads whatever it is sent, and leaves a call running until told."""
|
||||
|
||||
async def send_json(self, payload: dict) -> None:
|
||||
await super().send_json(payload)
|
||||
if payload["op"] == "compile":
|
||||
worker.deliver({"call_id": payload["call_id"], "ok": True})
|
||||
|
||||
socket = Agent()
|
||||
worker = RemoteWorker(name="gpu1", labels=["gpu"], send=socket.send_json, loop=loop)
|
||||
hub = RemoteWorkerHub()
|
||||
hub.attach(worker)
|
||||
controller = FlowController(
|
||||
store, workers=PythonWorkerPool(python=sys.executable, size=1), remote=hub
|
||||
)
|
||||
asyncio.run(controller.reload())
|
||||
|
||||
socket.arrived.clear()
|
||||
running = call_in_thread(lambda: controller.run_flow("voice"))
|
||||
assert socket.arrived.wait(5)
|
||||
call = socket.sent[-1]["call_id"]
|
||||
assert socket.sent[-1]["op"] == "run"
|
||||
|
||||
socket.arrived.clear()
|
||||
@@ -294,15 +310,62 @@ def test_deleting_a_flow_ends_its_call_and_drops_a_late_answer(loop, tmp_path):
|
||||
asyncio.run(controller.reload_flow("voice"))
|
||||
controller.forget_flow(key)
|
||||
assert socket.arrived.wait(5)
|
||||
assert socket.sent[-1] == {"op": "cancel", "call_id": "voice.gpu"}
|
||||
assert socket.sent[-1] == {"op": "cancel", "call_id": call}
|
||||
|
||||
worker.deliver({"call_id": "voice.gpu", "ok": True, "result": {"reply": "late"}})
|
||||
worker.deliver({"call_id": call, "ok": True, "result": {"reply": "late"}})
|
||||
running.join(timeout=5)
|
||||
assert not running.is_alive()
|
||||
assert worker.in_flight == 0
|
||||
assert "voice.reply" not in controller.state
|
||||
|
||||
|
||||
def test_stopping_a_flow_ends_its_calls_here_and_on_workers(loop, tmp_path):
|
||||
"""A stopped flow's calls ran on, here and on the worker, and what they
|
||||
returned was written to its state, or, once it was started again, taken
|
||||
for a new call's: a live call's id was its node's."""
|
||||
pool = PythonWorkerPool(python=sys.executable, size=1)
|
||||
pool.start()
|
||||
local = "import time\n\n\ndef process():\n time.sleep(60)\n return None\n"
|
||||
controller, _, socket = live_voice(loop, tmp_path, pool, local=local)
|
||||
worker = socket.worker
|
||||
try:
|
||||
running = call_in_thread(lambda: controller.run_flow("voice"))
|
||||
assert socket.arrived.wait(5)
|
||||
old = socket.sent[-1]["call_id"]
|
||||
deadline = time.monotonic() + 10
|
||||
while ("", "voice.cpu") not in pool._running:
|
||||
assert time.monotonic() < deadline, "the local node never started"
|
||||
time.sleep(0.01)
|
||||
|
||||
socket.arrived.clear()
|
||||
asyncio.run(controller.set_enabled("voice", "voice", False))
|
||||
assert socket.arrived.wait(5)
|
||||
assert socket.sent[-1] == {"op": "cancel", "call_id": old}
|
||||
deadline = time.monotonic() + 5
|
||||
while ("", "voice.cpu") in pool._running:
|
||||
assert time.monotonic() < deadline, "the local node was left running"
|
||||
time.sleep(0.01)
|
||||
|
||||
# Started again before the worker has answered the old call.
|
||||
asyncio.run(controller.set_enabled("voice", "voice", True))
|
||||
socket.arrived.clear()
|
||||
gpu = controller.get_node("voice.gpu")
|
||||
assert gpu is not None
|
||||
again = call_in_thread(lambda: controller.pipeline.run({}, nodes={gpu}))
|
||||
assert socket.arrived.wait(5)
|
||||
new = socket.sent[-1]["call_id"]
|
||||
|
||||
worker.deliver({"call_id": old, "ok": True, "result": {"reply": "old"}})
|
||||
running.join(timeout=5)
|
||||
assert not running.is_alive()
|
||||
assert "voice.reply" not in controller.state
|
||||
worker.deliver({"call_id": new, "ok": True, "result": {"reply": "new"}})
|
||||
again.join(timeout=5)
|
||||
assert controller.state.get("voice.reply") == "new"
|
||||
finally:
|
||||
pool.stop()
|
||||
|
||||
|
||||
def test_a_service_and_a_compile_leave_the_only_slot_to_calls(loop):
|
||||
"""A service holding the slot for good would leave the worker blocked."""
|
||||
hub = RemoteWorkerHub()
|
||||
|
||||
@@ -180,10 +180,11 @@ def test_a_service_that_never_gets_healthy_reads_down(monkeypatch):
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_a_stop_right_after_the_start_ends_the_serve_after_it(monkeypatch):
|
||||
def test_a_stop_right_after_the_start_leaves_no_server_running(monkeypatch):
|
||||
"""The stop's cancel went out on a thread of its own, so with the serve's
|
||||
thread slow to start it reached the worker first. A worker ignores a cancel
|
||||
for a call it does not know, and ran that server until its socket dropped.
|
||||
Ended before its thread has sent it, the serve now never goes out.
|
||||
"""
|
||||
monkeypatch.setattr(remote, "SILENCE_S", 1.0)
|
||||
monkeypatch.setattr(service, "STOP_WAIT_S", 1.0)
|
||||
@@ -228,4 +229,4 @@ def test_a_stop_right_after_the_start_ends_the_serve_after_it(monkeypatch):
|
||||
await node.stop()
|
||||
|
||||
asyncio.run(scenario())
|
||||
assert [frame["op"] for frame in agent.sent] == ["serve", "cancel"]
|
||||
assert agent.sent == []
|
||||
|
||||
Reference in New Issue
Block a user