Deleting a live flow ends the calls it has out on workers and drops what a removed node still returns, so a node that never returns gives its slot and process back and a late answer no longer recreates the flow's values; a cancel goes through the call's own inbox and is sent after the call's frame, so a service stopped right after starting no longer cancels before its serve

backend/tests/flow/test_remote.py

@@ -9,6 +9,7 @@ what happened rather than a wait that never ends.
 
 import asyncio
 import gc
+import sys
 import threading
 import time
 import warnings
@@ -17,15 +18,20 @@ from collections.abc import Iterator
 import pytest
 
 from fluksio.flow import remote
+from fluksio.flow.controller import FlowController
+from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.remote import (
     NoWorker,
     RemoteWorker,
     RemoteWorkerHub,
     WorkerInventory,
 )
+from fluksio.flow.schemas import FlowDef, NodeDef
+from fluksio.flow.store import FlowStore
 from fluksio.flow.workers import (
     MAX_REMOTE_NAME,
     NodeTimeout,
+    PythonWorkerPool,
     RemoteError,
     _remote_class,
 )
@@ -243,6 +249,60 @@ def test_cancelling_a_run_reaches_only_that_run(loop):
     assert [frame["call_id"] for frame in cancels] == ["run-a:flow.node"]
 
 
+def test_deleting_a_flow_ends_its_call_and_drops_a_late_answer(loop, tmp_path):
+    """A node that never returns kept its worker slot past its flow's delete,
+    and an answer that came anyway put the deleted flow's value back."""
+    store = FlowStore(tmp_path / "flows")
+    store.write_flow(
+        FlowDef(
+            name="voice",
+            nodes=[
+                NodeDef(
+                    id="gpu",
+                    worker="gpu",
+                    provides=[MessageSpec(name="reply", dtype=DType.STR)],
+                )
+            ],
+        )
+    )
+    store.write_node_source("voice", "gpu", "def process():\n    return None\n")
+
+    class Agent(FakeSocket):
+        """Loads whatever it is sent, and leaves a call running until told."""
+
+        async def send_json(self, payload: dict) -> None:
+            await super().send_json(payload)
+            if payload["op"] == "compile":
+                worker.deliver({"call_id": payload["call_id"], "ok": True})
+
+    socket = Agent()
+    worker = RemoteWorker(name="gpu1", labels=["gpu"], send=socket.send_json, loop=loop)
+    hub = RemoteWorkerHub()
+    hub.attach(worker)
+    controller = FlowController(
+        store, workers=PythonWorkerPool(python=sys.executable, size=1), remote=hub
+    )
+    asyncio.run(controller.reload())
+
+    socket.arrived.clear()
+    running = call_in_thread(lambda: controller.run_flow("voice"))
+    assert socket.arrived.wait(5)
+    assert socket.sent[-1]["op"] == "run"
+
+    socket.arrived.clear()
+    key = store.delete_flow("voice")
+    asyncio.run(controller.reload_flow("voice"))
+    controller.forget_flow(key)
+    assert socket.arrived.wait(5)
+    assert socket.sent[-1] == {"op": "cancel", "call_id": "voice.gpu"}
+
+    worker.deliver({"call_id": "voice.gpu", "ok": True, "result": {"reply": "late"}})
+    running.join(timeout=5)
+    assert not running.is_alive()
+    assert worker.in_flight == 0
+    assert "voice.reply" not in controller.state
+
+
 def test_a_service_and_a_compile_leave_the_only_slot_to_calls(loop):
     """A service holding the slot for good would leave the worker blocked."""
     hub = RemoteWorkerHub()