Deleting a live flow ends the calls it has out on workers and drops what a removed node still returns, so a node that never returns gives its slot and process back and a late answer no longer recreates the flow's values; a cancel goes through the call's own inbox and is sent after the call's frame, so a service stopped right after starting no longer cancels before its serve

backend/tests/flow/test_service_node.py

@@ -3,6 +3,7 @@
 import asyncio
 import time
 from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
 
 from fluksio.flow import remote
 from fluksio.flow.messages import DType, MessageSpec
@@ -177,3 +178,54 @@ def test_a_service_that_never_gets_healthy_reads_down(monkeypatch):
         await node.stop()
 
     asyncio.run(scenario())
+
+
+def test_a_stop_right_after_the_start_ends_the_serve_after_it(monkeypatch):
+    """The stop's cancel went out on a thread of its own, so with the serve's
+    thread slow to start it reached the worker first. A worker ignores a cancel
+    for a call it does not know, and ran that server until its socket dropped.
+    """
+    monkeypatch.setattr(remote, "SILENCE_S", 1.0)
+    monkeypatch.setattr(service, "STOP_WAIT_S", 1.0)
+
+    async def refused(client, url):
+        return "is not answering"
+
+    monkeypatch.setattr(ServiceNode, "_probe", staticmethod(refused))
+
+    class Busy(ThreadPoolExecutor):
+        """Starts what it is given late, as a pool with every thread taken."""
+
+        busy = True
+        late = 0
+
+        def submit(self, fn, /, *args, **kwargs):
+            if not self.busy:
+                return super().submit(fn, *args, **kwargs)
+            self.late += 1
+            return super().submit(lambda: time.sleep(0.2) or fn(*args, **kwargs))
+
+    agent = Agent()
+
+    async def scenario() -> None:
+        pool = Busy()
+        asyncio.get_running_loop().set_default_executor(pool)
+        agent.worker = RemoteWorker(
+            name="gpu-worker",
+            labels=["gpu"],
+            send=agent.send_json,
+            loop=asyncio.get_running_loop(),
+        )
+        hub = RemoteWorkerHub()
+        hub.attach(agent.worker)
+        node = ServiceNode(params={"command": "serve", "worker": "gpu"})
+        node.assign_flow("voice", "parrot")
+        node.hub = hub
+
+        await node.start()
+        await until(lambda: pool.late == 1)
+        pool.busy = False
+        await node.stop()
+
+    asyncio.run(scenario())
+    assert [frame["op"] for frame in agent.sent] == ["serve", "cancel"]