Deleting a live flow ends the calls it has out on workers and drops what a removed node still returns, so a node that never returns gives its slot and process back and a late answer no longer recreates the flow's values; a cancel goes through the call's own inbox and is sent after the call's frame, so a service stopped right after starting no longer cancels before its serve

backend/tests/flow/test_remote.py

@@ -9,6 +9,7 @@ what happened rather than a wait that never ends.
 
 import asyncio
 import gc
+import sys
 import threading
 import time
 import warnings
@@ -17,15 +18,20 @@ from collections.abc import Iterator
 import pytest
 
 from fluksio.flow import remote
+from fluksio.flow.controller import FlowController
+from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.remote import (
     NoWorker,
     RemoteWorker,
     RemoteWorkerHub,
     WorkerInventory,
 )
+from fluksio.flow.schemas import FlowDef, NodeDef
+from fluksio.flow.store import FlowStore
 from fluksio.flow.workers import (
     MAX_REMOTE_NAME,
     NodeTimeout,
+    PythonWorkerPool,
     RemoteError,
     _remote_class,
 )
@@ -243,6 +249,60 @@ def test_cancelling_a_run_reaches_only_that_run(loop):
     assert [frame["call_id"] for frame in cancels] == ["run-a:flow.node"]
 
 
+def test_deleting_a_flow_ends_its_call_and_drops_a_late_answer(loop, tmp_path):
+    """A node that never returns kept its worker slot past its flow's delete,
+    and an answer that came anyway put the deleted flow's value back."""
+    store = FlowStore(tmp_path / "flows")
+    store.write_flow(
+        FlowDef(
+            name="voice",
+            nodes=[
+                NodeDef(
+                    id="gpu",
+                    worker="gpu",
+                    provides=[MessageSpec(name="reply", dtype=DType.STR)],
+                )
+            ],
+        )
+    )
+    store.write_node_source("voice", "gpu", "def process():\n    return None\n")
+
+    class Agent(FakeSocket):
+        """Loads whatever it is sent, and leaves a call running until told."""
+
+        async def send_json(self, payload: dict) -> None:
+            await super().send_json(payload)
+            if payload["op"] == "compile":
+                worker.deliver({"call_id": payload["call_id"], "ok": True})
+
+    socket = Agent()
+    worker = RemoteWorker(name="gpu1", labels=["gpu"], send=socket.send_json, loop=loop)
+    hub = RemoteWorkerHub()
+    hub.attach(worker)
+    controller = FlowController(
+        store, workers=PythonWorkerPool(python=sys.executable, size=1), remote=hub
+    )
+    asyncio.run(controller.reload())
+
+    socket.arrived.clear()
+    running = call_in_thread(lambda: controller.run_flow("voice"))
+    assert socket.arrived.wait(5)
+    assert socket.sent[-1]["op"] == "run"
+
+    socket.arrived.clear()
+    key = store.delete_flow("voice")
+    asyncio.run(controller.reload_flow("voice"))
+    controller.forget_flow(key)
+    assert socket.arrived.wait(5)
+    assert socket.sent[-1] == {"op": "cancel", "call_id": "voice.gpu"}
+
+    worker.deliver({"call_id": "voice.gpu", "ok": True, "result": {"reply": "late"}})
+    running.join(timeout=5)
+    assert not running.is_alive()
+    assert worker.in_flight == 0
+    assert "voice.reply" not in controller.state
+
+
 def test_a_service_and_a_compile_leave_the_only_slot_to_calls(loop):
     """A service holding the slot for good would leave the worker blocked."""
     hub = RemoteWorkerHub()

backend/tests/flow/test_service_node.py

@@ -3,6 +3,7 @@
 import asyncio
 import time
 from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
 
 from fluksio.flow import remote
 from fluksio.flow.messages import DType, MessageSpec
@@ -177,3 +178,54 @@ def test_a_service_that_never_gets_healthy_reads_down(monkeypatch):
         await node.stop()
 
     asyncio.run(scenario())
+
+
+def test_a_stop_right_after_the_start_ends_the_serve_after_it(monkeypatch):
+    """The stop's cancel went out on a thread of its own, so with the serve's
+    thread slow to start it reached the worker first. A worker ignores a cancel
+    for a call it does not know, and ran that server until its socket dropped.
+    """
+    monkeypatch.setattr(remote, "SILENCE_S", 1.0)
+    monkeypatch.setattr(service, "STOP_WAIT_S", 1.0)
+
+    async def refused(client, url):
+        return "is not answering"
+
+    monkeypatch.setattr(ServiceNode, "_probe", staticmethod(refused))
+
+    class Busy(ThreadPoolExecutor):
+        """Starts what it is given late, as a pool with every thread taken."""
+
+        busy = True
+        late = 0
+
+        def submit(self, fn, /, *args, **kwargs):
+            if not self.busy:
+                return super().submit(fn, *args, **kwargs)
+            self.late += 1
+            return super().submit(lambda: time.sleep(0.2) or fn(*args, **kwargs))
+
+    agent = Agent()
+
+    async def scenario() -> None:
+        pool = Busy()
+        asyncio.get_running_loop().set_default_executor(pool)
+        agent.worker = RemoteWorker(
+            name="gpu-worker",
+            labels=["gpu"],
+            send=agent.send_json,
+            loop=asyncio.get_running_loop(),
+        )
+        hub = RemoteWorkerHub()
+        hub.attach(agent.worker)
+        node = ServiceNode(params={"command": "serve", "worker": "gpu"})
+        node.assign_flow("voice", "parrot")
+        node.hub = hub
+
+        await node.start()
+        await until(lambda: pool.late == 1)
+        pool.busy = False
+        await node.stop()
+
+    asyncio.run(scenario())
+    assert [frame["op"] for frame in agent.sent] == ["serve", "cancel"]