A local call waiting for a slot is reached by a stop, a delete or a cancel and ends without running; what a call emits after its flow was stopped is dropped as its result is; a node run by hand whose flow is stopped or deleted tells its caller but publishes no node_error; cancelling a node also ends its calls on workers

backend/tests/flow/test_remote.py

@@ -17,6 +17,7 @@ from collections.abc import Iterator
 
 import pytest
 
+from fluksio.api.routes.flows import cancel_node
 from fluksio.flow import remote
 from fluksio.flow.controller import FlowController
 from fluksio.flow.messages import DType, MessageSpec
@@ -263,7 +264,7 @@ class Loading(FakeSocket):
             self.worker.deliver({"call_id": payload["call_id"], "ok": True})
 
 
-def live_voice(loop, tmp_path, pool: PythonWorkerPool, local: str = ""):
+def live_voice(loop, tmp_path, pool: PythonWorkerPool, local: str = "", events=None):
     """A live flow 'voice' whose node 'gpu' runs on a worker and, given
     *local*, a node 'cpu' running that source in *pool*. Built and started."""
     store = FlowStore(tmp_path / "flows")
@@ -287,12 +288,23 @@ def live_voice(loop, tmp_path, pool: PythonWorkerPool, local: str = ""):
     )
     hub = RemoteWorkerHub()
     hub.attach(socket.worker)
-    controller = FlowController(store, workers=pool, remote=hub)
+    controller = FlowController(store, workers=pool, remote=hub, events=events)
     asyncio.run(controller.reload())
     socket.arrived.clear()
     return controller, store, socket
 
 
+class Events(list):
+    """Stands in for the event bus: keeps what was published."""
+
+    def publish(self, event: dict) -> None:
+        self.append(event)
+
+
+#: What a worker answers for a call it was told to cancel.
+CANCELLED = {"ok": False, "error": {"type": "NodeCancelled", "message": "cancelled"}}
+
+
 def test_deleting_a_flow_ends_its_call_and_drops_a_late_answer(loop, tmp_path):
     """A node that never returns kept its worker slot past its flow's delete,
     and an answer that came anyway put the deleted flow's value back."""
@@ -366,6 +378,70 @@ def test_stopping_a_flow_ends_its_calls_here_and_on_workers(loop, tmp_path):
         pool.stop()
 
 
+def test_a_node_run_by_hand_and_stopped_tells_the_caller_alone(loop, tmp_path):
+    """Stopping a flow failed the node a person had started by hand as an
+    error, which the alert rules fire on."""
+    pool = PythonWorkerPool(python=sys.executable, size=1)
+    events = Events()
+    controller, _, socket = live_voice(loop, tmp_path, pool, events=events)
+
+    answer: list[str] = []
+    running = call_in_thread(
+        lambda: answer.append(controller.trigger_node("voice.gpu"))
+    )
+    assert socket.arrived.wait(5)
+    call = socket.sent[-1]["call_id"]
+    socket.arrived.clear()
+    asyncio.run(controller.set_enabled("voice", "voice", False))
+    assert socket.arrived.wait(5)
+    socket.worker.deliver({"call_id": call, **CANCELLED})
+    running.join(timeout=5)
+
+    assert answer == ["NodeCancelled: cancelled"]
+    assert [event for event in events if event["type"] == "node_error"] == []
+
+
+def test_what_a_call_emits_after_its_flow_stopped_is_dropped(loop, tmp_path):
+    """An emission between a flow's stop and its call's end was still written,
+    though what the call returned no longer was."""
+    pool = PythonWorkerPool(python=sys.executable, size=1)
+    controller, _, socket = live_voice(loop, tmp_path, pool)
+
+    running = call_in_thread(lambda: controller.run_flow("voice"))
+    assert socket.arrived.wait(5)
+    call = socket.sent[-1]["call_id"]
+    socket.arrived.clear()
+    asyncio.run(controller.set_enabled("voice", "voice", False))
+    assert socket.arrived.wait(5)
+    socket.worker.deliver(
+        {"call_id": call, "event": "emit", "outputs": {"reply": "late"}}
+    )
+    socket.worker.deliver({"call_id": call, **CANCELLED})
+    running.join(timeout=5)
+
+    assert not running.is_alive()
+    assert "voice.reply" not in controller.state
+
+
+def test_stopping_a_node_reaches_its_call_on_a_worker(loop, tmp_path):
+    """The canvas's stop ended only a call in the local pool."""
+    pool = PythonWorkerPool(python=sys.executable, size=1)
+    controller, _, socket = live_voice(loop, tmp_path, pool)
+
+    running = call_in_thread(lambda: controller.run_flow("voice"))
+    assert socket.arrived.wait(5)
+    call = socket.sent[-1]["call_id"]
+    socket.arrived.clear()
+    assert cancel_node("voice", "gpu", controller).message == "Stopped 'gpu'"
+    assert socket.arrived.wait(5)
+    assert socket.sent[-1] == {"op": "cancel", "call_id": call}
+
+    socket.worker.deliver({"call_id": call, **CANCELLED})
+    running.join(timeout=5)
+    assert not running.is_alive()
+    assert socket.worker.in_flight == 0
+
+
 def test_a_service_and_a_compile_leave_the_only_slot_to_calls(loop):
     """A service holding the slot for good would leave the worker blocked."""
     hub = RemoteWorkerHub()

backend/tests/flow/test_workers.py

@@ -176,6 +176,40 @@ def test_a_running_node_can_be_cancelled(pool):
         stopper.join()
 
 
+def test_a_call_waiting_for_a_slot_is_cancelled_before_it_runs(pool):
+    """A cancel missed a call queued for a slot, so it took the slot that
+    cancelling the node ahead of it freed, and ran."""
+    outcome: dict[str, object] = {}
+
+    def call(node: str, code: str) -> None:
+        try:
+            outcome[node] = pool.run("demo", node, code, {}, f"demo.{node}", 30)
+        except NodeCancelled as exc:
+            outcome[node] = exc
+
+    sleep = "import time\n\n\ndef process():\n    time.sleep(30)\n"
+    ahead = threading.Thread(target=call, args=("ahead", sleep), daemon=True)
+    ahead.start()
+    deadline = time.monotonic() + 10
+    while ("", "demo.ahead") not in pool._running:
+        assert time.monotonic() < deadline, "the first call never started"
+        time.sleep(0.01)
+    queued = threading.Thread(
+        target=call, args=("queued", "def process():\n    return 1\n"), daemon=True
+    )
+    queued.start()
+    deadline = time.monotonic() + 5
+    while not pool.cancel("demo.queued"):
+        assert time.monotonic() < deadline, "the queued call was not found"
+        time.sleep(0.01)
+
+    assert pool.cancel("demo.ahead")
+    ahead.join(timeout=10)
+    queued.join(timeout=10)
+    assert isinstance(outcome["queued"], NodeCancelled)
+    assert "waited" in str(outcome["queued"])
+
+
 def test_a_result_that_is_not_json_is_refused(pool):
     with pytest.raises(Exception, match="cannot be sent back as JSON"):
         run(pool, "def process():\n    return {'out': {1, 2}}\n")