A local call waiting for a slot is reached by a stop, a delete or a cancel and ends without running; what a call emits after its flow was stopped is dropped as its result is; a node run by hand whose flow is stopped or deleted tells its caller but publishes no node_error; cancelling a node also ends its calls on workers

backend/tests/flow/test_remote.py

@@ -17,6 +17,7 @@ from collections.abc import Iterator
 
 import pytest
 
+from fluksio.api.routes.flows import cancel_node
 from fluksio.flow import remote
 from fluksio.flow.controller import FlowController
 from fluksio.flow.messages import DType, MessageSpec
@@ -263,7 +264,7 @@ class Loading(FakeSocket):
             self.worker.deliver({"call_id": payload["call_id"], "ok": True})
 
 
-def live_voice(loop, tmp_path, pool: PythonWorkerPool, local: str = ""):
+def live_voice(loop, tmp_path, pool: PythonWorkerPool, local: str = "", events=None):
     """A live flow 'voice' whose node 'gpu' runs on a worker and, given
     *local*, a node 'cpu' running that source in *pool*. Built and started."""
     store = FlowStore(tmp_path / "flows")
@@ -287,12 +288,23 @@ def live_voice(loop, tmp_path, pool: PythonWorkerPool, local: str = ""):
     )
     hub = RemoteWorkerHub()
     hub.attach(socket.worker)
-    controller = FlowController(store, workers=pool, remote=hub)
+    controller = FlowController(store, workers=pool, remote=hub, events=events)
     asyncio.run(controller.reload())
     socket.arrived.clear()
     return controller, store, socket
 
 
+class Events(list):
+    """Stands in for the event bus: keeps what was published."""
+
+    def publish(self, event: dict) -> None:
+        self.append(event)
+
+
+#: What a worker answers for a call it was told to cancel.
+CANCELLED = {"ok": False, "error": {"type": "NodeCancelled", "message": "cancelled"}}
+
+
 def test_deleting_a_flow_ends_its_call_and_drops_a_late_answer(loop, tmp_path):
     """A node that never returns kept its worker slot past its flow's delete,
     and an answer that came anyway put the deleted flow's value back."""
@@ -366,6 +378,70 @@ def test_stopping_a_flow_ends_its_calls_here_and_on_workers(loop, tmp_path):
         pool.stop()
 
 
+def test_a_node_run_by_hand_and_stopped_tells_the_caller_alone(loop, tmp_path):
+    """Stopping a flow failed the node a person had started by hand as an
+    error, which the alert rules fire on."""
+    pool = PythonWorkerPool(python=sys.executable, size=1)
+    events = Events()
+    controller, _, socket = live_voice(loop, tmp_path, pool, events=events)
+
+    answer: list[str] = []
+    running = call_in_thread(
+        lambda: answer.append(controller.trigger_node("voice.gpu"))
+    )
+    assert socket.arrived.wait(5)
+    call = socket.sent[-1]["call_id"]
+    socket.arrived.clear()
+    asyncio.run(controller.set_enabled("voice", "voice", False))
+    assert socket.arrived.wait(5)
+    socket.worker.deliver({"call_id": call, **CANCELLED})
+    running.join(timeout=5)
+
+    assert answer == ["NodeCancelled: cancelled"]
+    assert [event for event in events if event["type"] == "node_error"] == []
+
+
+def test_what_a_call_emits_after_its_flow_stopped_is_dropped(loop, tmp_path):
+    """An emission between a flow's stop and its call's end was still written,
+    though what the call returned no longer was."""
+    pool = PythonWorkerPool(python=sys.executable, size=1)
+    controller, _, socket = live_voice(loop, tmp_path, pool)
+
+    running = call_in_thread(lambda: controller.run_flow("voice"))
+    assert socket.arrived.wait(5)
+    call = socket.sent[-1]["call_id"]
+    socket.arrived.clear()
+    asyncio.run(controller.set_enabled("voice", "voice", False))
+    assert socket.arrived.wait(5)
+    socket.worker.deliver(
+        {"call_id": call, "event": "emit", "outputs": {"reply": "late"}}
+    )
+    socket.worker.deliver({"call_id": call, **CANCELLED})
+    running.join(timeout=5)
+
+    assert not running.is_alive()
+    assert "voice.reply" not in controller.state
+
+
+def test_stopping_a_node_reaches_its_call_on_a_worker(loop, tmp_path):
+    """The canvas's stop ended only a call in the local pool."""
+    pool = PythonWorkerPool(python=sys.executable, size=1)
+    controller, _, socket = live_voice(loop, tmp_path, pool)
+
+    running = call_in_thread(lambda: controller.run_flow("voice"))
+    assert socket.arrived.wait(5)
+    call = socket.sent[-1]["call_id"]
+    socket.arrived.clear()
+    assert cancel_node("voice", "gpu", controller).message == "Stopped 'gpu'"
+    assert socket.arrived.wait(5)
+    assert socket.sent[-1] == {"op": "cancel", "call_id": call}
+
+    socket.worker.deliver({"call_id": call, **CANCELLED})
+    running.join(timeout=5)
+    assert not running.is_alive()
+    assert socket.worker.in_flight == 0
+
+
 def test_a_service_and_a_compile_leave_the_only_slot_to_calls(loop):
     """A service holding the slot for good would leave the worker blocked."""
     hub = RemoteWorkerHub()