A local call waiting for a slot is reached by a stop, a delete or a cancel and ends without running; what a call emits after its flow was stopped is dropped as its result is; a node run by hand whose flow is stopped or deleted tells its caller but publishes no node_error; cancelling a node also ends its calls on workers

backend/fluksio/flow/workers.py

@@ -278,17 +278,21 @@ class PythonWorkerPool:
         # runs at once, and cancelling one of them must not kill the others.
         # The live pipeline's own executions carry an empty run.
         self._running: dict[tuple[str, str], _Worker] = {}
+        # Calls still waiting for a slot, each by its (run, node). A cancel
+        # marks one, and it ends once it has a slot instead of running.
+        self._queued: dict[threading.Event, tuple[str, str]] = {}
         self._generation = 0
         self._stopped = False
         self._requests = itertools.count(1)
         # Not a lock over the pool — the slot queue is what serialises calls.
-        # It covers the two places where bookkeeping must not interleave: a
-        # cancel taking a worker out of ``_running`` while ``_request`` is
-        # handing that same worker back to a slot, and two respawns draining
-        # the idle queue at once. Registering into ``_running``, the generation
-        # counter and the ``_stopped`` flag are deliberately outside it — each
-        # is one atomic operation, and a cancel that misses a node by a
-        # microsecond is a cancel that arrived a microsecond early.
+        # It covers the places where bookkeeping must not interleave: a cancel
+        # taking a worker out of ``_running`` while ``_request`` is handing
+        # that same worker back to a slot, a cancel looking for a call while it
+        # moves from ``_queued`` to ``_running``, and two respawns draining the
+        # idle queue at once. The generation counter and the ``_stopped`` flag
+        # are deliberately outside it — each is one atomic operation, and a
+        # cancel that misses a node by a microsecond is a cancel that arrived a
+        # microsecond early.
         self._lock = threading.Lock()
 
     # -------------------------------------------------------------------------
@@ -444,10 +448,27 @@ class PythonWorkerPool:
         run_id: str = "",
         on_event: Callable[[dict[str, Any]], None] | None = None,
     ) -> dict[str, Any]:
-        worker = self._acquire()
         key = (run_id, node_id)
+        # Registered before it waits for a slot, so a cancel reaches a call
+        # queued behind a busy one, and it ends there without running.
+        queued = threading.Event()
+        if node_id:
+            with self._lock:
+                self._queued[queued] = key
+        try:
+            worker = self._acquire()
+        except BaseException:
+            with self._lock:
+                self._queued.pop(queued, None)
+            raise
+        with self._lock:
+            self._queued.pop(queued, None)
+            if queued.is_set():
+                self._release(worker)
+                raise NodeCancelled("cancelled while it waited for a worker")
+            if node_id:
+                self._running[key] = worker
         if node_id:
-            self._running[key] = worker
             self._publish(
                 {
                     "type": "node_started",
@@ -679,20 +700,25 @@ class PythonWorkerPool:
         return call
 
     def cancel(self, node_id: str, run_id: str = "") -> bool:
-        """Stop a node that is running now. False when there was nothing to stop."""
+        """Stop a node that is running now, and its calls waiting for a slot.
+        False when there was nothing to stop."""
+        key = (run_id, node_id)
         with self._lock:
-            worker = self._running.get((run_id, node_id))
+            worker = self._running.get(key)
+            queued = [call for call, owner in self._queued.items() if owner == key]
+            for call in queued:
+                call.set()
             children = list(self._children.values())
             if worker is not None:
                 worker.cancelled = True
                 worker.kill()
-                return True
         # A node with declared resources runs in a pool of its own, and it is
         # the same node to whoever pressed cancel.
-        return any(child.cancel(node_id, run_id) for child in children)
+        stopped = [child.cancel(node_id, run_id) for child in children]
+        return worker is not None or bool(queued) or any(stopped)
 
     def cancel_run(self, run_id: str) -> int:
-        """Stop every node this run has in a worker right now."""
+        """Stop every node this run has in a worker right now or waiting for one."""
         with self._lock:
             workers = [
                 worker
@@ -702,8 +728,15 @@ class PythonWorkerPool:
             for worker in workers:
                 worker.cancelled = True
                 worker.kill()
+            queued = [call for call, key in self._queued.items() if key[0] == run_id]
+            for call in queued:
+                call.set()
             children = list(self._children.values())
-        return len(workers) + sum(child.cancel_run(run_id) for child in children)
+        return (
+            len(workers)
+            + len(queued)
+            + sum(child.cancel_run(run_id) for child in children)
+        )
 
     def _publish(self, event: dict[str, Any]) -> None:
         if self.events is not None: