A local call waiting for a slot is reached by a stop, a delete or a cancel and ends without running; what a call emits after its flow was stopped is dropped as its result is; a node run by hand whose flow is stopped or deleted tells its caller but publishes no node_error; cancelling a node also ends its calls on workers

backend/fluksio/api/routes/flows.py

@@ -1141,14 +1141,19 @@ async def trigger_node(
 
 @router.post("/{flow_id}/nodes/{node_id}/cancel", response_model=Message)
 def cancel_node(flow_id: str, node_id: str, controller: FlowControllerDep) -> Any:
-    """Stop a node that is running right now, by killing the worker running it.
+    """Stop a node that is running right now, here or on an attached worker,
+    by killing the process running it.
 
     Idempotent on purpose: by the time a click reaches here the node may well
     have finished, and that is the outcome that was asked for.
     """
-    name = _read_flow(controller, flow_id).name
-    pool = controller.workers
-    if pool is not None and pool.cancel(f"{name}.{node_id}"):
+    node = f"{_read_flow(controller, flow_id).name}.{node_id}"
+    pool, remote = controller.workers, controller.remote
+    stopped = pool is not None and pool.cancel(node)
+    # Not ``or``: a node may have calls in both places.
+    if remote is not None and remote.cancel_node(node):
+        stopped = True
+    if stopped:
         return Message(message=f"Stopped '{node_id}'")
     return Message(message=f"'{node_id}' was not running")
 

backend/fluksio/flow/controller.py

@@ -2113,6 +2113,8 @@ class FlowController:
         node = self.get_node(node_id)
         if node is None:
             raise KeyError(node_id)
+        pipeline = self.pipeline
+        ended = pipeline.ended(node.flow) if pipeline is not None else 0
         try:
             if node.requires and values:
                 node.trigger(values, durable=False)
@@ -2121,9 +2123,14 @@ class FlowController:
         except EngineHeld:
             raise
         except Exception as exc:
-            if self.pipeline is None:
+            if pipeline is None:
                 raise
-            return self.pipeline.publish_error(node, exc)
+            if pipeline.ended(node.flow) != ended:
+                # Its flow was stopped or deleted while it ran, most likely
+                # ending the call. The person who clicked is told, alert rules
+                # are not, as for a cascade.
+                return f"{type(exc).__name__}: {exc}"
+            return pipeline.publish_error(node, exc)
         return ""
 
     def _publish(self, event: dict[str, Any]) -> None:

backend/fluksio/flow/nodes/base.py

@@ -43,6 +43,11 @@ RESERVED_SETTINGS = frozenset({"synchronous"})
 #: worker request carries it to ``fluksio.arrived()`` inside the node.
 ARRIVED: ContextVar[list[str] | None] = ContextVar("arrived", default=None)
 
+#: What ``Pipeline.ended`` read for the running node's flow when its call
+#: began. What the call emits once that has moved is dropped, as is what it
+#: returns.
+STARTED: ContextVar[int | None] = ContextVar("started", default=None)
+
 
 class Node:
     """
@@ -466,8 +471,12 @@ class Node:
         if not outputs and not self.requires:
             ended = self._pipeline.ended(self.flow)
             collected = logs.Collector()
-            with logs.capture(collected):
-                outputs = self.f(params=self.params) or {}
+            token = STARTED.set(ended)
+            try:
+                with logs.capture(collected):
+                    outputs = self.f(params=self.params) or {}
+            finally:
+                STARTED.reset(token)
             if self._pipeline is None or self._pipeline.ended(self.flow) != ended:
                 # Its flow was stopped or deleted while it ran.
                 return None
@@ -492,7 +501,11 @@ class Node:
         :rtype: dict | None
         """
         ended = self._pipeline.ended(self.flow) if self._pipeline is not None else 0
-        outputs = self.execute(inputs)
+        token = STARTED.set(ended)
+        try:
+            outputs = self.execute(inputs)
+        finally:
+            STARTED.reset(token)
         if self._pipeline is None:
             return outputs
         if self._pipeline.ended(self.flow) != ended:

backend/fluksio/flow/pipeline.py

@@ -44,7 +44,7 @@ from fluksio.flow.namespace import (
     VERSION,
 )
 from fluksio.flow.nodes import Node
-from fluksio.flow.nodes.base import ARRIVED
+from fluksio.flow.nodes.base import ARRIVED, STARTED
 from fluksio.flow.queue import WorkQueue
 from fluksio.flow.state import MemoryState, StateBackend
 
@@ -1143,10 +1143,12 @@ class Pipeline:
                     return restored or {}
 
             token = ARRIVED.set(arrived)
+            began = STARTED.set(ended)
             try:
                 with logs.capture(collected):
                     result = node.execute(inputs)
             finally:
+                STARTED.reset(began)
                 ARRIVED.reset(token)
             if self.ended(node.flow) != ended:
                 # Its flow was stopped or deleted while it ran (`end_running`).
@@ -1289,6 +1291,11 @@ class Pipeline:
         three thousand mid-node cascades would leave "the run has finished"
         with no meaning.
         """
+        started = STARTED.get()
+        if started is not None and self.ended(node.flow) != started:
+            # Its flow was stopped or deleted after the call began
+            # (`end_running`), so this is dropped as its result will be.
+            return
         self._observe_emission(node, outputs)
         passed, stale = self._throttled(node, outputs)
         if not passed:

backend/fluksio/flow/remote.py

@@ -545,6 +545,11 @@ class RemoteWorkerHub:
         """Kill whatever this run has executing on any attached worker."""
         return sum(worker.end_calls(f"{run_id}:") for worker in self.workers())
 
+    def cancel_node(self, node_id: str) -> int:
+        """End what a live node has running on any attached worker, its calls
+        being ``flow.node:…``."""
+        return sum(worker.end_calls(f"{node_id}:") for worker in self.workers())
+
     def cancel_flow(self, flow: str) -> None:
         """End what a live flow has running on any attached worker.
 

backend/fluksio/flow/workers.py

@@ -278,17 +278,21 @@ class PythonWorkerPool:
         # runs at once, and cancelling one of them must not kill the others.
         # The live pipeline's own executions carry an empty run.
         self._running: dict[tuple[str, str], _Worker] = {}
+        # Calls still waiting for a slot, each by its (run, node). A cancel
+        # marks one, and it ends once it has a slot instead of running.
+        self._queued: dict[threading.Event, tuple[str, str]] = {}
         self._generation = 0
         self._stopped = False
         self._requests = itertools.count(1)
         # Not a lock over the pool — the slot queue is what serialises calls.
-        # It covers the two places where bookkeeping must not interleave: a
-        # cancel taking a worker out of ``_running`` while ``_request`` is
-        # handing that same worker back to a slot, and two respawns draining
-        # the idle queue at once. Registering into ``_running``, the generation
-        # counter and the ``_stopped`` flag are deliberately outside it — each
-        # is one atomic operation, and a cancel that misses a node by a
-        # microsecond is a cancel that arrived a microsecond early.
+        # It covers the places where bookkeeping must not interleave: a cancel
+        # taking a worker out of ``_running`` while ``_request`` is handing
+        # that same worker back to a slot, a cancel looking for a call while it
+        # moves from ``_queued`` to ``_running``, and two respawns draining the
+        # idle queue at once. The generation counter and the ``_stopped`` flag
+        # are deliberately outside it — each is one atomic operation, and a
+        # cancel that misses a node by a microsecond is a cancel that arrived a
+        # microsecond early.
         self._lock = threading.Lock()
 
     # -------------------------------------------------------------------------
@@ -444,10 +448,27 @@ class PythonWorkerPool:
         run_id: str = "",
         on_event: Callable[[dict[str, Any]], None] | None = None,
     ) -> dict[str, Any]:
-        worker = self._acquire()
         key = (run_id, node_id)
+        # Registered before it waits for a slot, so a cancel reaches a call
+        # queued behind a busy one, and it ends there without running.
+        queued = threading.Event()
+        if node_id:
+            with self._lock:
+                self._queued[queued] = key
+        try:
+            worker = self._acquire()
+        except BaseException:
+            with self._lock:
+                self._queued.pop(queued, None)
+            raise
+        with self._lock:
+            self._queued.pop(queued, None)
+            if queued.is_set():
+                self._release(worker)
+                raise NodeCancelled("cancelled while it waited for a worker")
+            if node_id:
+                self._running[key] = worker
         if node_id:
-            self._running[key] = worker
             self._publish(
                 {
                     "type": "node_started",
@@ -679,20 +700,25 @@ class PythonWorkerPool:
         return call
 
     def cancel(self, node_id: str, run_id: str = "") -> bool:
-        """Stop a node that is running now. False when there was nothing to stop."""
+        """Stop a node that is running now, and its calls waiting for a slot.
+        False when there was nothing to stop."""
+        key = (run_id, node_id)
         with self._lock:
-            worker = self._running.get((run_id, node_id))
+            worker = self._running.get(key)
+            queued = [call for call, owner in self._queued.items() if owner == key]
+            for call in queued:
+                call.set()
             children = list(self._children.values())
             if worker is not None:
                 worker.cancelled = True
                 worker.kill()
-                return True
         # A node with declared resources runs in a pool of its own, and it is
         # the same node to whoever pressed cancel.
-        return any(child.cancel(node_id, run_id) for child in children)
+        stopped = [child.cancel(node_id, run_id) for child in children]
+        return worker is not None or bool(queued) or any(stopped)
 
     def cancel_run(self, run_id: str) -> int:
-        """Stop every node this run has in a worker right now."""
+        """Stop every node this run has in a worker right now or waiting for one."""
         with self._lock:
             workers = [
                 worker
@@ -702,8 +728,15 @@ class PythonWorkerPool:
             for worker in workers:
                 worker.cancelled = True
                 worker.kill()
+            queued = [call for call, key in self._queued.items() if key[0] == run_id]
+            for call in queued:
+                call.set()
             children = list(self._children.values())
-        return len(workers) + sum(child.cancel_run(run_id) for child in children)
+        return (
+            len(workers)
+            + len(queued)
+            + sum(child.cancel_run(run_id) for child in children)
+        )
 
     def _publish(self, event: dict[str, Any]) -> None:
         if self.events is not None: