Bound the pipeline teardown so a stuck node cannot wedge the controller

A node's stop() and a supervised task's cancellation are both waited on
inside the rebuild lock, and neither had a deadline: an MQTT client whose
broker never acknowledges the disconnect leaves aiomqtt's __aexit__
waiting forever, so reload() never returned and every start, stop or
publish behind it hung until the container was restarted.

Each node now gets five seconds to close and is abandoned after that, and
cancel_all reports what is still running rather than waiting on it — it
also no longer swallows a cancellation aimed at the caller, which used to
make the lock holder unkillable. A rebuild asked for by a request gives up
on the lock after fifteen seconds with RebuildBusy, answered as a 503.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01StpRc2C6au1WJ1EUU7fsfu

backend/tests/flow/test_runtime_control.py

@@ -1,13 +1,16 @@
 """Stopping a flow takes it off the engine; pausing holds its nodes."""
 
+import asyncio
 from pathlib import Path
 from typing import Any
 
+import pytest
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 
 from fluksio.api.deps import get_current_user
 from fluksio.api.routes.flows import router
+from fluksio.flow.controller import FlowController, LoadedNode
 from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.nodes import Node
 from fluksio.flow.pipeline import Pipeline
@@ -165,3 +168,33 @@ def test_stepping_an_unknown_flow_is_a_404(tmp_path: Path):
 
     assert response.status_code == 404
     assert controller.calls == []
+
+
+def test_a_node_that_will_not_stop_does_not_wedge_the_rebuild(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+):
+    """A node closing a connection nobody answers used to hold the lock forever.
+
+    Everything that deploys rebuilds, so the next publish, start or stop then
+    waited on a lock that was never given back.
+    """
+    monkeypatch.setattr("fluksio.flow.controller.NODE_STOP_TIMEOUT", 0.05)
+
+    class NeverStops(Node):
+        async def stop(self, app: FastAPI | None = None) -> None:
+            await asyncio.Event().wait()
+
+    node = NeverStops(f=lambda params: None, name="stuck")
+    node.assign_flow("heating", "stuck")
+    engine = FlowController(FlowStore(tmp_path / "flows"))
+    engine.loaded = {
+        "heating.stuck": LoadedNode(id="heating.stuck", flow="heating", node=node)
+    }
+
+    async def scenario() -> None:
+        # The first rebuild gives up on the node it cannot stop, and the second
+        # is not left queueing behind a lock the first never released.
+        await engine.reload()
+        await engine.reload()
+
+    asyncio.run(asyncio.wait_for(scenario(), timeout=5))