A node's stop() and a supervised task's cancellation are both waited on inside the rebuild lock, and neither had a deadline: an MQTT client whose broker never acknowledges the disconnect leaves aiomqtt's __aexit__ waiting forever, so reload() never returned and every start, stop or publish behind it hung until the container was restarted. Each node now gets five seconds to close and is abandoned after that, and cancel_all reports what is still running rather than waiting on it — it also no longer swallows a cancellation aimed at the caller, which used to make the lock holder unkillable. A rebuild asked for by a request gives up on the lock after fifteen seconds with RebuildBusy, answered as a 503. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01StpRc2C6au1WJ1EUU7fsfu
201 lines
6.1 KiB
Python
201 lines
6.1 KiB
Python
"""Stopping a flow takes it off the engine; pausing holds its nodes."""
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
|
|
from fluksio.api.deps import get_current_user
|
|
from fluksio.api.routes.flows import router
|
|
from fluksio.flow.controller import FlowController, LoadedNode
|
|
from fluksio.flow.messages import DType, MessageSpec
|
|
from fluksio.flow.nodes import Node
|
|
from fluksio.flow.pipeline import Pipeline
|
|
from fluksio.flow.schemas import FlowDef
|
|
from fluksio.flow.store import FlowStore
|
|
|
|
|
|
def spec(name: str) -> MessageSpec:
|
|
return MessageSpec(name=name, dtype=DType.FLOAT)
|
|
|
|
|
|
def make_node(node_id: str, flow: str, f, requires=(), provides=()) -> Node:
|
|
node = Node(f=f, requires=list(requires), provides=list(provides), name=node_id)
|
|
node.assign_flow(flow, node_id)
|
|
return node
|
|
|
|
|
|
def a_chain(flow: str, ran: list[str]) -> list[Node]:
|
|
"""source → middle: two nodes, so stepping has somewhere to stop."""
|
|
|
|
def source(params):
|
|
ran.append(f"{flow}.source")
|
|
return {"temp": 20.0}
|
|
|
|
def middle(temp, params):
|
|
ran.append(f"{flow}.middle")
|
|
return {"warm": temp > 10}
|
|
|
|
return [
|
|
make_node("source", flow, source, provides=[spec("temp")]),
|
|
make_node(
|
|
"middle",
|
|
flow,
|
|
middle,
|
|
requires=[spec("temp")],
|
|
provides=[MessageSpec(name="warm", dtype=DType.BOOL)],
|
|
),
|
|
]
|
|
|
|
|
|
def test_a_stopped_flow_runs_nothing_and_its_neighbours_carry_on():
|
|
ran: list[str] = []
|
|
pipeline = Pipeline(
|
|
nodes=a_chain("stopped", ran) + a_chain("running", ran),
|
|
disabled_flows={"stopped"},
|
|
)
|
|
|
|
pipeline.run({})
|
|
|
|
assert not any(name.startswith("stopped.") for name in ran)
|
|
assert {"running.source", "running.middle"} <= set(ran)
|
|
|
|
|
|
def test_a_stopped_flow_ignores_a_trigger_from_its_own_nodes():
|
|
ran: list[str] = []
|
|
nodes = a_chain("stopped", ran)
|
|
pipeline = Pipeline(nodes=nodes, disabled_flows={"stopped"})
|
|
|
|
# What an MQTT subscription or a webhook would do.
|
|
nodes[0].inject({"temp": 20.0})
|
|
|
|
assert ran == []
|
|
assert "stopped.temp" not in pipeline.state
|
|
|
|
|
|
def test_a_paused_flow_still_takes_values_but_acts_on_none_of_them():
|
|
ran: list[str] = []
|
|
nodes = a_chain("demo", ran)
|
|
pipeline = Pipeline(nodes=nodes)
|
|
|
|
pipeline.pause("demo")
|
|
nodes[0].inject({"temp": 20.0})
|
|
|
|
# The value is there to look at; nothing downstream of it ran.
|
|
assert pipeline.state["demo.temp"] == 20.0
|
|
assert ran == []
|
|
|
|
|
|
def test_resuming_runs_what_was_held_back():
|
|
ran: list[str] = []
|
|
pipeline = Pipeline(nodes=a_chain("demo", ran))
|
|
|
|
pipeline.pause("demo")
|
|
pipeline.run({})
|
|
assert ran == []
|
|
|
|
pipeline.resume("demo")
|
|
assert ran == ["demo.source", "demo.middle"]
|
|
assert pipeline.paused_flows() == []
|
|
|
|
|
|
def test_stopped_survives_a_restart(tmp_path: Path):
|
|
store = FlowStore(tmp_path / "flows")
|
|
assert store.read_enabled("heating") is True
|
|
|
|
store.write_enabled("heating", False)
|
|
assert store.read_enabled("heating") is False
|
|
|
|
# A second store over the same directory is what a restart looks like.
|
|
assert FlowStore(tmp_path / "flows").read_enabled("heating") is False
|
|
|
|
|
|
class _StubController:
|
|
"""Only what the step route touches: the store, and the step itself."""
|
|
|
|
def __init__(self, store: FlowStore, stepped: str | None) -> None:
|
|
self.store = store
|
|
self.stepped = stepped
|
|
self.calls: list[str] = []
|
|
|
|
def step_flow(self, flow: str) -> str | None:
|
|
self.calls.append(flow)
|
|
return self.stepped
|
|
|
|
|
|
def _stub(tmp_path: Path, stepped: str | None) -> _StubController:
|
|
"""A controller over a store holding one flow, called 'heating'."""
|
|
store = FlowStore(tmp_path / "flows")
|
|
store.write_flow(FlowDef(name="heating"))
|
|
return _StubController(store, stepped)
|
|
|
|
|
|
def _client(controller: Any) -> TestClient:
|
|
app = FastAPI()
|
|
app.include_router(router, prefix="/api/v1")
|
|
app.state.flow_controller = controller
|
|
app.dependency_overrides[get_current_user] = lambda: None
|
|
return TestClient(app)
|
|
|
|
|
|
def test_the_step_endpoint_reports_what_it_ran(tmp_path: Path):
|
|
controller = _stub(tmp_path, "heating.sensor")
|
|
|
|
response = _client(controller).post("/api/v1/flows/heating/step")
|
|
|
|
assert response.status_code == 200
|
|
assert response.json() == {"message": "Stepped 'heating.sensor'"}
|
|
assert controller.calls == ["heating"]
|
|
|
|
|
|
def test_stepping_a_flow_with_nothing_parked_answers_plainly(tmp_path: Path):
|
|
"""A button pressed once too often is not an error."""
|
|
controller = _stub(tmp_path, None)
|
|
|
|
response = _client(controller).post("/api/v1/flows/heating/step")
|
|
|
|
assert response.status_code == 200
|
|
assert response.json() == {"message": "Nothing held back in flow 'heating'"}
|
|
|
|
|
|
def test_stepping_an_unknown_flow_is_a_404(tmp_path: Path):
|
|
controller = _stub(tmp_path, None)
|
|
|
|
response = _client(controller).post("/api/v1/flows/nosuchflow/step")
|
|
|
|
assert response.status_code == 404
|
|
assert controller.calls == []
|
|
|
|
|
|
def test_a_node_that_will_not_stop_does_not_wedge_the_rebuild(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
):
|
|
"""A node closing a connection nobody answers used to hold the lock forever.
|
|
|
|
Everything that deploys rebuilds, so the next publish, start or stop then
|
|
waited on a lock that was never given back.
|
|
"""
|
|
monkeypatch.setattr("fluksio.flow.controller.NODE_STOP_TIMEOUT", 0.05)
|
|
|
|
class NeverStops(Node):
|
|
async def stop(self, app: FastAPI | None = None) -> None:
|
|
await asyncio.Event().wait()
|
|
|
|
node = NeverStops(f=lambda params: None, name="stuck")
|
|
node.assign_flow("heating", "stuck")
|
|
engine = FlowController(FlowStore(tmp_path / "flows"))
|
|
engine.loaded = {
|
|
"heating.stuck": LoadedNode(id="heating.stuck", flow="heating", node=node)
|
|
}
|
|
|
|
async def scenario() -> None:
|
|
# The first rebuild gives up on the node it cannot stop, and the second
|
|
# is not left queueing behind a lock the first never released.
|
|
await engine.reload()
|
|
await engine.reload()
|
|
|
|
asyncio.run(asyncio.wait_for(scenario(), timeout=5))
|