Hold a node down for a minute before calling it down
A connector that misses one poll turned the health badge amber and the node on the canvas red until it recovered. `LoadedNode.health_since` marks when the status last actually changed — a rotating error detail under an unchanged status does not move it, or a connector retrying with a different errno each poll would never debounce — and `unhealthy_nodes` applies a 60 s floor to it. The floor is on the surfacing, not on the transition: the stored health is the truth the moment a node reports it, so a flow reading node health is never told a stale story. Only going down is held back; recovery clears at once. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CL9zvnnvcp1mvA8o7impxk
This commit is contained in:
@@ -5,8 +5,10 @@ and no screen ever asked. These cover the derivation that closes that gap.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from fluksio.flow.controller import FlowController, LoadedNode
|
||||
from fluksio.flow.controller import HEALTH_FLOOR, FlowController, LoadedNode
|
||||
from fluksio.flow.store import FlowStore
|
||||
|
||||
|
||||
@@ -40,3 +42,62 @@ def test_the_issue_clears_when_the_node_reports_itself_well(tmp_path: Path) -> N
|
||||
controller.loaded["house.owm"].health = "ok"
|
||||
|
||||
assert controller.flow_issues("house") == []
|
||||
|
||||
|
||||
def _reporting_node(node_id: str, flow: str) -> Any:
|
||||
"""The bit of a node `_health_changed` reads: which node, on which flow."""
|
||||
return SimpleNamespace(id=node_id, flow=flow)
|
||||
|
||||
|
||||
def a_running_controller(tmp_path: Path) -> tuple[FlowController, Any]:
|
||||
"""A controller holding one healthy node, plus its health reporter."""
|
||||
controller = FlowController(FlowStore(tmp_path / "flows"))
|
||||
controller.loaded["house.owm"] = LoadedNode(id="house.owm", flow="house")
|
||||
return controller, _reporting_node("house.owm", "house")
|
||||
|
||||
|
||||
def test_a_node_that_drops_one_poll_never_surfaces(tmp_path: Path) -> None:
|
||||
controller, node = a_running_controller(tmp_path)
|
||||
|
||||
controller._health_changed(node, "down", "TimeoutError: no reply")
|
||||
|
||||
# Stored truthfully — a flow reading node health sees it — but held back
|
||||
# from the screens until the floor passes.
|
||||
assert controller.loaded["house.owm"].health == "down"
|
||||
assert controller.flow_issues("house") == []
|
||||
|
||||
controller._health_changed(node, "ok", None)
|
||||
assert controller.flow_issues("house") == []
|
||||
|
||||
|
||||
def test_a_node_still_down_past_the_floor_surfaces(tmp_path: Path) -> None:
|
||||
controller, node = a_running_controller(tmp_path)
|
||||
|
||||
controller._health_changed(node, "down", "TimeoutError: no reply")
|
||||
controller.loaded["house.owm"].health_since -= HEALTH_FLOOR
|
||||
|
||||
assert [issue.code for issue in controller.flow_issues("house")] == [
|
||||
"node_unhealthy"
|
||||
]
|
||||
|
||||
# Recovery is not debounced: the floor is only on the way down.
|
||||
controller._health_changed(node, "ok", None)
|
||||
assert controller.flow_issues("house") == []
|
||||
|
||||
|
||||
def test_a_rotating_error_detail_does_not_restart_the_floor(tmp_path: Path) -> None:
|
||||
"""The case that would quietly disable the debounce for real connectors.
|
||||
|
||||
A connector retrying a dead device reports a different errno each poll. If
|
||||
the detail moved the mark, the node would never reach the floor and would
|
||||
never be reported down at all.
|
||||
"""
|
||||
controller, node = a_running_controller(tmp_path)
|
||||
|
||||
controller._health_changed(node, "down", "OSError: [Errno 113] no route")
|
||||
controller.loaded["house.owm"].health_since -= HEALTH_FLOOR
|
||||
controller._health_changed(node, "down", "OSError: [Errno 110] timed out")
|
||||
|
||||
issues = controller.flow_issues("house")
|
||||
assert [issue.code for issue in issues] == ["node_unhealthy"]
|
||||
assert "Errno 110" in issues[0].message
|
||||
|
||||
Reference in New Issue
Block a user