Supervise the engine's host: deep health, loop watchdog, one worker
The API image ran four uvicorn workers, and each one built a full flow controller — four sets of MQTT subscriptions, cron ticks and webhooks. Runs one worker now; scaling out is the worker split, not more processes. Adds a loop-lag watchdog and a deep /utils/health/ that fails when the event loop is wedged or Redis is unreachable, the two failure modes a process-alive check never sees. Autoheal restarts on that signal, behind a compose profile because it mounts the Docker socket. The private user-seeding routes now need an explicit opt-in rather than just ENVIRONMENT=local, so a deployment that kept the default never exposes them. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011LF61rxW1FG5YCD2J9YqjY
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
"""Loop-lag watchdog: what the deep health check reads."""
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.flow.events import EventBus
|
||||
from app.flow.watchdog import DEGRADED_STRIKES, LoopWatchdog
|
||||
from app.main import app
|
||||
|
||||
|
||||
def test_a_responsive_loop_stays_healthy():
|
||||
watchdog = LoopWatchdog()
|
||||
|
||||
for _ in range(10):
|
||||
watchdog._record(0.002)
|
||||
|
||||
assert not watchdog.degraded
|
||||
assert watchdog.snapshot()["max_60s"] == 2.0
|
||||
|
||||
|
||||
def test_sustained_lag_marks_the_engine_degraded():
|
||||
watchdog = LoopWatchdog()
|
||||
|
||||
for _ in range(20):
|
||||
watchdog._record(1.0)
|
||||
|
||||
assert watchdog.degraded
|
||||
|
||||
|
||||
def test_a_blocked_loop_is_announced_once_it_keeps_happening():
|
||||
events = []
|
||||
bus = EventBus()
|
||||
bus.publish = events.append # type: ignore[method-assign]
|
||||
watchdog = LoopWatchdog(bus)
|
||||
|
||||
for _ in range(DEGRADED_STRIKES - 1):
|
||||
watchdog._record(6.0)
|
||||
assert events == []
|
||||
|
||||
watchdog._record(6.0)
|
||||
assert [e["type"] for e in events] == ["engine_degraded"]
|
||||
|
||||
# A recovered loop resets the count, so the next stall is announced again.
|
||||
watchdog._record(0.001)
|
||||
for _ in range(DEGRADED_STRIKES):
|
||||
watchdog._record(6.0)
|
||||
assert len(events) == 2
|
||||
|
||||
|
||||
def test_health_reports_503_once_the_loop_is_wedged():
|
||||
"""The point of the deep check: unhealthy without the process being dead."""
|
||||
watchdog = LoopWatchdog()
|
||||
app.state.watchdog = watchdog
|
||||
# No lifespan here, so there is no controller either — the endpoint has to
|
||||
# cope with a half-built app rather than assume the engine is up.
|
||||
client = TestClient(app)
|
||||
try:
|
||||
assert client.get("/api/v1/utils/health/").status_code == 200
|
||||
|
||||
for _ in range(20):
|
||||
watchdog._record(1.0)
|
||||
|
||||
response = client.get("/api/v1/utils/health/")
|
||||
assert response.status_code == 503
|
||||
body = response.json()
|
||||
assert body["status"] == "degraded"
|
||||
assert body["problems"] == ["event loop lagging"]
|
||||
finally:
|
||||
del app.state.watchdog
|
||||
Reference in New Issue
Block a user