Supervise the engine's host: deep health, loop watchdog, one worker

The API image ran four uvicorn workers, and each one built a full flow
controller — four sets of MQTT subscriptions, cron ticks and webhooks.
Runs one worker now; scaling out is the worker split, not more processes.

Adds a loop-lag watchdog and a deep /utils/health/ that fails when the
event loop is wedged or Redis is unreachable, the two failure modes a
process-alive check never sees. Autoheal restarts on that signal, behind
a compose profile because it mounts the Docker socket.

The private user-seeding routes now need an explicit opt-in rather than
just ENVIRONMENT=local, so a deployment that kept the default never
exposes them.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011LF61rxW1FG5YCD2J9YqjY

backend/tests/__init__.py

@@ -13,3 +13,6 @@ os.environ["POSTGRES_DB"] = "app_test"
 # builds a TestClient — and so a lifespan — per test module. Tests that want the
 # endpoint mount it themselves.
 os.environ["MCP_ENABLED"] = "false"
+# The private seeding endpoints are opt-in; the suite is one of the two places
+# (with the dev stack) where they are meant to work.
+os.environ["PRIVATE_API_ENABLED"] = "true"

backend/tests/api/routes/test_private.py

@@ -1,3 +1,4 @@
+import pytest
 from fastapi.testclient import TestClient
 from sqlmodel import Session, select
 
@@ -24,3 +25,22 @@ def test_create_user(client: TestClient, db: Session) -> None:
     assert user
     assert user.email == "pollo@listo.com"
     assert user.full_name == "Pollo Listo"
+
+
+def test_creating_a_user_is_refused_unless_the_private_api_is_enabled(
+    client: TestClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The route is always mounted so the SDK keeps its shape; the opt-in is
+    what decides whether unauthenticated user seeding actually works."""
+    monkeypatch.setattr(settings, "PRIVATE_API_ENABLED", False)
+
+    r = client.post(
+        f"{settings.API_V1_STR}/private/users/",
+        json={
+            "email": "nobody@listo.com",
+            "password": "password123",
+            "full_name": "Nobody",
+        },
+    )
+
+    assert r.status_code == 403

backend/tests/flow/test_watchdog.py

@@ -0,0 +1,68 @@
+"""Loop-lag watchdog: what the deep health check reads."""
+
+from fastapi.testclient import TestClient
+
+from app.flow.events import EventBus
+from app.flow.watchdog import DEGRADED_STRIKES, LoopWatchdog
+from app.main import app
+
+
+def test_a_responsive_loop_stays_healthy():
+    watchdog = LoopWatchdog()
+
+    for _ in range(10):
+        watchdog._record(0.002)
+
+    assert not watchdog.degraded
+    assert watchdog.snapshot()["max_60s"] == 2.0
+
+
+def test_sustained_lag_marks_the_engine_degraded():
+    watchdog = LoopWatchdog()
+
+    for _ in range(20):
+        watchdog._record(1.0)
+
+    assert watchdog.degraded
+
+
+def test_a_blocked_loop_is_announced_once_it_keeps_happening():
+    events = []
+    bus = EventBus()
+    bus.publish = events.append  # type: ignore[method-assign]
+    watchdog = LoopWatchdog(bus)
+
+    for _ in range(DEGRADED_STRIKES - 1):
+        watchdog._record(6.0)
+    assert events == []
+
+    watchdog._record(6.0)
+    assert [e["type"] for e in events] == ["engine_degraded"]
+
+    # A recovered loop resets the count, so the next stall is announced again.
+    watchdog._record(0.001)
+    for _ in range(DEGRADED_STRIKES):
+        watchdog._record(6.0)
+    assert len(events) == 2
+
+
+def test_health_reports_503_once_the_loop_is_wedged():
+    """The point of the deep check: unhealthy without the process being dead."""
+    watchdog = LoopWatchdog()
+    app.state.watchdog = watchdog
+    # No lifespan here, so there is no controller either — the endpoint has to
+    # cope with a half-built app rather than assume the engine is up.
+    client = TestClient(app)
+    try:
+        assert client.get("/api/v1/utils/health/").status_code == 200
+
+        for _ in range(20):
+            watchdog._record(1.0)
+
+        response = client.get("/api/v1/utils/health/")
+        assert response.status_code == 503
+        body = response.json()
+        assert body["status"] == "degraded"
+        assert body["problems"] == ["event loop lagging"]
+    finally:
+        del app.state.watchdog