Supervise the engine's host: deep health, loop watchdog, one worker

The API image ran four uvicorn workers, and each one built a full flow
controller — four sets of MQTT subscriptions, cron ticks and webhooks.
Runs one worker now; scaling out is the worker split, not more processes.

Adds a loop-lag watchdog and a deep /utils/health/ that fails when the
event loop is wedged or Redis is unreachable, the two failure modes a
process-alive check never sees. Autoheal restarts on that signal, behind
a compose profile because it mounts the Docker socket.

The private user-seeding routes now need an explicit opt-in rather than
just ENVIRONMENT=local, so a deployment that kept the default never
exposes them.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011LF61rxW1FG5YCD2J9YqjY

backend/app/api/main.py

@@ -1,7 +1,6 @@
 from fastapi import APIRouter
 
 from app.api.routes import flows, login, oauth, private, secrets, users, utils
-from app.core.config import settings
 
 api_router = APIRouter()
 api_router.include_router(login.router)
@@ -15,5 +14,6 @@ api_router.include_router(secrets.router)
 api_router.include_router(oauth.router)
 
 
-if settings.ENVIRONMENT == "local":
-    api_router.include_router(private.router)
+# Always mounted like oauth, so the generated SDK keeps its shape; the
+# endpoints refuse to work unless the private API is explicitly enabled.
+api_router.include_router(private.router)

backend/app/api/routes/private.py

@@ -1,9 +1,10 @@
 from typing import Any
 
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 
 from app.api.deps import SessionDep
+from app.core.config import settings
 from app.core.security import get_password_hash
 from app.models import (
     User,
@@ -13,6 +14,11 @@ from app.models import (
 router = APIRouter(tags=["private"], prefix="/private")
 
 
+def _require_private_api() -> None:
+    if not (settings.ENVIRONMENT == "local" and settings.PRIVATE_API_ENABLED):
+        raise HTTPException(status_code=403, detail="Private API is disabled")
+
+
 class PrivateUserCreate(BaseModel):
     email: str
     password: str
@@ -25,6 +31,7 @@ def create_user(user_in: PrivateUserCreate, session: SessionDep) -> Any:
     """
     Create a new user.
     """
+    _require_private_api()
 
     user = User(
         email=user_in.email,

backend/app/api/routes/utils.py

@@ -1,7 +1,12 @@
-from fastapi import APIRouter, Depends
+import time
+from typing import Any
+
+from fastapi import APIRouter, Depends, Request, Response
+from fastapi.concurrency import run_in_threadpool
 from pydantic.networks import EmailStr
 
 from app.api.deps import get_current_active_superuser
+from app.flow.state import RedisState
 from app.models import Message
 from app.utils import generate_test_email, send_email
 
@@ -29,3 +34,61 @@ def test_email(email_to: EmailStr) -> Message:
 @router.get("/health-check/")
 async def health_check() -> bool:
     return True
+
+
+@router.get("/health/")
+async def health(request: Request, response: Response) -> dict[str, Any]:
+    """Deep health: 200 while the engine can serve its purpose, 503 otherwise.
+
+    "Serve its purpose" means the event loop is responsive and the configured
+    state backend answers — the two failure modes a process-alive check never
+    sees. The queue and pool sections are filled by the execution service.
+    """
+    controller = getattr(request.app.state, "flow_controller", None)
+    watchdog = getattr(request.app.state, "watchdog", None)
+    problems: list[str] = []
+
+    loop_lag = watchdog.snapshot() if watchdog else {"ewma": 0.0, "max_60s": 0.0}
+    if watchdog is not None and watchdog.degraded:
+        problems.append("event loop lagging")
+
+    redis_info: dict[str, Any] = {
+        "configured": False,
+        "connected": None,
+        "rtt_ms": None,
+    }
+    engine: dict[str, Any] = {"flows": 0, "nodes": 0, "quarantined": []}
+    queue: dict[str, Any] = {}
+    if controller is not None:
+        state = controller.state
+        if isinstance(state, RedisState):
+            redis_info["configured"] = True
+            start = time.perf_counter()
+            connected = await run_in_threadpool(state.ping)
+            redis_info["connected"] = connected
+            redis_info["rtt_ms"] = round((time.perf_counter() - start) * 1000, 1)
+            if not connected:
+                problems.append("redis unreachable")
+        engine["flows"] = len(getattr(controller, "loaded", {}) or {})
+        engine["nodes"] = len(
+            getattr(getattr(controller, "pipeline", None), "nodes", []) or []
+        )
+        engine["quarantined"] = sorted(getattr(controller, "quarantined", ()) or ())
+        stats = getattr(controller, "queue_stats", None)
+        if callable(stats):
+            queue = await run_in_threadpool(stats)
+            if queue.get("oldest_pending_s", 0) > 120:
+                problems.append("queue stalled")
+
+    status = "degraded" if problems else "ok"
+    if problems:
+        response.status_code = 503
+    return {
+        "status": status,
+        "problems": problems,
+        "loop_lag_ms": loop_lag,
+        "redis": redis_info,
+        "engine": engine,
+        "queue": queue,
+        "ts": time.time(),
+    }