Keep the engine's own history, and a screen that reads it

A second bus subscriber folds executions, errors, timings and queue lag
into per-minute rollups, keeps failures with their traceback and an audit
trail of who published what, and records one row per cascade — manual runs
and previews included, under an id of their own that writes no idempotency
markers. Read back through /observability/*, which always answers 200 so a
degraded engine still renders its own health screen.

Also fixes two things found on the way: node-health alerts read `status`
where the engine publishes `health`, so a device dropping never alerted
anyone, and the Redis queue reported `parked: 0` whatever was held.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt

backend/app/api/main.py

@@ -8,6 +8,7 @@ from app.api.routes import (
     messages,
     modules,
     oauth,
+    observability,
     private,
     secrets,
     users,
@@ -25,6 +26,7 @@ api_router.include_router(alerts.router)
 api_router.include_router(dashboards.router)
 api_router.include_router(messages.router)
 api_router.include_router(modules.router)
+api_router.include_router(observability.router)
 # Always mounted so the generated SDK stays the same shape; the endpoints
 # themselves refuse to work unless MCP is switched on.
 api_router.include_router(oauth.router)

backend/app/api/routes/flows.py

@@ -1,6 +1,7 @@
 """The flow API. Everything the editor can do is available here first."""
 
 import asyncio
+import time
 from typing import Any
 
 from fastapi import (
@@ -14,7 +15,12 @@ from fastapi.concurrency import run_in_threadpool
 from pydantic import BaseModel
 from sqlmodel import Session
 
-from app.api.deps import FlowControllerDep, get_current_user, user_from_token
+from app.api.deps import (
+    CurrentUser,
+    FlowControllerDep,
+    get_current_user,
+    user_from_token,
+)
 from app.core.db import engine
 from app.flow.controller import FlowController
 from app.flow.dashboards import DashboardStore
@@ -184,6 +190,19 @@ def _read_flow(controller: FlowController, name: str) -> FlowDef:
         raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
 
 
+def _audit(action: str, flow: str, user: CurrentUser) -> None:
+    """Record who changed what. The collector writes it down; the bus carries it."""
+    event_bus.publish(
+        {
+            "type": "audit",
+            "action": action,
+            "flow": flow,
+            "user": user.email,
+            "ts": time.time(),
+        }
+    )
+
+
 def _source_ref(definition: FlowDef, node_id: str) -> str | None:
     """The library source this node runs, if it is a shared one."""
     node = next((n for n in definition.nodes if n.id == node_id), None)
@@ -323,6 +342,7 @@ async def publish_flow(
     name: str,
     body: PublishRequest,
     controller: FlowControllerDep,
+    user: CurrentUser,
 ) -> Any:
     """Deploy the unpublished changes: the engine picks them up from here."""
     _read_flow(controller, name)
@@ -339,6 +359,7 @@ async def publish_flow(
             status_code=409,
             detail={"message": str(exc), "current_version": exc.current},
         )
+    _audit("published", name, user)
     await controller.reload()
     return _detail(controller, published)
 
@@ -364,12 +385,15 @@ async def discard_draft(name: str, controller: FlowControllerDep) -> Any:
 
 
 @router.delete("/{name}", response_model=Message)
-async def delete_flow(name: str, controller: FlowControllerDep) -> Any:
+async def delete_flow(
+    name: str, controller: FlowControllerDep, user: CurrentUser
+) -> Any:
     """Delete a flow and everything in it."""
     try:
         await run_in_threadpool(controller.store.delete_flow, name)
     except FlowNotFound:
         raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
+    _audit("deleted", name, user)
     # Its files are gone; its values and queued work would otherwise linger.
     await run_in_threadpool(controller.forget_flow, name)
     await controller.reload()
@@ -510,18 +534,22 @@ async def unshare_node(
 
 
 @router.post("/{name}/start", response_model=FlowDetail)
-async def start_flow(name: str, controller: FlowControllerDep) -> Any:
+async def start_flow(
+    name: str, controller: FlowControllerDep, user: CurrentUser
+) -> Any:
     """Let the engine run this flow again."""
     _read_flow(controller, name)
     await controller.set_enabled(name, True)
+    _audit("started", name, user)
     return _detail(controller, _read_flow(controller, name))
 
 
 @router.post("/{name}/stop", response_model=FlowDetail)
-async def stop_flow(name: str, controller: FlowControllerDep) -> Any:
+async def stop_flow(name: str, controller: FlowControllerDep, user: CurrentUser) -> Any:
     """Take this flow off the engine: no subscriptions, schedules or webhooks."""
     _read_flow(controller, name)
     await controller.set_enabled(name, False)
+    _audit("stopped", name, user)
     return _detail(controller, _read_flow(controller, name))
 
 

backend/app/api/routes/modules.py

@@ -6,13 +6,20 @@ the worker pool retires its processes, and the next node call picks up the new
 packages.
 """
 
+import time
 from typing import Any
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.concurrency import run_in_threadpool
 
-from app.api.deps import FlowControllerDep, WorkerPoolDep, get_current_user
+from app.api.deps import (
+    CurrentUser,
+    FlowControllerDep,
+    WorkerPoolDep,
+    get_current_user,
+)
 from app.flow import modules
+from app.flow.events import event_bus
 from app.flow.schemas import ApplyRequest, ApplyResult, ModulesInfo
 
 router = APIRouter(
@@ -28,7 +35,10 @@ async def read_modules(controller: FlowControllerDep) -> Any:
 
 @router.post("/apply", response_model=ApplyResult)
 async def apply_modules(
-    body: ApplyRequest, controller: FlowControllerDep, pool: WorkerPoolDep
+    body: ApplyRequest,
+    controller: FlowControllerDep,
+    pool: WorkerPoolDep,
+    user: CurrentUser,
 ) -> Any:
     """Install exactly these requirements, then hand them to the workers.
 
@@ -42,6 +52,15 @@ async def apply_modules(
             detail=output or "These requirements could not be installed",
         )
     await run_in_threadpool(controller.store.write_requirements, body.requirements)
+    event_bus.publish(
+        {
+            "type": "audit",
+            "action": "installed modules",
+            "flow": "",
+            "user": user.email,
+            "ts": time.time(),
+        }
+    )
     # Retire the workers first, so the rebuild compiles every node against the
     # packages that were just installed — a node that could not import one is
     # the reason this was called, and it stays red until it is built again.

backend/app/api/routes/observability.py

@@ -0,0 +1,310 @@
+"""What the engine has been doing: health now, and history since.
+
+The live state comes from the controller; everything older than the websocket's
+memory comes from the rollups the metrics collector writes. Deliberately not
+built on ``/utils/health/``: that endpoint answers 503 when something is wrong,
+which the generated SDK turns into a thrown error — and a health page that
+cannot render while the engine is degraded is the wrong way round.
+"""
+
+from datetime import datetime, timedelta, timezone
+from typing import Any, Literal
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.concurrency import run_in_threadpool
+from pydantic import BaseModel
+from sqlalchemy import func
+from sqlmodel import col, select
+
+from app.api.deps import FlowControllerDep, SessionDep, get_current_user
+from app.flow.controller import NodeStatus
+from app.models import EngineEvent, FlowRun, MetricBucket
+
+router = APIRouter(
+    prefix="/observability",
+    tags=["observability"],
+    dependencies=[Depends(get_current_user)],
+)
+
+#: How many slices a per-flow sparkline is folded into.
+SPARK_SLICES = 60
+
+
+class HealthSummary(BaseModel):
+    status: str
+    problems: list[str]
+    flows: dict[str, int]
+    nodes: dict[str, int]
+    queue: dict[str, Any]
+    loop_lag: dict[str, float]
+    failures_24h: int
+
+
+class SeriesPoint(BaseModel):
+    ts: float
+    executions: int
+    errors: int
+    messages: int
+    avg_ms: float
+    max_ms: float
+    avg_lag_ms: float
+
+
+class FlowRollup(BaseModel):
+    flow: str
+    executions: int
+    errors: int
+    messages: int
+    avg_ms: float
+    avg_lag_ms: float
+    spark: list[int]
+    last_error_ts: float | None = None
+
+
+class RunRow(BaseModel):
+    id: str
+    flow: str
+    source: str
+    status: str
+    started_at: datetime
+    finished_at: datetime | None = None
+    nodes: int
+    errors: int
+    duration_ms: float
+    deliveries: int
+
+
+class EventRow(BaseModel):
+    id: int
+    ts: datetime
+    type: str
+    flow: str
+    node: str
+    detail: str
+    actor: str
+
+
+class DeadLetter(BaseModel):
+    id: str
+    ts: float
+    flow: str
+    node: str
+    cause: str
+    reason: str
+
+
+def _since(hours: int) -> datetime:
+    return datetime.now(timezone.utc) - timedelta(hours=hours)
+
+
+@router.get("/summary", response_model=HealthSummary)
+async def read_summary(
+    request: Request, controller: FlowControllerDep, session: SessionDep
+) -> Any:
+    """How the engine is doing right now. Always 200, degraded or not."""
+    watchdog = getattr(request.app.state, "watchdog", None)
+    problems: list[str] = []
+    if watchdog is not None and watchdog.degraded:
+        problems.append("event loop lagging")
+
+    queue = await run_in_threadpool(controller.queue_stats)
+    if queue.get("oldest_pending_s", 0) > 120:
+        problems.append("queue stalled")
+    if queue.get("error"):
+        problems.append("work queue unreachable")
+
+    names = await run_in_threadpool(controller.store.list_flows)
+    quarantined = controller.quarantined
+    paused = set(controller.paused_flows())
+    entries = list(controller.loaded.values())
+    errored = [e for e in entries if e.status is NodeStatus.ERROR]
+    if quarantined:
+        problems.append(f"{len(quarantined)} flow(s) quarantined")
+    if errored:
+        problems.append(f"{len(errored)} node(s) failed to load")
+
+    failures = session.exec(
+        select(func.count())
+        .select_from(EngineEvent)
+        .where(col(EngineEvent.ts) >= _since(24), col(EngineEvent.type) != "audit")
+    ).one()
+
+    return HealthSummary(
+        status="degraded" if problems else "ok",
+        problems=problems,
+        flows={
+            "total": len(names),
+            "running": len(
+                [n for n in names if controller.is_enabled(n) and n not in quarantined]
+            ),
+            "paused": len(paused),
+            "quarantined": len(quarantined),
+        },
+        nodes={"total": len(entries), "error": len(errored)},
+        queue=queue,
+        loop_lag=(
+            watchdog.snapshot()
+            if watchdog is not None
+            else {"ewma": 0.0, "max_60s": 0.0}
+        ),
+        failures_24h=int(failures),
+    )
+
+
+@router.get("/timeseries", response_model=list[SeriesPoint])
+def read_timeseries(
+    session: SessionDep,
+    flow: str | None = None,
+    node: str | None = None,
+    hours: int = 24,
+    bucket_s: int = 60,
+) -> Any:
+    """Executions, errors and timings over time, summed across nodes."""
+    # ponytail: the fold is in Python — a day is at most 1440 rows per node.
+    # date_bin() if the window ever grows past that.
+    statement = select(MetricBucket).where(col(MetricBucket.bucket) >= _since(hours))
+    if flow:
+        statement = statement.where(col(MetricBucket.flow) == flow)
+    if node:
+        statement = statement.where(col(MetricBucket.node) == node)
+
+    slices: dict[float, dict[str, float]] = {}
+    for row in session.exec(statement.order_by(col(MetricBucket.bucket))):
+        stamp = row.bucket.timestamp()
+        key = stamp - stamp % max(60, bucket_s)
+        point = slices.setdefault(
+            key,
+            {
+                "executions": 0.0,
+                "errors": 0.0,
+                "messages": 0.0,
+                "duration_sum_ms": 0.0,
+                "max_ms": 0.0,
+                "lag_sum_ms": 0.0,
+                "items": 0.0,
+            },
+        )
+        point["executions"] += row.executions
+        point["errors"] += row.errors
+        point["messages"] += row.messages
+        point["duration_sum_ms"] += row.duration_sum_ms
+        point["max_ms"] = max(point["max_ms"], row.duration_max_ms)
+        point["lag_sum_ms"] += row.lag_sum_ms
+        point["items"] += row.items
+
+    return [
+        SeriesPoint(
+            ts=ts,
+            executions=int(point["executions"]),
+            errors=int(point["errors"]),
+            messages=int(point["messages"]),
+            avg_ms=round(point["duration_sum_ms"] / (point["executions"] or 1), 2),
+            max_ms=round(point["max_ms"], 2),
+            avg_lag_ms=round(point["lag_sum_ms"] / (point["items"] or 1), 2),
+        )
+        for ts, point in sorted(slices.items())
+    ]
+
+
+@router.get("/flows", response_model=list[FlowRollup])
+def read_flow_rollups(session: SessionDep, hours: int = 24) -> Any:
+    """One row per flow, with a coarse trend of how much it ran."""
+    since = _since(hours)
+    window = hours * 3600
+    start = since.timestamp()
+
+    rollups: dict[str, dict[str, Any]] = {}
+    for row in session.exec(
+        select(MetricBucket).where(col(MetricBucket.bucket) >= since)
+    ):
+        entry = rollups.setdefault(
+            row.flow,
+            {
+                "executions": 0,
+                "errors": 0,
+                "messages": 0,
+                "duration_sum_ms": 0.0,
+                "lag_sum_ms": 0.0,
+                "items": 0,
+                "spark": [0] * SPARK_SLICES,
+            },
+        )
+        entry["executions"] += row.executions
+        entry["errors"] += row.errors
+        entry["messages"] += row.messages
+        entry["duration_sum_ms"] += row.duration_sum_ms
+        entry["lag_sum_ms"] += row.lag_sum_ms
+        entry["items"] += row.items
+        slot = min(
+            SPARK_SLICES - 1,
+            max(0, int((row.bucket.timestamp() - start) / window * SPARK_SLICES)),
+        )
+        entry["spark"][slot] += row.executions
+
+    # `.all()` first: a Result has `keys()`, so dict() would read it as a
+    # mapping and subscript it.
+    last_errors = dict(
+        session.exec(
+            select(col(EngineEvent.flow), func.max(col(EngineEvent.ts)))
+            .where(col(EngineEvent.type) == "node_error", col(EngineEvent.ts) >= since)
+            .group_by(col(EngineEvent.flow))
+        ).all()
+    )
+
+    return [
+        FlowRollup(
+            flow=flow,
+            executions=entry["executions"],
+            errors=entry["errors"],
+            messages=entry["messages"],
+            avg_ms=round(entry["duration_sum_ms"] / (entry["executions"] or 1), 2),
+            avg_lag_ms=round(entry["lag_sum_ms"] / (entry["items"] or 1), 2),
+            spark=entry["spark"],
+            last_error_ts=(
+                last_errors[flow].timestamp() if flow in last_errors else None
+            ),
+        )
+        for flow, entry in sorted(rollups.items())
+    ]
+
+
+@router.get("/runs", response_model=list[RunRow])
+def read_runs(
+    session: SessionDep,
+    flow: str | None = None,
+    status: str | None = None,
+    limit: int = 50,
+) -> Any:
+    """Recent cascades, newest first."""
+    statement = select(FlowRun).order_by(col(FlowRun.started_at).desc())
+    if flow:
+        statement = statement.where(col(FlowRun.flow) == flow)
+    if status:
+        statement = statement.where(col(FlowRun.status) == status)
+    return list(session.exec(statement.limit(min(limit, 200))))
+
+
+@router.get("/events", response_model=list[EventRow])
+def read_events(
+    session: SessionDep,
+    kind: Literal["failure", "audit"] = "failure",
+    flow: str | None = None,
+    limit: int = 100,
+) -> Any:
+    """What went wrong, or who changed what. Newest first."""
+    statement = select(EngineEvent).order_by(col(EngineEvent.ts).desc())
+    if kind == "audit":
+        statement = statement.where(col(EngineEvent.type) == "audit")
+    else:
+        statement = statement.where(col(EngineEvent.type) != "audit")
+    if flow:
+        statement = statement.where(col(EngineEvent.flow) == flow)
+    return list(session.exec(statement.limit(min(limit, 500))))
+
+
+@router.get("/dead-letter", response_model=list[DeadLetter])
+async def read_dead_letters(controller: FlowControllerDep, limit: int = 50) -> Any:
+    """Work the engine gave up on, which nothing else surfaces."""
+    if controller.execution is None:
+        return []
+    return await run_in_threadpool(controller.execution.queue.dead_letters, limit)