Keep the engine's own history, and a screen that reads it

A second bus subscriber folds executions, errors, timings and queue lag
into per-minute rollups, keeps failures with their traceback and an audit
trail of who published what, and records one row per cascade — manual runs
and previews included, under an id of their own that writes no idempotency
markers. Read back through /observability/*, which always answers 200 so a
degraded engine still renders its own health screen.

Also fixes two things found on the way: node-health alerts read `status`
where the engine publishes `health`, so a device dropping never alerted
anyone, and the Redis queue reported `parked: 0` whatever was held.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt
This commit is contained in:
2026-08-16 22:29:32 +02:00
co-authored by Claude Fable 5
parent f300c43f3a
commit af3ba51571
30 changed files with 2610 additions and 22 deletions
+77 -2
View File
@@ -27,11 +27,48 @@ type FlowEvent =
source?: ValueSource
}
| { type: "node_started"; node: string }
| { type: "node_executed"; node: string; outputs: number }
| { type: "node_error"; node: string; error: string }
| {
type: "node_executed"
flow?: string
node: string
outputs: number
duration_ms?: number
}
| {
type: "node_error"
flow?: string
node: string
error: string
ts?: number
}
| { type: "node_status"; node: string; status: string; error?: string | null }
| ({ type: "node_log" } & LogLine)
| { type: "flow_paused"; flow: string; paused: boolean }
| {
type: "node_health"
flow?: string
node: string
health: "ok" | "down" | "unknown"
detail?: string | null
ts?: number
}
| { type: "flow_quarantined"; flow: string; error?: string; ts?: number }
| { type: "engine_degraded"; reason?: string; ts?: number }
| { type: "engine_fatal"; reason?: string; ts?: number }
| {
type: "cascade_dropped"
flow?: string
node?: string
deliveries?: number
ts?: number
}
| {
type: "queue_unavailable"
flow?: string
node?: string
error?: string
ts?: number
}
| {
type: "pipeline_rebuilt"
nodes: { id: string; status: string; error?: string | null }[]
@@ -110,6 +147,44 @@ export function useFlowSocket(onAuthFailure?: () => void): void {
status: "error",
error: message.error,
})
liveStore.recordEngineEvent({
type: message.type,
flow: message.flow,
node: message.node,
detail: message.error,
ts: message.ts ?? Date.now() / 1000,
})
break
case "node_health":
liveStore.setHealth(message.node, {
health: message.health,
detail: message.detail,
})
if (message.health === "down") {
liveStore.recordEngineEvent({
type: message.type,
flow: message.flow,
node: message.node,
detail: message.detail ?? "Reported itself down.",
ts: message.ts ?? Date.now() / 1000,
})
}
break
case "flow_quarantined":
case "engine_degraded":
case "engine_fatal":
case "cascade_dropped":
case "queue_unavailable":
liveStore.recordEngineEvent({
type: message.type,
flow: "flow" in message ? message.flow : undefined,
node: "node" in message ? message.node : undefined,
detail:
("error" in message ? message.error : undefined) ??
("reason" in message ? message.reason : undefined) ??
"",
ts: message.ts ?? Date.now() / 1000,
})
break
case "node_status":
liveStore.setStatus(message.node, {