Keep the engine's own history, and a screen that reads it

A second bus subscriber folds executions, errors, timings and queue lag
into per-minute rollups, keeps failures with their traceback and an audit
trail of who published what, and records one row per cascade — manual runs
and previews included, under an id of their own that writes no idempotency
markers. Read back through /observability/*, which always answers 200 so a
degraded engine still renders its own health screen.

Also fixes two things found on the way: node-health alerts read `status`
where the engine publishes `health`, so a device dropping never alerted
anyone, and the Redis queue reported `parked: 0` whatever was held.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt

frontend/src/components/Common/UplotChart.tsx

@@ -0,0 +1,163 @@
+import { useEffect, useLayoutEffect, useRef } from "react"
+import uPlot from "uplot"
+import "uplot/dist/uPlot.min.css"
+
+import type { HistoryPoint } from "@/client"
+import { useTheme } from "@/components/theme-provider"
+
+/**
+ * How many lines one chart carries.
+ *
+ * The bound is the palette's: `--chart-1…5` is one designed ramp, and a sixth
+ * line would either repeat a step or invent a colour outside the system.
+ */
+export const MAX_SERIES = 5
+
+/** Room for the axis ticks; uPlot measures the rest of the box itself. */
+const PADDING: uPlot.Padding = [10, 12, 0, 0]
+
+/** The legend sits under the canvas, so the canvas has to leave it room. */
+const LEGEND_HEIGHT = 26
+
+const canvasHeight = (element: HTMLElement) =>
+  Math.max(60, (element.clientHeight || 180) - LEGEND_HEIGHT)
+
+/** A token, resolved for the canvas — which cannot read CSS variables. */
+function token(name: string): string {
+  return getComputedStyle(document.documentElement)
+    .getPropertyValue(name)
+    .trim()
+}
+
+const seriesColor = (index: number) => token(`--chart-${(index % 5) + 1}`)
+
+/** The series joined onto one x axis, which is what uPlot draws. */
+function table(plots: HistoryPoint[][]): uPlot.AlignedData {
+  return uPlot.join(
+    plots.map(
+      (plot) =>
+        [
+          plot.map((point) => point.ts),
+          plot.map((point) => point.value),
+        ] as uPlot.AlignedData,
+    ),
+  )
+}
+
+/**
+ * Several series over time, drawn on one axis.
+ *
+ * uPlot rather than SVG: a chart may hold five series of hundreds of points
+ * each, which is more path data than React should be rebuilding on every value
+ * that arrives. Its own legend doubles as the hover readout, so the cursor
+ * tells you what each line was worth at that moment — and with more than one
+ * line a legend is required anyway.
+ */
+export function UplotChart({
+  labels,
+  plots,
+  empty = "Nothing has come through yet.",
+}: {
+  /** One label per series; the set of them is the chart's identity. */
+  labels: string[]
+  /** The points of each series, in the same order as `labels`. */
+  plots: HistoryPoint[][]
+  empty?: string
+}) {
+  const host = useRef<HTMLDivElement>(null)
+  const chart = useRef<uPlot | null>(null)
+  const { resolvedTheme } = useTheme()
+
+  const points = plots.reduce((total, plot) => total + plot.length, 0)
+  // The identity of the series set: the chart is rebuilt when it changes,
+  // while a new reading only sets its data.
+  const key = labels.join(" ")
+  // uPlot leaves its axes half-initialised while the scales have no range, and
+  // a resize in that window (a card still settling, say) draws them anyway and
+  // throws. Waiting for the first reading avoids the state altogether.
+  const ready = points > 0
+
+  // biome-ignore lint/correctness/useExhaustiveDependencies: the label string is the identity of the series set.
+  useLayoutEffect(() => {
+    const element = host.current
+    if (!element || labels.length === 0 || !ready) return
+
+    const axis = {
+      stroke: () => token("--muted-foreground"),
+      grid: { stroke: () => token("--border"), width: 1 },
+      ticks: { stroke: () => token("--border"), width: 1 },
+      font: `11px ${getComputedStyle(element).fontFamily}`,
+    }
+
+    const plot = new uPlot(
+      {
+        width: element.clientWidth || 320,
+        height: canvasHeight(element),
+        padding: PADDING,
+        cursor: { y: false },
+        legend: { live: true },
+        scales: { x: { time: true } },
+        axes: [
+          { ...axis, size: 28 },
+          { ...axis, size: 46 },
+        ],
+        series: [
+          {},
+          ...labels.map((label, index) => ({
+            label,
+            width: 2,
+            // Read at draw time, so a theme toggle is a redraw rather than a
+            // rebuilt chart.
+            stroke: () => seriesColor(index),
+            // Series arrive on their own clocks; a joined table is mostly
+            // holes, and a line with a hole per point is not a line.
+            spanGaps: true,
+            points: { show: false },
+          })),
+        ],
+      },
+      // Built with the readings it already has: uPlot's axes are only half
+      // initialised while its scales have no range.
+      table(plots),
+      element,
+    )
+    chart.current = plot
+
+    const observer = new ResizeObserver(() => {
+      plot.setSize({
+        width: element.clientWidth,
+        height: canvasHeight(element),
+      })
+    })
+    observer.observe(element)
+
+    return () => {
+      observer.disconnect()
+      plot.destroy()
+      chart.current = null
+    }
+  }, [key, ready])
+
+  // biome-ignore lint/correctness/useExhaustiveDependencies: rebuilding the joined table is what the point count stands for.
+  useEffect(() => {
+    if (!chart.current || plots.length === 0) return
+    chart.current.setData(table(plots))
+  }, [points, key])
+
+  // The canvas cannot follow a CSS variable, so a theme swap is a redraw.
+  // biome-ignore lint/correctness/useExhaustiveDependencies: the theme is the signal, not something the effect reads.
+  useEffect(() => {
+    chart.current?.redraw()
+  }, [resolvedTheme])
+
+  return (
+    <div className="relative min-h-0 flex-1">
+      <div ref={host} className="absolute inset-0" />
+      {points === 0 ? (
+        <p className="absolute inset-0 flex items-center justify-center text-sm text-muted-foreground">
+          {empty}
+        </p>
+      ) : null}
+    </div>
+  )
+}

frontend/src/components/Flow/MessageSparkline.tsx

@@ -49,7 +49,7 @@ function useSettled(value: string): string {
  * that never moved has no span to divide by, and one spanning decades is only
  * legible once the exponent is what varies.
  */
-function shape(points: HistoryPoint[]) {
+export function shape(points: HistoryPoint[]) {
   const values = points.map((point) => point.value)
   const low = Math.min(...values)
   const high = Math.max(...values)

frontend/src/components/Flow/liveStore.ts

@@ -24,6 +24,19 @@ export type LiveStatus = {
   status: "active" | "error" | "running" | "success"
   error?: string | null
 }
+/** How a node's connection is doing, which is not how its last run went. */
+export type NodeHealth = {
+  health: "ok" | "down" | "unknown"
+  detail?: string | null
+}
+/** Something the engine reported about itself, for the health page. */
+export type EngineEvent = {
+  type: string
+  flow?: string
+  node?: string
+  detail?: string
+  ts: number
+}
 /** One node execution's output, as the log panel shows it. */
 export type LogLine = {
   flow: string
@@ -38,9 +51,13 @@ type Listener = () => void
 
 /** Enough to see what a flow has been doing, not a log store. */
 const LOG_LIMIT = 500
+/** The health page reads these to know when to refetch; it is not a history. */
+const ENGINE_EVENT_LIMIT = 100
 
 const values = new Map<string, LiveValue>()
 const statuses = new Map<string, LiveStatus>()
+const health = new Map<string, NodeHealth>()
+let engineEvents: EngineEvent[] = []
 // How many times a node has emitted. The number itself means nothing; a change
 // is what restarts the pulse.
 const emits = new Map<string, number>()
@@ -100,6 +117,18 @@ export const liveStore = {
   getStatus(nodeId: string) {
     return statuses.get(nodeId)
   },
+  setHealth(nodeId: string, entry: NodeHealth) {
+    health.set(nodeId, entry)
+    notify(`health:${nodeId}`)
+  },
+  getHealth(nodeId: string) {
+    return health.get(nodeId)
+  },
+  recordEngineEvent(event: EngineEvent) {
+    // A new array each time, so the hook's snapshot comparison sees the change.
+    engineEvents = [...engineEvents, event].slice(-ENGINE_EVENT_LIMIT)
+    notify("engine")
+  },
   recordEmit(nodeId: string) {
     emits.set(nodeId, (emits.get(nodeId) ?? 0) + 1)
     notify(`emit:${nodeId}`)
@@ -146,6 +175,10 @@ export const liveStore = {
     statuses.clear()
     for (const key of emits.keys()) notify(`emit:${key}`)
     emits.clear()
+    for (const key of health.keys()) notify(`health:${key}`)
+    health.clear()
+    engineEvents = []
+    notify("engine")
     logLines = []
     notify("logs")
     for (const flow of paused) notify(`paused:${flow}`)
@@ -167,6 +200,22 @@ export function useNodeStatus(nodeId: string): LiveStatus | undefined {
   )
 }
 
+/** How the node's connection is doing, once it has said anything about it. */
+export function useNodeHealth(nodeId: string): NodeHealth | undefined {
+  return useSyncExternalStore(
+    (listener) => subscribeKey(`health:${nodeId}`, listener),
+    () => health.get(nodeId),
+  )
+}
+
+/** The last hundred things the engine said about itself, oldest first. */
+export function useEngineEvents(): EngineEvent[] {
+  return useSyncExternalStore(
+    (listener) => subscribeKey("engine", listener),
+    () => engineEvents,
+  )
+}
+
 /** Increments each time the node publishes something. */
 export function useNodeEmits(nodeId: string): number {
   return useSyncExternalStore(

frontend/src/components/Flow/useFlowSocket.ts

@@ -27,11 +27,48 @@ type FlowEvent =
       source?: ValueSource
     }
   | { type: "node_started"; node: string }
-  | { type: "node_executed"; node: string; outputs: number }
-  | { type: "node_error"; node: string; error: string }
+  | {
+      type: "node_executed"
+      flow?: string
+      node: string
+      outputs: number
+      duration_ms?: number
+    }
+  | {
+      type: "node_error"
+      flow?: string
+      node: string
+      error: string
+      ts?: number
+    }
   | { type: "node_status"; node: string; status: string; error?: string | null }
   | ({ type: "node_log" } & LogLine)
   | { type: "flow_paused"; flow: string; paused: boolean }
+  | {
+      type: "node_health"
+      flow?: string
+      node: string
+      health: "ok" | "down" | "unknown"
+      detail?: string | null
+      ts?: number
+    }
+  | { type: "flow_quarantined"; flow: string; error?: string; ts?: number }
+  | { type: "engine_degraded"; reason?: string; ts?: number }
+  | { type: "engine_fatal"; reason?: string; ts?: number }
+  | {
+      type: "cascade_dropped"
+      flow?: string
+      node?: string
+      deliveries?: number
+      ts?: number
+    }
+  | {
+      type: "queue_unavailable"
+      flow?: string
+      node?: string
+      error?: string
+      ts?: number
+    }
   | {
       type: "pipeline_rebuilt"
       nodes: { id: string; status: string; error?: string | null }[]
@@ -110,6 +147,44 @@ export function useFlowSocket(onAuthFailure?: () => void): void {
               status: "error",
               error: message.error,
             })
+            liveStore.recordEngineEvent({
+              type: message.type,
+              flow: message.flow,
+              node: message.node,
+              detail: message.error,
+              ts: message.ts ?? Date.now() / 1000,
+            })
+            break
+          case "node_health":
+            liveStore.setHealth(message.node, {
+              health: message.health,
+              detail: message.detail,
+            })
+            if (message.health === "down") {
+              liveStore.recordEngineEvent({
+                type: message.type,
+                flow: message.flow,
+                node: message.node,
+                detail: message.detail ?? "Reported itself down.",
+                ts: message.ts ?? Date.now() / 1000,
+              })
+            }
+            break
+          case "flow_quarantined":
+          case "engine_degraded":
+          case "engine_fatal":
+          case "cascade_dropped":
+          case "queue_unavailable":
+            liveStore.recordEngineEvent({
+              type: message.type,
+              flow: "flow" in message ? message.flow : undefined,
+              node: "node" in message ? message.node : undefined,
+              detail:
+                ("error" in message ? message.error : undefined) ??
+                ("reason" in message ? message.reason : undefined) ??
+                "",
+              ts: message.ts ?? Date.now() / 1000,
+            })
             break
           case "node_status":
             liveStore.setStatus(message.node, {

frontend/src/components/Sidebar/AppSidebar.tsx

@@ -1,4 +1,5 @@
 import {
+  Activity,
   Bell,
   Home,
   KeyRound,
@@ -26,6 +27,7 @@ const baseItems: Item[] = [
   { icon: Home, title: "Home", path: "/" },
   { icon: Workflow, title: "Flows", path: "/flows" },
   { icon: LayoutDashboard, title: "Dashboards", path: "/dashboards" },
+  { icon: Activity, title: "Health", path: "/health" },
   // Both are engine-wide operator settings rather than personal ones, so they
   // sit here and not among the per-user tabs under Settings.
   { icon: KeyRound, title: "Secrets", path: "/secrets" },