Keep the engine's own history, and a screen that reads it
A second bus subscriber folds executions, errors, timings and queue lag into per-minute rollups, keeps failures with their traceback and an audit trail of who published what, and records one row per cascade — manual runs and previews included, under an id of their own that writes no idempotency markers. Read back through /observability/*, which always answers 200 so a degraded engine still renders its own health screen. Also fixes two things found on the way: node-health alerts read `status` where the engine publishes `health`, so a device dropping never alerted anyone, and the Redis queue reported `parked: 0` whatever was held. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,457 @@
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query"
|
||||
import { createFileRoute, Link } from "@tanstack/react-router"
|
||||
import { ChevronDown, ChevronRight } from "lucide-react"
|
||||
import { useEffect, useState } from "react"
|
||||
|
||||
import {
|
||||
type EventRow,
|
||||
type FlowRollup,
|
||||
type HistoryPoint,
|
||||
ObservabilityService,
|
||||
} from "@/client"
|
||||
import { UplotChart } from "@/components/Common/UplotChart"
|
||||
import { useEngineEvents } from "@/components/Flow/liveStore"
|
||||
import { shape } from "@/components/Flow/MessageSparkline"
|
||||
import { PANEL_SECTION } from "@/components/Flow/SidePanel"
|
||||
import { useFlowSocket } from "@/components/Flow/useFlowSocket"
|
||||
import { Badge } from "@/components/ui/badge"
|
||||
|
||||
export const Route = createFileRoute("/_layout/health")({
|
||||
component: Health,
|
||||
head: () => ({
|
||||
meta: [
|
||||
{
|
||||
title: "Health - Fluksio",
|
||||
},
|
||||
],
|
||||
}),
|
||||
})
|
||||
|
||||
const healthKeys = {
|
||||
all: ["observability"] as const,
|
||||
summary: ["observability", "summary"] as const,
|
||||
events: ["observability", "events"] as const,
|
||||
}
|
||||
|
||||
/** The live view; anything older is the collector's rollups. */
|
||||
const HOURS = 24
|
||||
const CARD = "rounded-lg border border-border bg-card p-4 shadow-e1"
|
||||
|
||||
function ago(ts: string | number | null | undefined): string {
|
||||
if (!ts) return "—"
|
||||
const stamp = typeof ts === "number" ? ts * 1000 : Date.parse(ts)
|
||||
const seconds = Math.max(0, (Date.now() - stamp) / 1000)
|
||||
if (seconds < 90) return `${Math.round(seconds)}s ago`
|
||||
if (seconds < 5400) return `${Math.round(seconds / 60)}m ago`
|
||||
if (seconds < 172800) return `${Math.round(seconds / 3600)}h ago`
|
||||
return `${Math.round(seconds / 86400)}d ago`
|
||||
}
|
||||
|
||||
const round = (value: number) =>
|
||||
value >= 100 ? Math.round(value) : +value.toFixed(1)
|
||||
|
||||
function Tile({
|
||||
label,
|
||||
value,
|
||||
note,
|
||||
}: {
|
||||
label: string
|
||||
value: string
|
||||
note?: string
|
||||
}) {
|
||||
return (
|
||||
<div className={CARD}>
|
||||
<div className={PANEL_SECTION}>{label}</div>
|
||||
<div className="mt-1 text-2xl">{value}</div>
|
||||
{note ? (
|
||||
<div className="text-xs text-muted-foreground">{note}</div>
|
||||
) : null}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** A flow's execution trend, drawn from the 60 slices the rollup carries. */
|
||||
function Spark({ counts }: { counts: number[] }) {
|
||||
const points: HistoryPoint[] = counts.map((value, index) => ({
|
||||
ts: index,
|
||||
value,
|
||||
}))
|
||||
if (points.every((point) => point.value === 0)) {
|
||||
return <span className="text-xs text-muted-foreground">nothing yet</span>
|
||||
}
|
||||
const { line } = shape(points)
|
||||
return (
|
||||
<svg
|
||||
viewBox="0 0 100 100"
|
||||
preserveAspectRatio="none"
|
||||
className="h-6 w-24 overflow-visible"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path
|
||||
d={line}
|
||||
fill="none"
|
||||
stroke="var(--chart-1)"
|
||||
strokeWidth="1.5"
|
||||
vectorEffect="non-scaling-stroke"
|
||||
/>
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
function Failure({ event }: { event: EventRow }) {
|
||||
const [open, setOpen] = useState(false)
|
||||
const [first, ...rest] = event.detail.split("\n")
|
||||
return (
|
||||
<div className="border-b border-border py-2 last:border-0">
|
||||
<button
|
||||
type="button"
|
||||
className="flex w-full items-start gap-2 text-left"
|
||||
onClick={() => setOpen(!open)}
|
||||
disabled={rest.length === 0}
|
||||
>
|
||||
{rest.length ? (
|
||||
open ? (
|
||||
<ChevronDown className="mt-0.5 size-4 shrink-0 text-muted-foreground" />
|
||||
) : (
|
||||
<ChevronRight className="mt-0.5 size-4 shrink-0 text-muted-foreground" />
|
||||
)
|
||||
) : (
|
||||
<span className="size-4 shrink-0" />
|
||||
)}
|
||||
<span className="min-w-0 flex-1">
|
||||
<span className="font-mono text-sm">
|
||||
{event.node || event.flow || "engine"}
|
||||
</span>
|
||||
<span className="ml-2 text-sm text-muted-foreground">{first}</span>
|
||||
</span>
|
||||
<span className="shrink-0 text-xs text-muted-foreground">
|
||||
{ago(event.ts)}
|
||||
</span>
|
||||
</button>
|
||||
{open && rest.length ? (
|
||||
<pre className="mt-2 max-h-64 overflow-auto rounded-md border border-border bg-muted/40 p-3 font-mono text-xs">
|
||||
{rest.join("\n")}
|
||||
</pre>
|
||||
) : null}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Health() {
|
||||
// The page is a socket subscriber like the editor: a failure should appear
|
||||
// without waiting for the next poll.
|
||||
useFlowSocket()
|
||||
const live = useEngineEvents()
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
const { data: summary } = useQuery({
|
||||
queryKey: healthKeys.summary,
|
||||
queryFn: () => ObservabilityService.readSummary(),
|
||||
refetchInterval: 10000,
|
||||
})
|
||||
const { data: series } = useQuery({
|
||||
queryKey: ["observability", "timeseries", HOURS],
|
||||
queryFn: () => ObservabilityService.readTimeseries({ hours: HOURS }),
|
||||
refetchInterval: 30000,
|
||||
})
|
||||
const { data: flows } = useQuery({
|
||||
queryKey: ["observability", "flows", HOURS],
|
||||
queryFn: () => ObservabilityService.readFlowRollups({ hours: HOURS }),
|
||||
refetchInterval: 30000,
|
||||
})
|
||||
const { data: runs } = useQuery({
|
||||
queryKey: ["observability", "runs"],
|
||||
queryFn: () => ObservabilityService.readRuns({ limit: 15 }),
|
||||
refetchInterval: 30000,
|
||||
})
|
||||
const { data: failures } = useQuery({
|
||||
queryKey: [...healthKeys.events, "failure"],
|
||||
queryFn: () =>
|
||||
ObservabilityService.readEvents({ kind: "failure", limit: 25 }),
|
||||
refetchInterval: 30000,
|
||||
})
|
||||
const { data: audit } = useQuery({
|
||||
queryKey: [...healthKeys.events, "audit"],
|
||||
queryFn: () =>
|
||||
ObservabilityService.readEvents({ kind: "audit", limit: 15 }),
|
||||
refetchInterval: 30000,
|
||||
})
|
||||
const { data: dead } = useQuery({
|
||||
queryKey: ["observability", "dead-letter"],
|
||||
queryFn: () => ObservabilityService.readDeadLetters({ limit: 20 }),
|
||||
refetchInterval: 30000,
|
||||
})
|
||||
|
||||
// Something just went wrong on the socket. The row for it is written on the
|
||||
// collector's next flush, so the refetch waits that out rather than asking
|
||||
// for a failure the database does not have yet.
|
||||
const seen = live.length
|
||||
useEffect(() => {
|
||||
if (!seen) return
|
||||
const timer = setTimeout(() => {
|
||||
queryClient.invalidateQueries({ queryKey: healthKeys.events })
|
||||
queryClient.invalidateQueries({ queryKey: healthKeys.summary })
|
||||
}, 16000)
|
||||
return () => clearTimeout(timer)
|
||||
}, [seen, queryClient])
|
||||
|
||||
const points = series ?? []
|
||||
const at = (
|
||||
pick: (point: (typeof points)[number]) => number,
|
||||
): HistoryPoint[] =>
|
||||
points.map((point) => ({ ts: point.ts, value: pick(point) }))
|
||||
|
||||
const queue = (summary?.queue ?? {}) as Record<string, number>
|
||||
const degraded = summary?.status === "degraded"
|
||||
|
||||
return (
|
||||
<div className="grid gap-6">
|
||||
<div className="grid gap-1">
|
||||
<div className="flex items-center gap-3">
|
||||
<h1 className="text-2xl">Health</h1>
|
||||
<Badge variant={degraded ? "destructive" : "secondary"}>
|
||||
{degraded ? "Degraded" : "Running normally"}
|
||||
</Badge>
|
||||
</div>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{summary?.problems.length
|
||||
? summary.problems.join(" · ")
|
||||
: "What the engine has been doing over the last day, and what it is doing now."}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<section className="grid gap-3 sm:grid-cols-2 lg:grid-cols-5">
|
||||
<Tile
|
||||
label="Nodes"
|
||||
value={String(summary?.nodes.total ?? 0)}
|
||||
note={
|
||||
summary?.nodes.error
|
||||
? `${summary.nodes.error} failed to load`
|
||||
: "all loaded"
|
||||
}
|
||||
/>
|
||||
<Tile
|
||||
label="Flows running"
|
||||
value={`${summary?.flows.running ?? 0}/${summary?.flows.total ?? 0}`}
|
||||
note={
|
||||
summary?.flows.quarantined
|
||||
? `${summary.flows.quarantined} quarantined`
|
||||
: summary?.flows.paused
|
||||
? `${summary.flows.paused} paused`
|
||||
: "none paused"
|
||||
}
|
||||
/>
|
||||
<Tile
|
||||
label="Failures (24h)"
|
||||
value={String(summary?.failures_24h ?? 0)}
|
||||
note={
|
||||
failures?.length
|
||||
? `latest ${ago(failures[0].ts)}`
|
||||
: "nothing recorded"
|
||||
}
|
||||
/>
|
||||
<Tile
|
||||
label="Queue in flight"
|
||||
value={String(queue.pending ?? 0)}
|
||||
note={`${queue.delayed ?? 0} waiting · ${queue.parked ?? 0} parked`}
|
||||
/>
|
||||
<Tile
|
||||
label="Loop lag"
|
||||
value={`${round(summary?.loop_lag.ewma ?? 0)} ms`}
|
||||
note={`peak ${round(summary?.loop_lag.max_60s ?? 0)} ms in the last minute`}
|
||||
/>
|
||||
</section>
|
||||
|
||||
<section className="grid gap-4 lg:grid-cols-2">
|
||||
<div className={`${CARD} flex h-64 flex-col`}>
|
||||
<h2 className={PANEL_SECTION}>Throughput per minute</h2>
|
||||
<UplotChart
|
||||
labels={["messages", "executions"]}
|
||||
plots={[
|
||||
at((point) => point.messages),
|
||||
at((point) => point.executions),
|
||||
]}
|
||||
empty="Nothing has run yet."
|
||||
/>
|
||||
</div>
|
||||
<div className={`${CARD} flex h-64 flex-col`}>
|
||||
<h2 className={PANEL_SECTION}>Failures and timing</h2>
|
||||
<UplotChart
|
||||
labels={["errors", "avg ms", "avg lag ms"]}
|
||||
plots={[
|
||||
at((point) => point.errors),
|
||||
at((point) => point.avg_ms),
|
||||
at((point) => point.avg_lag_ms),
|
||||
]}
|
||||
empty="Nothing has run yet."
|
||||
/>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="grid gap-3">
|
||||
<h2 className={PANEL_SECTION}>Flows</h2>
|
||||
<div className={`${CARD} overflow-x-auto`}>
|
||||
<table className="w-full text-sm">
|
||||
<thead className="text-xs text-muted-foreground">
|
||||
<tr className="text-left">
|
||||
<th className="pb-2 font-medium">Flow</th>
|
||||
<th className="pb-2 font-medium">Executions</th>
|
||||
<th className="pb-2 font-medium">Errors</th>
|
||||
<th className="pb-2 font-medium">Avg</th>
|
||||
<th className="pb-2 font-medium">Lag</th>
|
||||
<th className="pb-2 font-medium">Trend</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{(flows ?? []).map((row: FlowRollup) => (
|
||||
<tr key={row.flow} className="border-t border-border">
|
||||
<td className="py-2">
|
||||
<Link
|
||||
to="/flows/$flowName"
|
||||
params={{ flowName: row.flow }}
|
||||
className="font-mono hover:underline"
|
||||
>
|
||||
{row.flow || "—"}
|
||||
</Link>
|
||||
</td>
|
||||
<td className="py-2">{row.executions}</td>
|
||||
<td className="py-2">
|
||||
{row.errors ? (
|
||||
<Badge variant="destructive">{row.errors} failed</Badge>
|
||||
) : (
|
||||
<span className="text-muted-foreground">none</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="py-2">{round(row.avg_ms)} ms</td>
|
||||
<td className="py-2">{round(row.avg_lag_ms)} ms</td>
|
||||
<td className="py-2">
|
||||
<Spark counts={row.spark} />
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
{flows?.length === 0 ? (
|
||||
<tr>
|
||||
<td
|
||||
colSpan={6}
|
||||
className="py-6 text-center text-muted-foreground"
|
||||
>
|
||||
No flow has run in the last day.
|
||||
</td>
|
||||
</tr>
|
||||
) : null}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="grid content-start gap-4 lg:grid-cols-2">
|
||||
<div className="grid content-start gap-3">
|
||||
<h2 className={PANEL_SECTION}>Recent runs</h2>
|
||||
<div className={CARD}>
|
||||
{runs?.length ? (
|
||||
runs.map((run) => (
|
||||
<div
|
||||
key={run.id}
|
||||
className="flex items-baseline gap-3 border-b border-border py-2 text-sm last:border-0"
|
||||
>
|
||||
<span className="min-w-0 flex-1 truncate font-mono">
|
||||
{run.flow}
|
||||
</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{run.source}
|
||||
</span>
|
||||
<span
|
||||
className={
|
||||
run.status === "error"
|
||||
? "text-destructive"
|
||||
: run.status === "ok"
|
||||
? "text-status-success"
|
||||
: "text-muted-foreground"
|
||||
}
|
||||
>
|
||||
{run.status}
|
||||
</span>
|
||||
<span className="w-16 text-right text-muted-foreground">
|
||||
{round(run.duration_ms)} ms
|
||||
</span>
|
||||
<span className="w-16 text-right text-xs text-muted-foreground">
|
||||
{ago(run.started_at)}
|
||||
</span>
|
||||
</div>
|
||||
))
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
No runs recorded yet.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid content-start gap-3">
|
||||
<h2 className={PANEL_SECTION}>Failures</h2>
|
||||
<div className={CARD}>
|
||||
{failures?.length ? (
|
||||
failures.map((event) => <Failure key={event.id} event={event} />)
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Nothing has failed in the last day.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{dead?.length ? (
|
||||
<section className="grid gap-3">
|
||||
<h2 className={PANEL_SECTION}>Given up on</h2>
|
||||
<div className={CARD}>
|
||||
{dead.map((item) => (
|
||||
<div
|
||||
key={item.id}
|
||||
className="flex items-baseline gap-3 border-b border-border py-2 text-sm last:border-0"
|
||||
>
|
||||
<span className="min-w-0 flex-1 truncate font-mono">
|
||||
{item.node}
|
||||
</span>
|
||||
<span className="text-muted-foreground">{item.reason}</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{ago(item.ts)}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
) : null}
|
||||
|
||||
<section className="grid gap-3">
|
||||
<h2 className={PANEL_SECTION}>Changes</h2>
|
||||
<div className={CARD}>
|
||||
{audit?.length ? (
|
||||
audit.map((event) => (
|
||||
<div
|
||||
key={event.id}
|
||||
className="flex items-baseline gap-3 border-b border-border py-2 text-sm last:border-0"
|
||||
>
|
||||
<span className="min-w-0 flex-1 truncate">
|
||||
{event.actor} {event.detail}
|
||||
{event.flow ? (
|
||||
<span className="ml-1 font-mono text-muted-foreground">
|
||||
{event.flow}
|
||||
</span>
|
||||
) : null}
|
||||
</span>
|
||||
<span className="shrink-0 text-xs text-muted-foreground">
|
||||
{ago(event.ts)}
|
||||
</span>
|
||||
</div>
|
||||
))
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Nothing has changed yet.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user