The worker script is handed to the interpreter by path, so app/flow was sys.path[0] for every node: `import queue` got the engine's. It now drops its own directory before anything else imports, and runs with the deployment's credentials scrubbed out of its environment. Also: reload builds off the event loop, the pool wakes what is blocked on it when it stops, a refused metrics flush is kept for the next one rather than dropped, and the cascade events are paired through failures.
462 lines
15 KiB
TypeScript
462 lines
15 KiB
TypeScript
import { useQuery, useQueryClient } from "@tanstack/react-query"
|
|
import { createFileRoute, Link } from "@tanstack/react-router"
|
|
import { ChevronDown, ChevronRight } from "lucide-react"
|
|
import { useEffect, useState } from "react"
|
|
|
|
import {
|
|
type EventRow,
|
|
type FlowRollup,
|
|
type HistoryPoint,
|
|
ObservabilityService,
|
|
} from "@/client"
|
|
import { UplotChart } from "@/components/Common/UplotChart"
|
|
import { useEngineEvents } from "@/components/Flow/liveStore"
|
|
import { shape } from "@/components/Flow/MessageSparkline"
|
|
import { PANEL_SECTION } from "@/components/Flow/SidePanel"
|
|
import { useFlowSocket } from "@/components/Flow/useFlowSocket"
|
|
import { Badge } from "@/components/ui/badge"
|
|
|
|
export const Route = createFileRoute("/_layout/health")({
|
|
component: Health,
|
|
head: () => ({
|
|
meta: [
|
|
{
|
|
title: "Health - Fluksio",
|
|
},
|
|
],
|
|
}),
|
|
})
|
|
|
|
const healthKeys = {
|
|
all: ["observability"] as const,
|
|
summary: ["observability", "summary"] as const,
|
|
events: ["observability", "events"] as const,
|
|
}
|
|
|
|
/** The live view; anything older is the collector's rollups. */
|
|
const HOURS = 24
|
|
const CARD = "rounded-lg border border-border bg-card p-4 shadow-e1"
|
|
|
|
function ago(ts: string | number | null | undefined): string {
|
|
if (!ts) return "—"
|
|
const stamp = typeof ts === "number" ? ts * 1000 : Date.parse(ts)
|
|
const seconds = Math.max(0, (Date.now() - stamp) / 1000)
|
|
if (seconds < 90) return `${Math.round(seconds)}s ago`
|
|
if (seconds < 5400) return `${Math.round(seconds / 60)}m ago`
|
|
if (seconds < 172800) return `${Math.round(seconds / 3600)}h ago`
|
|
return `${Math.round(seconds / 86400)}d ago`
|
|
}
|
|
|
|
const round = (value: number) =>
|
|
value >= 100 ? Math.round(value) : +value.toFixed(1)
|
|
|
|
function Tile({
|
|
label,
|
|
value,
|
|
note,
|
|
}: {
|
|
label: string
|
|
value: string
|
|
note?: string
|
|
}) {
|
|
return (
|
|
<div className={CARD}>
|
|
<div className={PANEL_SECTION}>{label}</div>
|
|
<div className="mt-1 text-2xl">{value}</div>
|
|
{note ? (
|
|
<div className="text-xs text-muted-foreground">{note}</div>
|
|
) : null}
|
|
</div>
|
|
)
|
|
}
|
|
|
|
/** A flow's execution trend, drawn from the 60 slices the rollup carries. */
|
|
function Spark({ counts }: { counts: number[] }) {
|
|
const points: HistoryPoint[] = counts.map((value, index) => ({
|
|
ts: index,
|
|
value,
|
|
}))
|
|
if (points.every((point) => point.value === 0)) {
|
|
return <span className="text-xs text-muted-foreground">nothing yet</span>
|
|
}
|
|
const { line } = shape(points)
|
|
return (
|
|
<svg
|
|
viewBox="0 0 100 100"
|
|
preserveAspectRatio="none"
|
|
className="h-6 w-24 overflow-visible"
|
|
aria-hidden="true"
|
|
>
|
|
<path
|
|
d={line}
|
|
fill="none"
|
|
stroke="var(--chart-1)"
|
|
strokeWidth="1.5"
|
|
vectorEffect="non-scaling-stroke"
|
|
/>
|
|
</svg>
|
|
)
|
|
}
|
|
|
|
function Failure({ event }: { event: EventRow }) {
|
|
const [open, setOpen] = useState(false)
|
|
const [first, ...rest] = event.detail.split("\n")
|
|
return (
|
|
<div className="border-b border-border py-2 last:border-0">
|
|
<button
|
|
type="button"
|
|
className="flex w-full items-start gap-2 text-left"
|
|
onClick={() => setOpen(!open)}
|
|
disabled={rest.length === 0}
|
|
>
|
|
{rest.length ? (
|
|
open ? (
|
|
<ChevronDown className="mt-0.5 size-4 shrink-0 text-muted-foreground" />
|
|
) : (
|
|
<ChevronRight className="mt-0.5 size-4 shrink-0 text-muted-foreground" />
|
|
)
|
|
) : (
|
|
<span className="size-4 shrink-0" />
|
|
)}
|
|
<span className="min-w-0 flex-1">
|
|
<span className="font-mono text-sm">
|
|
{event.node || event.flow || "engine"}
|
|
</span>
|
|
<span className="ml-2 text-sm text-muted-foreground">{first}</span>
|
|
</span>
|
|
<span className="shrink-0 text-xs text-muted-foreground">
|
|
{ago(event.ts)}
|
|
</span>
|
|
</button>
|
|
{open && rest.length ? (
|
|
<pre className="mt-2 max-h-64 overflow-auto rounded-md border border-border bg-muted/40 p-3 font-mono text-xs">
|
|
{rest.join("\n")}
|
|
</pre>
|
|
) : null}
|
|
</div>
|
|
)
|
|
}
|
|
|
|
function Health() {
|
|
// The page is a socket subscriber like the editor: a failure should appear
|
|
// without waiting for the next poll.
|
|
useFlowSocket()
|
|
const live = useEngineEvents()
|
|
const queryClient = useQueryClient()
|
|
|
|
const { data: summary } = useQuery({
|
|
queryKey: healthKeys.summary,
|
|
queryFn: () => ObservabilityService.readSummary(),
|
|
refetchInterval: 10000,
|
|
})
|
|
const { data: series } = useQuery({
|
|
queryKey: ["observability", "timeseries", HOURS],
|
|
queryFn: () => ObservabilityService.readTimeseries({ hours: HOURS }),
|
|
refetchInterval: 30000,
|
|
})
|
|
const { data: flows } = useQuery({
|
|
queryKey: ["observability", "flows", HOURS],
|
|
queryFn: () => ObservabilityService.readFlowRollups({ hours: HOURS }),
|
|
refetchInterval: 30000,
|
|
})
|
|
const { data: runs } = useQuery({
|
|
queryKey: ["observability", "runs"],
|
|
queryFn: () => ObservabilityService.readRuns({ limit: 15 }),
|
|
refetchInterval: 30000,
|
|
})
|
|
const { data: failures } = useQuery({
|
|
queryKey: [...healthKeys.events, "failure"],
|
|
queryFn: () =>
|
|
ObservabilityService.readEvents({ kind: "failure", limit: 25 }),
|
|
refetchInterval: 30000,
|
|
})
|
|
const { data: audit } = useQuery({
|
|
queryKey: [...healthKeys.events, "audit"],
|
|
queryFn: () =>
|
|
ObservabilityService.readEvents({ kind: "audit", limit: 15 }),
|
|
refetchInterval: 30000,
|
|
})
|
|
const { data: dead } = useQuery({
|
|
queryKey: ["observability", "dead-letter"],
|
|
queryFn: () => ObservabilityService.readDeadLetters({ limit: 20 }),
|
|
refetchInterval: 30000,
|
|
})
|
|
|
|
// Something just went wrong on the socket. The row for it is written on the
|
|
// collector's next flush, so the refetch waits that out rather than asking
|
|
// for a failure the database does not have yet.
|
|
//
|
|
// The newest event's minute, not the count: the count stops changing once the
|
|
// ring is full, and a per-event key would let a flapping node restart the
|
|
// timer forever without it ever firing.
|
|
const seen = live.length ? Math.floor(live[live.length - 1].ts / 60) : 0
|
|
useEffect(() => {
|
|
if (!seen) return
|
|
const timer = setTimeout(() => {
|
|
queryClient.invalidateQueries({ queryKey: healthKeys.events })
|
|
queryClient.invalidateQueries({ queryKey: healthKeys.summary })
|
|
}, 16000)
|
|
return () => clearTimeout(timer)
|
|
}, [seen, queryClient])
|
|
|
|
const points = series ?? []
|
|
const at = (
|
|
pick: (point: (typeof points)[number]) => number,
|
|
): HistoryPoint[] =>
|
|
points.map((point) => ({ ts: point.ts, value: pick(point) }))
|
|
|
|
const queue = (summary?.queue ?? {}) as Record<string, number>
|
|
const degraded = summary?.status === "degraded"
|
|
|
|
return (
|
|
<div className="grid gap-6">
|
|
<div className="grid gap-1">
|
|
<div className="flex items-center gap-3">
|
|
<h1 className="text-2xl">Health</h1>
|
|
<Badge variant={degraded ? "destructive" : "secondary"}>
|
|
{degraded ? "Degraded" : "Running normally"}
|
|
</Badge>
|
|
</div>
|
|
<p className="text-sm text-muted-foreground">
|
|
{summary?.problems.length
|
|
? summary.problems.join(" · ")
|
|
: "What the engine has been doing over the last day, and what it is doing now."}
|
|
</p>
|
|
</div>
|
|
|
|
<section className="grid gap-3 sm:grid-cols-2 lg:grid-cols-5">
|
|
<Tile
|
|
label="Nodes"
|
|
value={String(summary?.nodes.total ?? 0)}
|
|
note={
|
|
summary?.nodes.error
|
|
? `${summary.nodes.error} failed to load`
|
|
: "all loaded"
|
|
}
|
|
/>
|
|
<Tile
|
|
label="Flows running"
|
|
value={`${summary?.flows.running ?? 0}/${summary?.flows.total ?? 0}`}
|
|
note={
|
|
summary?.flows.quarantined
|
|
? `${summary.flows.quarantined} quarantined`
|
|
: summary?.flows.paused
|
|
? `${summary.flows.paused} paused`
|
|
: "none paused"
|
|
}
|
|
/>
|
|
<Tile
|
|
label="Failures (24h)"
|
|
value={String(summary?.failures_24h ?? 0)}
|
|
note={
|
|
failures?.length
|
|
? `latest ${ago(failures[0].ts)}`
|
|
: "nothing recorded"
|
|
}
|
|
/>
|
|
<Tile
|
|
label="Queue in flight"
|
|
value={String(queue.pending ?? 0)}
|
|
note={`${queue.delayed ?? 0} waiting · ${queue.parked ?? 0} parked`}
|
|
/>
|
|
<Tile
|
|
label="Loop lag"
|
|
value={`${round(summary?.loop_lag.ewma ?? 0)} ms`}
|
|
note={`peak ${round(summary?.loop_lag.max_60s ?? 0)} ms in the last minute`}
|
|
/>
|
|
</section>
|
|
|
|
<section className="grid gap-4 lg:grid-cols-2">
|
|
<div className={`${CARD} flex h-64 flex-col`}>
|
|
<h2 className={PANEL_SECTION}>Throughput per minute</h2>
|
|
<UplotChart
|
|
labels={["messages", "executions"]}
|
|
plots={[
|
|
at((point) => point.messages),
|
|
at((point) => point.executions),
|
|
]}
|
|
empty="Nothing has run yet."
|
|
/>
|
|
</div>
|
|
<div className={`${CARD} flex h-64 flex-col`}>
|
|
<h2 className={PANEL_SECTION}>Failures and timing</h2>
|
|
<UplotChart
|
|
labels={["errors", "avg ms", "avg lag ms"]}
|
|
plots={[
|
|
at((point) => point.errors),
|
|
at((point) => point.avg_ms),
|
|
at((point) => point.avg_lag_ms),
|
|
]}
|
|
empty="Nothing has run yet."
|
|
/>
|
|
</div>
|
|
</section>
|
|
|
|
<section className="grid gap-3">
|
|
<h2 className={PANEL_SECTION}>Flows</h2>
|
|
<div className={`${CARD} overflow-x-auto`}>
|
|
<table className="w-full text-sm">
|
|
<thead className="text-xs text-muted-foreground">
|
|
<tr className="text-left">
|
|
<th className="pb-2 font-medium">Flow</th>
|
|
<th className="pb-2 font-medium">Executions</th>
|
|
<th className="pb-2 font-medium">Errors</th>
|
|
<th className="pb-2 font-medium">Avg</th>
|
|
<th className="pb-2 font-medium">Lag</th>
|
|
<th className="pb-2 font-medium">Trend</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{(flows ?? []).map((row: FlowRollup) => (
|
|
<tr key={row.flow} className="border-t border-border">
|
|
<td className="py-2">
|
|
<Link
|
|
to="/flows/$flowName"
|
|
params={{ flowName: row.flow }}
|
|
className="font-mono hover:underline"
|
|
>
|
|
{row.flow || "—"}
|
|
</Link>
|
|
</td>
|
|
<td className="py-2">{row.executions}</td>
|
|
<td className="py-2">
|
|
{row.errors ? (
|
|
<Badge variant="destructive">{row.errors} failed</Badge>
|
|
) : (
|
|
<span className="text-muted-foreground">none</span>
|
|
)}
|
|
</td>
|
|
<td className="py-2">{round(row.avg_ms)} ms</td>
|
|
<td className="py-2">{round(row.avg_lag_ms)} ms</td>
|
|
<td className="py-2">
|
|
<Spark counts={row.spark} />
|
|
</td>
|
|
</tr>
|
|
))}
|
|
{flows?.length === 0 ? (
|
|
<tr>
|
|
<td
|
|
colSpan={6}
|
|
className="py-6 text-center text-muted-foreground"
|
|
>
|
|
No flow has run in the last day.
|
|
</td>
|
|
</tr>
|
|
) : null}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</section>
|
|
|
|
<section className="grid content-start gap-4 lg:grid-cols-2">
|
|
<div className="grid content-start gap-3">
|
|
<h2 className={PANEL_SECTION}>Recent runs</h2>
|
|
<div className={CARD}>
|
|
{runs?.length ? (
|
|
runs.map((run) => (
|
|
<div
|
|
key={run.id}
|
|
className="flex items-baseline gap-3 border-b border-border py-2 text-sm last:border-0"
|
|
>
|
|
<span className="min-w-0 flex-1 truncate font-mono">
|
|
{run.flow}
|
|
</span>
|
|
<span className="text-xs text-muted-foreground">
|
|
{run.source}
|
|
</span>
|
|
<span
|
|
className={
|
|
run.status === "error"
|
|
? "text-destructive"
|
|
: run.status === "ok"
|
|
? "text-status-success"
|
|
: "text-muted-foreground"
|
|
}
|
|
>
|
|
{run.status}
|
|
</span>
|
|
<span className="w-16 text-right text-muted-foreground">
|
|
{round(run.duration_ms)} ms
|
|
</span>
|
|
<span className="w-16 text-right text-xs text-muted-foreground">
|
|
{ago(run.started_at)}
|
|
</span>
|
|
</div>
|
|
))
|
|
) : (
|
|
<p className="text-sm text-muted-foreground">
|
|
No runs recorded yet.
|
|
</p>
|
|
)}
|
|
</div>
|
|
</div>
|
|
|
|
<div className="grid content-start gap-3">
|
|
<h2 className={PANEL_SECTION}>Failures</h2>
|
|
<div className={CARD}>
|
|
{failures?.length ? (
|
|
failures.map((event) => <Failure key={event.id} event={event} />)
|
|
) : (
|
|
<p className="text-sm text-muted-foreground">
|
|
Nothing has failed in the last day.
|
|
</p>
|
|
)}
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
{dead?.length ? (
|
|
<section className="grid gap-3">
|
|
<h2 className={PANEL_SECTION}>Given up on</h2>
|
|
<div className={CARD}>
|
|
{dead.map((item) => (
|
|
<div
|
|
key={item.id}
|
|
className="flex items-baseline gap-3 border-b border-border py-2 text-sm last:border-0"
|
|
>
|
|
<span className="min-w-0 flex-1 truncate font-mono">
|
|
{item.node}
|
|
</span>
|
|
<span className="text-muted-foreground">{item.reason}</span>
|
|
<span className="text-xs text-muted-foreground">
|
|
{ago(item.ts)}
|
|
</span>
|
|
</div>
|
|
))}
|
|
</div>
|
|
</section>
|
|
) : null}
|
|
|
|
<section className="grid gap-3">
|
|
<h2 className={PANEL_SECTION}>Changes</h2>
|
|
<div className={CARD}>
|
|
{audit?.length ? (
|
|
audit.map((event) => (
|
|
<div
|
|
key={event.id}
|
|
className="flex items-baseline gap-3 border-b border-border py-2 text-sm last:border-0"
|
|
>
|
|
<span className="min-w-0 flex-1 truncate">
|
|
{event.actor} {event.detail}
|
|
{event.flow ? (
|
|
<span className="ml-1 font-mono text-muted-foreground">
|
|
{event.flow}
|
|
</span>
|
|
) : null}
|
|
</span>
|
|
<span className="shrink-0 text-xs text-muted-foreground">
|
|
{ago(event.ts)}
|
|
</span>
|
|
</div>
|
|
))
|
|
) : (
|
|
<p className="text-sm text-muted-foreground">
|
|
Nothing has changed yet.
|
|
</p>
|
|
)}
|
|
</div>
|
|
</section>
|
|
</div>
|
|
)
|
|
}
|