Home charts execution time per node type on a log scale, from a /observability/node-types rollup
This commit is contained in:
@@ -86,6 +86,17 @@ class FlowRollup(BaseModel):
|
|||||||
last_error_ts: float | None = None
|
last_error_ts: float | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class NodeTypeTiming(BaseModel):
|
||||||
|
"""How long one kind of node takes to run, across every node of it."""
|
||||||
|
|
||||||
|
type: str
|
||||||
|
#: How many published nodes of this type ran inside the window.
|
||||||
|
nodes: int
|
||||||
|
executions: int
|
||||||
|
avg_ms: float
|
||||||
|
max_ms: float
|
||||||
|
|
||||||
|
|
||||||
class RunRow(BaseModel):
|
class RunRow(BaseModel):
|
||||||
id: str
|
id: str
|
||||||
flow: str
|
flow: str
|
||||||
@@ -428,6 +439,58 @@ def read_flow_rollups(
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/node-types", response_model=list[NodeTypeTiming])
|
||||||
|
def read_node_type_timings(
|
||||||
|
session: SessionDep,
|
||||||
|
controller: FlowControllerDep,
|
||||||
|
hours: Annotated[int, Query(ge=0, le=8760)] = 24,
|
||||||
|
) -> Any:
|
||||||
|
"""Execution time per node type, slowest first, to point at the hotspots.
|
||||||
|
|
||||||
|
The rollups are kept per node, so the type is joined on from the published
|
||||||
|
flows. A node deleted since it ran has no type left to count under.
|
||||||
|
"""
|
||||||
|
hours = _window_hours(hours)
|
||||||
|
types = {
|
||||||
|
f"{flow.name}.{node.id}": node.type
|
||||||
|
for flow in controller.store.read_all()
|
||||||
|
for node in flow.nodes
|
||||||
|
}
|
||||||
|
statement = (
|
||||||
|
sa_select(
|
||||||
|
col(MetricBucket.node),
|
||||||
|
func.sum(col(MetricBucket.executions)).label("executions"),
|
||||||
|
func.sum(col(MetricBucket.duration_sum_ms)).label("duration_sum_ms"),
|
||||||
|
func.max(col(MetricBucket.duration_max_ms)).label("max_ms"),
|
||||||
|
)
|
||||||
|
.where(col(MetricBucket.bucket) >= _since(hours))
|
||||||
|
.group_by(col(MetricBucket.node))
|
||||||
|
)
|
||||||
|
totals: dict[str, dict[str, float]] = {}
|
||||||
|
for row in session.execute(statement):
|
||||||
|
kind = types.get(row.node)
|
||||||
|
if kind is None or not row.executions:
|
||||||
|
continue
|
||||||
|
entry = totals.setdefault(
|
||||||
|
kind, {"nodes": 0, "executions": 0, "duration_sum_ms": 0.0, "max_ms": 0.0}
|
||||||
|
)
|
||||||
|
entry["nodes"] += 1
|
||||||
|
entry["executions"] += row.executions
|
||||||
|
entry["duration_sum_ms"] += row.duration_sum_ms
|
||||||
|
entry["max_ms"] = max(entry["max_ms"], row.max_ms)
|
||||||
|
timings = [
|
||||||
|
NodeTypeTiming(
|
||||||
|
type=kind,
|
||||||
|
nodes=int(entry["nodes"]),
|
||||||
|
executions=int(entry["executions"]),
|
||||||
|
avg_ms=round(entry["duration_sum_ms"] / entry["executions"], 3),
|
||||||
|
max_ms=round(entry["max_ms"], 3),
|
||||||
|
)
|
||||||
|
for kind, entry in totals.items()
|
||||||
|
]
|
||||||
|
return sorted(timings, key=lambda timing: timing.avg_ms, reverse=True)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/runs", response_model=RunPage)
|
@router.get("/runs", response_model=RunPage)
|
||||||
def read_runs(
|
def read_runs(
|
||||||
session: SessionDep,
|
session: SessionDep,
|
||||||
|
|||||||
@@ -512,3 +512,59 @@ def test_a_superuser_can_flush_the_collector_now(
|
|||||||
)
|
)
|
||||||
).all()
|
).all()
|
||||||
assert len(written) == 1
|
assert len(written) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_time_is_summed_per_node_type(
|
||||||
|
client: TestClient, superuser_token_headers: dict[str, str], db: Session
|
||||||
|
) -> None:
|
||||||
|
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||||
|
|
||||||
|
flow = "node_type_test"
|
||||||
|
controller = client.app.state.flow_controller
|
||||||
|
before = controller.store.read_all
|
||||||
|
controller.store.read_all = lambda: [
|
||||||
|
FlowDef(
|
||||||
|
name=flow,
|
||||||
|
nodes=[
|
||||||
|
NodeDef(id="a", type="python"),
|
||||||
|
NodeDef(id="b", type="python"),
|
||||||
|
NodeDef(id="tick", type="inject"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
]
|
||||||
|
now = datetime.now(UTC).replace(second=0, microsecond=0)
|
||||||
|
for node, executions, total, peak in (
|
||||||
|
("a", 2, 30.0, 20.0),
|
||||||
|
("b", 2, 10.0, 6.0),
|
||||||
|
("tick", 4, 0.4, 0.2),
|
||||||
|
# Deleted since it ran: nothing to count it under.
|
||||||
|
("gone", 1, 500.0, 500.0),
|
||||||
|
):
|
||||||
|
db.add(
|
||||||
|
MetricBucket(
|
||||||
|
flow=flow,
|
||||||
|
node=f"{flow}.{node}",
|
||||||
|
bucket=now - timedelta(minutes=2),
|
||||||
|
executions=executions,
|
||||||
|
duration_sum_ms=total,
|
||||||
|
duration_max_ms=peak,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
try:
|
||||||
|
rows = client.get(
|
||||||
|
f"{PREFIX}/node-types", headers=superuser_token_headers
|
||||||
|
).json()
|
||||||
|
finally:
|
||||||
|
controller.store.read_all = before
|
||||||
|
|
||||||
|
# Slowest first.
|
||||||
|
assert [row["type"] for row in rows] == ["python", "inject"]
|
||||||
|
assert rows[0] == {
|
||||||
|
"type": "python",
|
||||||
|
"nodes": 2,
|
||||||
|
"executions": 4,
|
||||||
|
"avg_ms": 10.0,
|
||||||
|
"max_ms": 20.0,
|
||||||
|
}
|
||||||
|
assert rows[1]["avg_ms"] == 0.1
|
||||||
|
|||||||
@@ -58,6 +58,12 @@ Health shows flow status, node load errors, queue depth and event-loop lag.
|
|||||||
After you have used batch runs, it also shows running and queued runs.
|
After you have used batch runs, it also shows running and queued runs.
|
||||||
A degraded instance lists the problems that need attention.
|
A degraded instance lists the problems that need attention.
|
||||||
|
|
||||||
|
### Execution time per node type
|
||||||
|
|
||||||
|
One bar per node type at its average time per run, slowest first, on a log
|
||||||
|
scale. Each row also shows how often the type ran and its slowest single run,
|
||||||
|
so a slow or busy kind of node stands out.
|
||||||
|
|
||||||
### Activity
|
### Activity
|
||||||
|
|
||||||
Choose 1h, 6h, 24h or 7d to review executions and failures, recent runs,
|
Choose 1h, 6h, 24h or 7d to review executions and failures, recent runs,
|
||||||
|
|||||||
@@ -2657,6 +2657,35 @@ export const NodeTypeInfoSchema = {
|
|||||||
description: 'A node type the editor can offer, with its parameter schema.'
|
description: 'A node type the editor can offer, with its parameter schema.'
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
|
export const NodeTypeTimingSchema = {
|
||||||
|
properties: {
|
||||||
|
type: {
|
||||||
|
type: 'string',
|
||||||
|
title: 'Type'
|
||||||
|
},
|
||||||
|
nodes: {
|
||||||
|
type: 'integer',
|
||||||
|
title: 'Nodes'
|
||||||
|
},
|
||||||
|
executions: {
|
||||||
|
type: 'integer',
|
||||||
|
title: 'Executions'
|
||||||
|
},
|
||||||
|
avg_ms: {
|
||||||
|
type: 'number',
|
||||||
|
title: 'Avg Ms'
|
||||||
|
},
|
||||||
|
max_ms: {
|
||||||
|
type: 'number',
|
||||||
|
title: 'Max Ms'
|
||||||
|
}
|
||||||
|
},
|
||||||
|
type: 'object',
|
||||||
|
required: ['type', 'nodes', 'executions', 'avg_ms', 'max_ms'],
|
||||||
|
title: 'NodeTypeTiming',
|
||||||
|
description: 'How long one kind of node takes to run, across every node of it.'
|
||||||
|
} as const;
|
||||||
|
|
||||||
export const OAuthAuthorizeInfoSchema = {
|
export const OAuthAuthorizeInfoSchema = {
|
||||||
properties: {
|
properties: {
|
||||||
client_name: {
|
client_name: {
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -919,6 +919,17 @@ export type NodeTypeInfo = {
|
|||||||
plugin?: (string | null);
|
plugin?: (string | null);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How long one kind of node takes to run, across every node of it.
|
||||||
|
*/
|
||||||
|
export type NodeTypeTiming = {
|
||||||
|
type: string;
|
||||||
|
nodes: number;
|
||||||
|
executions: number;
|
||||||
|
avg_ms: number;
|
||||||
|
max_ms: number;
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* What the consent page shows, all of it validated server-side.
|
* What the consent page shows, all of it validated server-side.
|
||||||
*/
|
*/
|
||||||
@@ -2025,6 +2036,12 @@ export type ObservabilityReadFlowRollupsData = {
|
|||||||
|
|
||||||
export type ObservabilityReadFlowRollupsResponse = (Array<FlowRollup>);
|
export type ObservabilityReadFlowRollupsResponse = (Array<FlowRollup>);
|
||||||
|
|
||||||
|
export type ObservabilityReadNodeTypeTimingsData = {
|
||||||
|
hours?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ObservabilityReadNodeTypeTimingsResponse = (Array<NodeTypeTiming>);
|
||||||
|
|
||||||
export type ObservabilityReadRunsData = {
|
export type ObservabilityReadRunsData = {
|
||||||
flow?: (string | null);
|
flow?: (string | null);
|
||||||
limit?: number;
|
limit?: number;
|
||||||
|
|||||||
@@ -0,0 +1,153 @@
|
|||||||
|
import { useQuery } from "@tanstack/react-query"
|
||||||
|
|
||||||
|
import type { NodeTypeTiming } from "@/client"
|
||||||
|
import type { Range } from "@/components/Common/RangePicker"
|
||||||
|
import { nodeTypesQueryOptions } from "@/components/Flow/queries"
|
||||||
|
import { PANEL_SECTION } from "@/components/Flow/SidePanel"
|
||||||
|
import { Skeleton } from "@/components/ui/skeleton"
|
||||||
|
import { dur, si } from "@/lib/utils"
|
||||||
|
import { CARD, nodeTypeTimingsQueryOptions } from "./queries"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One row's columns: the type, its bar, its average. A phone puts the bar on a
|
||||||
|
* line of its own under the other two, where it has the card's whole width.
|
||||||
|
*/
|
||||||
|
const COLUMNS =
|
||||||
|
"grid grid-cols-[minmax(0,1fr)_auto] gap-x-3 sm:grid-cols-[12rem_minmax(0,1fr)_5rem]"
|
||||||
|
|
||||||
|
/** The fastest the axis reaches down to, in ms: a microsecond. */
|
||||||
|
const FLOOR = 0.001
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The axis, in whole decades of milliseconds: from below the fastest type to
|
||||||
|
* above the slowest, two decades at least so a bar has room to differ.
|
||||||
|
*/
|
||||||
|
function decades(timings: NodeTypeTiming[]): number[] {
|
||||||
|
const measured = timings.map((t) => t.avg_ms).filter((ms) => ms >= FLOOR)
|
||||||
|
if (!measured.length) return []
|
||||||
|
const low = Math.floor(Math.log10(Math.min(...measured)))
|
||||||
|
const high = Math.max(Math.ceil(Math.log10(Math.max(...measured))), low + 2)
|
||||||
|
return Array.from({ length: high - low + 1 }, (_, i) => 10 ** (low + i))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Where a duration sits along the axis, as a percentage. */
|
||||||
|
function at(ms: number, axis: number[]): number {
|
||||||
|
const low = Math.log10(axis[0])
|
||||||
|
const high = Math.log10(axis[axis.length - 1])
|
||||||
|
return ((Math.log10(ms) - low) / (high - low)) * 100
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How long each kind of node takes per run, on a log scale.
|
||||||
|
*
|
||||||
|
* One bar per node type at its average, slowest first, so a hotspot is the top
|
||||||
|
* row. The scale is logarithmic because node types sit decades apart, a timer
|
||||||
|
* at microseconds and a model at seconds, and a linear one shows only the
|
||||||
|
* slowest. A type that ran without a measurable duration has no bar.
|
||||||
|
*/
|
||||||
|
export function NodeTypeTimes({ range }: { range: Range }) {
|
||||||
|
const { data, isPending } = useQuery(nodeTypeTimingsQueryOptions(range))
|
||||||
|
const { data: types } = useQuery(nodeTypesQueryOptions())
|
||||||
|
const titles = new Map((types ?? []).map((info) => [info.type, info.title]))
|
||||||
|
|
||||||
|
const timings = data ?? []
|
||||||
|
const axis = decades(timings)
|
||||||
|
|
||||||
|
return (
|
||||||
|
<section className="grid gap-3" data-testid="node-type-times">
|
||||||
|
<div className="flex flex-wrap items-baseline gap-x-3">
|
||||||
|
<h2 className={PANEL_SECTION}>Execution time per node type</h2>
|
||||||
|
<p className="text-muted-foreground text-xs">
|
||||||
|
average per run over the last {range.label}, log scale
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className={CARD}>
|
||||||
|
{isPending ? (
|
||||||
|
<div className="grid gap-3">
|
||||||
|
<Skeleton className="h-5 w-2/3" />
|
||||||
|
<Skeleton className="h-5 w-1/2" />
|
||||||
|
</div>
|
||||||
|
) : timings.length === 0 ? (
|
||||||
|
<p className="text-muted-foreground text-sm">Nothing has run yet.</p>
|
||||||
|
) : (
|
||||||
|
<div className="grid">
|
||||||
|
{timings.map((timing) => (
|
||||||
|
<Row
|
||||||
|
key={timing.type}
|
||||||
|
timing={timing}
|
||||||
|
title={titles.get(timing.type) ?? timing.type}
|
||||||
|
axis={axis}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
{axis.length ? (
|
||||||
|
<div className={COLUMNS}>
|
||||||
|
<div className="relative col-span-2 mt-1 h-4 text-muted-foreground text-xs sm:col-span-1 sm:col-start-2">
|
||||||
|
{axis.map((ms, index) => (
|
||||||
|
<span
|
||||||
|
key={ms}
|
||||||
|
className="absolute top-0 whitespace-nowrap"
|
||||||
|
style={{
|
||||||
|
left: `${at(ms, axis)}%`,
|
||||||
|
// The ends stay inside the track; the rest centre on it.
|
||||||
|
transform:
|
||||||
|
index === 0
|
||||||
|
? undefined
|
||||||
|
: index === axis.length - 1
|
||||||
|
? "translateX(-100%)"
|
||||||
|
: "translateX(-50%)",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{dur(ms)}
|
||||||
|
</span>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function Row({
|
||||||
|
timing,
|
||||||
|
title,
|
||||||
|
axis,
|
||||||
|
}: {
|
||||||
|
timing: NodeTypeTiming
|
||||||
|
title: string
|
||||||
|
axis: number[]
|
||||||
|
}) {
|
||||||
|
const width =
|
||||||
|
timing.avg_ms >= FLOOR && axis.length ? at(timing.avg_ms, axis) : 0
|
||||||
|
return (
|
||||||
|
<div className={COLUMNS}>
|
||||||
|
<div className="col-start-1 row-start-1 min-w-0 pt-1.5 sm:py-1.5">
|
||||||
|
<p className="truncate text-sm">{title}</p>
|
||||||
|
<p className="truncate text-muted-foreground text-xs">
|
||||||
|
{si(timing.executions)} runs · peak {dur(timing.max_ms)}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
{/* The decade lines run through every row, so together they read as
|
||||||
|
the grid; the bar starts at the axis's own floor. */}
|
||||||
|
<div className="relative col-span-2 row-start-2 h-4 sm:col-span-1 sm:col-start-2 sm:row-start-1 sm:h-full">
|
||||||
|
{axis.map((ms) => (
|
||||||
|
<span
|
||||||
|
key={ms}
|
||||||
|
className="absolute inset-y-0 w-px bg-border"
|
||||||
|
style={{ left: `${at(ms, axis)}%` }}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
<span
|
||||||
|
className="absolute top-1/2 left-0 h-2.5 -translate-y-1/2 rounded-e-[4px] bg-chart-1"
|
||||||
|
style={{ width: `${width}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<span className="col-start-2 row-start-1 self-center whitespace-nowrap text-right font-mono text-sm sm:col-start-3">
|
||||||
|
{dur(timing.avg_ms)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -147,3 +147,10 @@ export const clock = (ts: number) =>
|
|||||||
hour: "2-digit",
|
hour: "2-digit",
|
||||||
minute: "2-digit",
|
minute: "2-digit",
|
||||||
})
|
})
|
||||||
|
|
||||||
|
export const nodeTypeTimingsQueryOptions = (range: Range) => ({
|
||||||
|
queryKey: ["observability", "node-types", range.hours] as const,
|
||||||
|
queryFn: () =>
|
||||||
|
ObservabilityService.readNodeTypeTimings({ hours: range.hours }),
|
||||||
|
refetchInterval: REFRESH,
|
||||||
|
})
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import { FlowTable } from "@/components/Health/FlowTable"
|
|||||||
import { HealthActivity } from "@/components/Health/HealthActivity"
|
import { HealthActivity } from "@/components/Health/HealthActivity"
|
||||||
import { HealthOverview } from "@/components/Health/HealthOverview"
|
import { HealthOverview } from "@/components/Health/HealthOverview"
|
||||||
import { LiveIndicator } from "@/components/Health/LiveIndicator"
|
import { LiveIndicator } from "@/components/Health/LiveIndicator"
|
||||||
|
import { NodeTypeTimes } from "@/components/Health/NodeTypeTimes"
|
||||||
import { Card } from "@/components/ui/card"
|
import { Card } from "@/components/ui/card"
|
||||||
|
|
||||||
export const Route = createFileRoute("/_layout/")({
|
export const Route = createFileRoute("/_layout/")({
|
||||||
@@ -35,8 +36,8 @@ const HEADER =
|
|||||||
* than living at routes of their own.
|
* than living at routes of their own.
|
||||||
*
|
*
|
||||||
* Top to bottom it is a widening lens: the whole instance as a graph, what
|
* Top to bottom it is a widening lens: the whole instance as a graph, what
|
||||||
* has been built on it, whether it is well, then flow by flow and finally
|
* has been built on it, whether it is well, then flow by flow, node type by
|
||||||
* moment by moment.
|
* node type, and finally moment by moment.
|
||||||
*/
|
*/
|
||||||
function Home() {
|
function Home() {
|
||||||
// The health block's window: one choice, read by the tiles, the flow table,
|
// The health block's window: one choice, read by the tiles, the flow table,
|
||||||
@@ -91,6 +92,7 @@ function Home() {
|
|||||||
|
|
||||||
<HealthOverview range={range} onRangeChange={setRange} />
|
<HealthOverview range={range} onRangeChange={setRange} />
|
||||||
<FlowTable range={range} />
|
<FlowTable range={range} />
|
||||||
|
<NodeTypeTimes range={range} />
|
||||||
<HealthActivity range={range} />
|
<HealthActivity range={range} />
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user