Keep the engine's own history, and a screen that reads it
A second bus subscriber folds executions, errors, timings and queue lag into per-minute rollups, keeps failures with their traceback and an audit trail of who published what, and records one row per cascade — manual runs and previews included, under an id of their own that writes no idempotency markers. Read back through /observability/*, which always answers 200 so a degraded engine still renders its own health screen. Also fixes two things found on the way: node-health alerts read `status` where the engine publishes `health`, so a device dropping never alerted anyone, and the Redis queue reported `parked: 0` whatever was held. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt
This commit is contained in:
Claude Fable 5
@@ -379,6 +379,38 @@ export const DashboardsPublicSchema = {
|
||||
title: 'DashboardsPublic'
|
||||
} as const;
|
||||
|
||||
export const DeadLetterSchema = {
|
||||
properties: {
|
||||
id: {
|
||||
type: 'string',
|
||||
title: 'Id'
|
||||
},
|
||||
ts: {
|
||||
type: 'number',
|
||||
title: 'Ts'
|
||||
},
|
||||
flow: {
|
||||
type: 'string',
|
||||
title: 'Flow'
|
||||
},
|
||||
node: {
|
||||
type: 'string',
|
||||
title: 'Node'
|
||||
},
|
||||
cause: {
|
||||
type: 'string',
|
||||
title: 'Cause'
|
||||
},
|
||||
reason: {
|
||||
type: 'string',
|
||||
title: 'Reason'
|
||||
}
|
||||
},
|
||||
type: 'object',
|
||||
required: ['id', 'ts', 'flow', 'node', 'cause', 'reason'],
|
||||
title: 'DeadLetter'
|
||||
} as const;
|
||||
|
||||
export const EndpointSchema = {
|
||||
properties: {
|
||||
kind: {
|
||||
@@ -426,6 +458,43 @@ these so a value never appears to come from nowhere — or worse, appears to
|
||||
come from whichever node happens to be drawn as a producer.`
|
||||
} as const;
|
||||
|
||||
export const EventRowSchema = {
|
||||
properties: {
|
||||
id: {
|
||||
type: 'integer',
|
||||
title: 'Id'
|
||||
},
|
||||
ts: {
|
||||
type: 'string',
|
||||
format: 'date-time',
|
||||
title: 'Ts'
|
||||
},
|
||||
type: {
|
||||
type: 'string',
|
||||
title: 'Type'
|
||||
},
|
||||
flow: {
|
||||
type: 'string',
|
||||
title: 'Flow'
|
||||
},
|
||||
node: {
|
||||
type: 'string',
|
||||
title: 'Node'
|
||||
},
|
||||
detail: {
|
||||
type: 'string',
|
||||
title: 'Detail'
|
||||
},
|
||||
actor: {
|
||||
type: 'string',
|
||||
title: 'Actor'
|
||||
}
|
||||
},
|
||||
type: 'object',
|
||||
required: ['id', 'ts', 'type', 'flow', 'node', 'detail', 'actor'],
|
||||
title: 'EventRow'
|
||||
} as const;
|
||||
|
||||
export const FlowDef_InputSchema = {
|
||||
properties: {
|
||||
name: {
|
||||
@@ -597,6 +666,56 @@ export const FlowInput_OutputSchema = {
|
||||
description: 'A message the flow starts with rather than computes.'
|
||||
} as const;
|
||||
|
||||
export const FlowRollupSchema = {
|
||||
properties: {
|
||||
flow: {
|
||||
type: 'string',
|
||||
title: 'Flow'
|
||||
},
|
||||
executions: {
|
||||
type: 'integer',
|
||||
title: 'Executions'
|
||||
},
|
||||
errors: {
|
||||
type: 'integer',
|
||||
title: 'Errors'
|
||||
},
|
||||
messages: {
|
||||
type: 'integer',
|
||||
title: 'Messages'
|
||||
},
|
||||
avg_ms: {
|
||||
type: 'number',
|
||||
title: 'Avg Ms'
|
||||
},
|
||||
avg_lag_ms: {
|
||||
type: 'number',
|
||||
title: 'Avg Lag Ms'
|
||||
},
|
||||
spark: {
|
||||
items: {
|
||||
type: 'integer'
|
||||
},
|
||||
type: 'array',
|
||||
title: 'Spark'
|
||||
},
|
||||
last_error_ts: {
|
||||
anyOf: [
|
||||
{
|
||||
type: 'number'
|
||||
},
|
||||
{
|
||||
type: 'null'
|
||||
}
|
||||
],
|
||||
title: 'Last Error Ts'
|
||||
}
|
||||
},
|
||||
type: 'object',
|
||||
required: ['flow', 'executions', 'errors', 'messages', 'avg_ms', 'avg_lag_ms', 'spark'],
|
||||
title: 'FlowRollup'
|
||||
} as const;
|
||||
|
||||
export const FlowStatePublicSchema = {
|
||||
properties: {
|
||||
values: {
|
||||
@@ -698,6 +817,55 @@ export const HTTPValidationErrorSchema = {
|
||||
title: 'HTTPValidationError'
|
||||
} as const;
|
||||
|
||||
export const HealthSummarySchema = {
|
||||
properties: {
|
||||
status: {
|
||||
type: 'string',
|
||||
title: 'Status'
|
||||
},
|
||||
problems: {
|
||||
items: {
|
||||
type: 'string'
|
||||
},
|
||||
type: 'array',
|
||||
title: 'Problems'
|
||||
},
|
||||
flows: {
|
||||
additionalProperties: {
|
||||
type: 'integer'
|
||||
},
|
||||
type: 'object',
|
||||
title: 'Flows'
|
||||
},
|
||||
nodes: {
|
||||
additionalProperties: {
|
||||
type: 'integer'
|
||||
},
|
||||
type: 'object',
|
||||
title: 'Nodes'
|
||||
},
|
||||
queue: {
|
||||
additionalProperties: true,
|
||||
type: 'object',
|
||||
title: 'Queue'
|
||||
},
|
||||
loop_lag: {
|
||||
additionalProperties: {
|
||||
type: 'number'
|
||||
},
|
||||
type: 'object',
|
||||
title: 'Loop Lag'
|
||||
},
|
||||
failures_24h: {
|
||||
type: 'integer',
|
||||
title: 'Failures 24H'
|
||||
}
|
||||
},
|
||||
type: 'object',
|
||||
required: ['status', 'problems', 'flows', 'nodes', 'queue', 'loop_lag', 'failures_24h'],
|
||||
title: 'HealthSummary'
|
||||
} as const;
|
||||
|
||||
export const HistoryPointSchema = {
|
||||
properties: {
|
||||
ts: {
|
||||
@@ -1630,6 +1798,63 @@ export const RunRequestSchema = {
|
||||
title: 'RunRequest'
|
||||
} as const;
|
||||
|
||||
export const RunRowSchema = {
|
||||
properties: {
|
||||
id: {
|
||||
type: 'string',
|
||||
title: 'Id'
|
||||
},
|
||||
flow: {
|
||||
type: 'string',
|
||||
title: 'Flow'
|
||||
},
|
||||
source: {
|
||||
type: 'string',
|
||||
title: 'Source'
|
||||
},
|
||||
status: {
|
||||
type: 'string',
|
||||
title: 'Status'
|
||||
},
|
||||
started_at: {
|
||||
type: 'string',
|
||||
format: 'date-time',
|
||||
title: 'Started At'
|
||||
},
|
||||
finished_at: {
|
||||
anyOf: [
|
||||
{
|
||||
type: 'string',
|
||||
format: 'date-time'
|
||||
},
|
||||
{
|
||||
type: 'null'
|
||||
}
|
||||
],
|
||||
title: 'Finished At'
|
||||
},
|
||||
nodes: {
|
||||
type: 'integer',
|
||||
title: 'Nodes'
|
||||
},
|
||||
errors: {
|
||||
type: 'integer',
|
||||
title: 'Errors'
|
||||
},
|
||||
duration_ms: {
|
||||
type: 'number',
|
||||
title: 'Duration Ms'
|
||||
},
|
||||
deliveries: {
|
||||
type: 'integer',
|
||||
title: 'Deliveries'
|
||||
}
|
||||
},
|
||||
type: 'object',
|
||||
required: ['id', 'flow', 'source', 'status', 'started_at', 'nodes', 'errors', 'duration_ms', 'deliveries'],
|
||||
title: 'RunRow'
|
||||
} as const;
|
||||
|
||||
export const SecretNamesSchema = {
|
||||
properties: {
|
||||
data: {
|
||||
@@ -1711,6 +1936,42 @@ export const SectionDef_OutputSchema = {
|
||||
description: 'A grid of widgets under a heading.'
|
||||
} as const;
|
||||
|
||||
export const SeriesPointSchema = {
|
||||
properties: {
|
||||
ts: {
|
||||
type: 'number',
|
||||
title: 'Ts'
|
||||
},
|
||||
executions: {
|
||||
type: 'integer',
|
||||
title: 'Executions'
|
||||
},
|
||||
errors: {
|
||||
type: 'integer',
|
||||
title: 'Errors'
|
||||
},
|
||||
messages: {
|
||||
type: 'integer',
|
||||
title: 'Messages'
|
||||
},
|
||||
avg_ms: {
|
||||
type: 'number',
|
||||
title: 'Avg Ms'
|
||||
},
|
||||
max_ms: {
|
||||
type: 'number',
|
||||
title: 'Max Ms'
|
||||
},
|
||||
avg_lag_ms: {
|
||||
type: 'number',
|
||||
title: 'Avg Lag Ms'
|
||||
}
|
||||
},
|
||||
type: 'object',
|
||||
required: ['ts', 'executions', 'errors', 'messages', 'avg_ms', 'max_ms', 'avg_lag_ms'],
|
||||
title: 'SeriesPoint'
|
||||
} as const;
|
||||
|
||||
export const ShareRequestSchema = {
|
||||
properties: {
|
||||
lib_name: {
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import type { CancelablePromise } from './core/CancelablePromise';
|
||||
import { OpenAPI } from './core/OpenAPI';
|
||||
import { request as __request } from './core/request';
|
||||
import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsTestChannelData, AlertsTestChannelResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, PrivateCreateUserData, PrivateCreateUserResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse } from './types.gen';
|
||||
import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsTestChannelData, AlertsTestChannelResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, PrivateCreateUserData, PrivateCreateUserResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse } from './types.gen';
|
||||
|
||||
export class AlertsService {
|
||||
/**
|
||||
@@ -1069,6 +1069,140 @@ export class OauthService {
|
||||
}
|
||||
}
|
||||
|
||||
export class ObservabilityService {
|
||||
/**
|
||||
* Read Summary
|
||||
* How the engine is doing right now. Always 200, degraded or not.
|
||||
* @returns HealthSummary Successful Response
|
||||
* @throws ApiError
|
||||
*/
|
||||
public static readSummary(): CancelablePromise<ObservabilityReadSummaryResponse> {
|
||||
return __request(OpenAPI, {
|
||||
method: 'GET',
|
||||
url: '/api/v1/observability/summary'
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Read Timeseries
|
||||
* Executions, errors and timings over time, summed across nodes.
|
||||
* @param data The data for the request.
|
||||
* @param data.flow
|
||||
* @param data.node
|
||||
* @param data.hours
|
||||
* @param data.bucketS
|
||||
* @returns SeriesPoint Successful Response
|
||||
* @throws ApiError
|
||||
*/
|
||||
public static readTimeseries(data: ObservabilityReadTimeseriesData = {}): CancelablePromise<ObservabilityReadTimeseriesResponse> {
|
||||
return __request(OpenAPI, {
|
||||
method: 'GET',
|
||||
url: '/api/v1/observability/timeseries',
|
||||
query: {
|
||||
flow: data.flow,
|
||||
node: data.node,
|
||||
hours: data.hours,
|
||||
bucket_s: data.bucketS
|
||||
},
|
||||
errors: {
|
||||
422: 'Validation Error'
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Read Flow Rollups
|
||||
* One row per flow, with a coarse trend of how much it ran.
|
||||
* @param data The data for the request.
|
||||
* @param data.hours
|
||||
* @returns FlowRollup Successful Response
|
||||
* @throws ApiError
|
||||
*/
|
||||
public static readFlowRollups(data: ObservabilityReadFlowRollupsData = {}): CancelablePromise<ObservabilityReadFlowRollupsResponse> {
|
||||
return __request(OpenAPI, {
|
||||
method: 'GET',
|
||||
url: '/api/v1/observability/flows',
|
||||
query: {
|
||||
hours: data.hours
|
||||
},
|
||||
errors: {
|
||||
422: 'Validation Error'
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Read Runs
|
||||
* Recent cascades, newest first.
|
||||
* @param data The data for the request.
|
||||
* @param data.flow
|
||||
* @param data.status
|
||||
* @param data.limit
|
||||
* @returns RunRow Successful Response
|
||||
* @throws ApiError
|
||||
*/
|
||||
public static readRuns(data: ObservabilityReadRunsData = {}): CancelablePromise<ObservabilityReadRunsResponse> {
|
||||
return __request(OpenAPI, {
|
||||
method: 'GET',
|
||||
url: '/api/v1/observability/runs',
|
||||
query: {
|
||||
flow: data.flow,
|
||||
status: data.status,
|
||||
limit: data.limit
|
||||
},
|
||||
errors: {
|
||||
422: 'Validation Error'
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Read Events
|
||||
* What went wrong, or who changed what. Newest first.
|
||||
* @param data The data for the request.
|
||||
* @param data.kind
|
||||
* @param data.flow
|
||||
* @param data.limit
|
||||
* @returns EventRow Successful Response
|
||||
* @throws ApiError
|
||||
*/
|
||||
public static readEvents(data: ObservabilityReadEventsData = {}): CancelablePromise<ObservabilityReadEventsResponse> {
|
||||
return __request(OpenAPI, {
|
||||
method: 'GET',
|
||||
url: '/api/v1/observability/events',
|
||||
query: {
|
||||
kind: data.kind,
|
||||
flow: data.flow,
|
||||
limit: data.limit
|
||||
},
|
||||
errors: {
|
||||
422: 'Validation Error'
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Read Dead Letters
|
||||
* Work the engine gave up on, which nothing else surfaces.
|
||||
* @param data The data for the request.
|
||||
* @param data.limit
|
||||
* @returns DeadLetter Successful Response
|
||||
* @throws ApiError
|
||||
*/
|
||||
public static readDeadLetters(data: ObservabilityReadDeadLettersData = {}): CancelablePromise<ObservabilityReadDeadLettersResponse> {
|
||||
return __request(OpenAPI, {
|
||||
method: 'GET',
|
||||
url: '/api/v1/observability/dead-letter',
|
||||
query: {
|
||||
limit: data.limit
|
||||
},
|
||||
errors: {
|
||||
422: 'Validation Error'
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export class PrivateService {
|
||||
/**
|
||||
* Create User
|
||||
|
||||
@@ -129,6 +129,15 @@ export type DashboardSummary = {
|
||||
widget_count?: number;
|
||||
};
|
||||
|
||||
export type DeadLetter = {
|
||||
id: string;
|
||||
ts: number;
|
||||
flow: string;
|
||||
node: string;
|
||||
cause: string;
|
||||
reason: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Serializable payload types.
|
||||
*
|
||||
@@ -154,6 +163,16 @@ export type Endpoint = {
|
||||
requires?: Array<(string)>;
|
||||
};
|
||||
|
||||
export type EventRow = {
|
||||
id: number;
|
||||
ts: string;
|
||||
type: string;
|
||||
flow: string;
|
||||
node: string;
|
||||
detail: string;
|
||||
actor: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* One atomic flow.
|
||||
*/
|
||||
@@ -209,6 +228,17 @@ export type FlowInput_Output = {
|
||||
initial?: (unknown | null);
|
||||
};
|
||||
|
||||
export type FlowRollup = {
|
||||
flow: string;
|
||||
executions: number;
|
||||
errors: number;
|
||||
messages: number;
|
||||
avg_ms: number;
|
||||
avg_lag_ms: number;
|
||||
spark: Array<(number)>;
|
||||
last_error_ts?: (number | null);
|
||||
};
|
||||
|
||||
export type FlowsPublic = {
|
||||
data: Array<FlowSummary>;
|
||||
count: number;
|
||||
@@ -232,6 +262,24 @@ export type FlowSummary = {
|
||||
quarantined?: boolean;
|
||||
};
|
||||
|
||||
export type HealthSummary = {
|
||||
status: string;
|
||||
problems: Array<(string)>;
|
||||
flows: {
|
||||
[key: string]: (number);
|
||||
};
|
||||
nodes: {
|
||||
[key: string]: (number);
|
||||
};
|
||||
queue: {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
loop_lag: {
|
||||
[key: string]: (number);
|
||||
};
|
||||
failures_24h: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* One numeric value a message carried, and when.
|
||||
*/
|
||||
@@ -530,6 +578,19 @@ export type RunRequest = {
|
||||
};
|
||||
};
|
||||
|
||||
export type RunRow = {
|
||||
id: string;
|
||||
flow: string;
|
||||
source: string;
|
||||
status: string;
|
||||
started_at: string;
|
||||
finished_at?: (string | null);
|
||||
nodes: number;
|
||||
errors: number;
|
||||
duration_ms: number;
|
||||
deliveries: number;
|
||||
};
|
||||
|
||||
export type SecretNames = {
|
||||
data: Array<(string)>;
|
||||
count: number;
|
||||
@@ -557,6 +618,16 @@ export type SectionDef_Output = {
|
||||
widgets?: Array<WidgetDef>;
|
||||
};
|
||||
|
||||
export type SeriesPoint = {
|
||||
ts: number;
|
||||
executions: number;
|
||||
errors: number;
|
||||
messages: number;
|
||||
avg_ms: number;
|
||||
max_ms: number;
|
||||
avg_lag_ms: number;
|
||||
};
|
||||
|
||||
export type ShareRequest = {
|
||||
lib_name: string;
|
||||
};
|
||||
@@ -946,6 +1017,45 @@ export type OauthRevokeClientData = {
|
||||
|
||||
export type OauthRevokeClientResponse = (Message);
|
||||
|
||||
export type ObservabilityReadSummaryResponse = (HealthSummary);
|
||||
|
||||
export type ObservabilityReadTimeseriesData = {
|
||||
bucketS?: number;
|
||||
flow?: (string | null);
|
||||
hours?: number;
|
||||
node?: (string | null);
|
||||
};
|
||||
|
||||
export type ObservabilityReadTimeseriesResponse = (Array<SeriesPoint>);
|
||||
|
||||
export type ObservabilityReadFlowRollupsData = {
|
||||
hours?: number;
|
||||
};
|
||||
|
||||
export type ObservabilityReadFlowRollupsResponse = (Array<FlowRollup>);
|
||||
|
||||
export type ObservabilityReadRunsData = {
|
||||
flow?: (string | null);
|
||||
limit?: number;
|
||||
status?: (string | null);
|
||||
};
|
||||
|
||||
export type ObservabilityReadRunsResponse = (Array<RunRow>);
|
||||
|
||||
export type ObservabilityReadEventsData = {
|
||||
flow?: (string | null);
|
||||
kind?: 'failure' | 'audit';
|
||||
limit?: number;
|
||||
};
|
||||
|
||||
export type ObservabilityReadEventsResponse = (Array<EventRow>);
|
||||
|
||||
export type ObservabilityReadDeadLettersData = {
|
||||
limit?: number;
|
||||
};
|
||||
|
||||
export type ObservabilityReadDeadLettersResponse = (Array<DeadLetter>);
|
||||
|
||||
export type PrivateCreateUserData = {
|
||||
requestBody: PrivateUserCreate;
|
||||
};
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
import { useEffect, useLayoutEffect, useRef } from "react"
|
||||
import uPlot from "uplot"
|
||||
import "uplot/dist/uPlot.min.css"
|
||||
|
||||
import type { HistoryPoint } from "@/client"
|
||||
import { useTheme } from "@/components/theme-provider"
|
||||
|
||||
/**
|
||||
* How many lines one chart carries.
|
||||
*
|
||||
* The bound is the palette's: `--chart-1…5` is one designed ramp, and a sixth
|
||||
* line would either repeat a step or invent a colour outside the system.
|
||||
*/
|
||||
export const MAX_SERIES = 5
|
||||
|
||||
/** Room for the axis ticks; uPlot measures the rest of the box itself. */
|
||||
const PADDING: uPlot.Padding = [10, 12, 0, 0]
|
||||
|
||||
/** The legend sits under the canvas, so the canvas has to leave it room. */
|
||||
const LEGEND_HEIGHT = 26
|
||||
|
||||
const canvasHeight = (element: HTMLElement) =>
|
||||
Math.max(60, (element.clientHeight || 180) - LEGEND_HEIGHT)
|
||||
|
||||
/** A token, resolved for the canvas — which cannot read CSS variables. */
|
||||
function token(name: string): string {
|
||||
return getComputedStyle(document.documentElement)
|
||||
.getPropertyValue(name)
|
||||
.trim()
|
||||
}
|
||||
|
||||
const seriesColor = (index: number) => token(`--chart-${(index % 5) + 1}`)
|
||||
|
||||
/** The series joined onto one x axis, which is what uPlot draws. */
|
||||
function table(plots: HistoryPoint[][]): uPlot.AlignedData {
|
||||
return uPlot.join(
|
||||
plots.map(
|
||||
(plot) =>
|
||||
[
|
||||
plot.map((point) => point.ts),
|
||||
plot.map((point) => point.value),
|
||||
] as uPlot.AlignedData,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Several series over time, drawn on one axis.
|
||||
*
|
||||
* uPlot rather than SVG: a chart may hold five series of hundreds of points
|
||||
* each, which is more path data than React should be rebuilding on every value
|
||||
* that arrives. Its own legend doubles as the hover readout, so the cursor
|
||||
* tells you what each line was worth at that moment — and with more than one
|
||||
* line a legend is required anyway.
|
||||
*/
|
||||
export function UplotChart({
|
||||
labels,
|
||||
plots,
|
||||
empty = "Nothing has come through yet.",
|
||||
}: {
|
||||
/** One label per series; the set of them is the chart's identity. */
|
||||
labels: string[]
|
||||
/** The points of each series, in the same order as `labels`. */
|
||||
plots: HistoryPoint[][]
|
||||
empty?: string
|
||||
}) {
|
||||
const host = useRef<HTMLDivElement>(null)
|
||||
const chart = useRef<uPlot | null>(null)
|
||||
const { resolvedTheme } = useTheme()
|
||||
|
||||
const points = plots.reduce((total, plot) => total + plot.length, 0)
|
||||
// The identity of the series set: the chart is rebuilt when it changes,
|
||||
// while a new reading only sets its data.
|
||||
const key = labels.join(" ")
|
||||
// uPlot leaves its axes half-initialised while the scales have no range, and
|
||||
// a resize in that window (a card still settling, say) draws them anyway and
|
||||
// throws. Waiting for the first reading avoids the state altogether.
|
||||
const ready = points > 0
|
||||
|
||||
// biome-ignore lint/correctness/useExhaustiveDependencies: the label string is the identity of the series set.
|
||||
useLayoutEffect(() => {
|
||||
const element = host.current
|
||||
if (!element || labels.length === 0 || !ready) return
|
||||
|
||||
const axis = {
|
||||
stroke: () => token("--muted-foreground"),
|
||||
grid: { stroke: () => token("--border"), width: 1 },
|
||||
ticks: { stroke: () => token("--border"), width: 1 },
|
||||
font: `11px ${getComputedStyle(element).fontFamily}`,
|
||||
}
|
||||
|
||||
const plot = new uPlot(
|
||||
{
|
||||
width: element.clientWidth || 320,
|
||||
height: canvasHeight(element),
|
||||
padding: PADDING,
|
||||
cursor: { y: false },
|
||||
legend: { live: true },
|
||||
scales: { x: { time: true } },
|
||||
axes: [
|
||||
{ ...axis, size: 28 },
|
||||
{ ...axis, size: 46 },
|
||||
],
|
||||
series: [
|
||||
{},
|
||||
...labels.map((label, index) => ({
|
||||
label,
|
||||
width: 2,
|
||||
// Read at draw time, so a theme toggle is a redraw rather than a
|
||||
// rebuilt chart.
|
||||
stroke: () => seriesColor(index),
|
||||
// Series arrive on their own clocks; a joined table is mostly
|
||||
// holes, and a line with a hole per point is not a line.
|
||||
spanGaps: true,
|
||||
points: { show: false },
|
||||
})),
|
||||
],
|
||||
},
|
||||
// Built with the readings it already has: uPlot's axes are only half
|
||||
// initialised while its scales have no range.
|
||||
table(plots),
|
||||
element,
|
||||
)
|
||||
chart.current = plot
|
||||
|
||||
const observer = new ResizeObserver(() => {
|
||||
plot.setSize({
|
||||
width: element.clientWidth,
|
||||
height: canvasHeight(element),
|
||||
})
|
||||
})
|
||||
observer.observe(element)
|
||||
|
||||
return () => {
|
||||
observer.disconnect()
|
||||
plot.destroy()
|
||||
chart.current = null
|
||||
}
|
||||
}, [key, ready])
|
||||
|
||||
// biome-ignore lint/correctness/useExhaustiveDependencies: rebuilding the joined table is what the point count stands for.
|
||||
useEffect(() => {
|
||||
if (!chart.current || plots.length === 0) return
|
||||
chart.current.setData(table(plots))
|
||||
}, [points, key])
|
||||
|
||||
// The canvas cannot follow a CSS variable, so a theme swap is a redraw.
|
||||
// biome-ignore lint/correctness/useExhaustiveDependencies: the theme is the signal, not something the effect reads.
|
||||
useEffect(() => {
|
||||
chart.current?.redraw()
|
||||
}, [resolvedTheme])
|
||||
|
||||
return (
|
||||
<div className="relative min-h-0 flex-1">
|
||||
<div ref={host} className="absolute inset-0" />
|
||||
{points === 0 ? (
|
||||
<p className="absolute inset-0 flex items-center justify-center text-sm text-muted-foreground">
|
||||
{empty}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Binary file not shown.
@@ -49,7 +49,7 @@ function useSettled(value: string): string {
|
||||
* that never moved has no span to divide by, and one spanning decades is only
|
||||
* legible once the exponent is what varies.
|
||||
*/
|
||||
function shape(points: HistoryPoint[]) {
|
||||
export function shape(points: HistoryPoint[]) {
|
||||
const values = points.map((point) => point.value)
|
||||
const low = Math.min(...values)
|
||||
const high = Math.max(...values)
|
||||
|
||||
@@ -24,6 +24,19 @@ export type LiveStatus = {
|
||||
status: "active" | "error" | "running" | "success"
|
||||
error?: string | null
|
||||
}
|
||||
/** How a node's connection is doing, which is not how its last run went. */
|
||||
export type NodeHealth = {
|
||||
health: "ok" | "down" | "unknown"
|
||||
detail?: string | null
|
||||
}
|
||||
/** Something the engine reported about itself, for the health page. */
|
||||
export type EngineEvent = {
|
||||
type: string
|
||||
flow?: string
|
||||
node?: string
|
||||
detail?: string
|
||||
ts: number
|
||||
}
|
||||
/** One node execution's output, as the log panel shows it. */
|
||||
export type LogLine = {
|
||||
flow: string
|
||||
@@ -38,9 +51,13 @@ type Listener = () => void
|
||||
|
||||
/** Enough to see what a flow has been doing, not a log store. */
|
||||
const LOG_LIMIT = 500
|
||||
/** The health page reads these to know when to refetch; it is not a history. */
|
||||
const ENGINE_EVENT_LIMIT = 100
|
||||
|
||||
const values = new Map<string, LiveValue>()
|
||||
const statuses = new Map<string, LiveStatus>()
|
||||
const health = new Map<string, NodeHealth>()
|
||||
let engineEvents: EngineEvent[] = []
|
||||
// How many times a node has emitted. The number itself means nothing; a change
|
||||
// is what restarts the pulse.
|
||||
const emits = new Map<string, number>()
|
||||
@@ -100,6 +117,18 @@ export const liveStore = {
|
||||
getStatus(nodeId: string) {
|
||||
return statuses.get(nodeId)
|
||||
},
|
||||
setHealth(nodeId: string, entry: NodeHealth) {
|
||||
health.set(nodeId, entry)
|
||||
notify(`health:${nodeId}`)
|
||||
},
|
||||
getHealth(nodeId: string) {
|
||||
return health.get(nodeId)
|
||||
},
|
||||
recordEngineEvent(event: EngineEvent) {
|
||||
// A new array each time, so the hook's snapshot comparison sees the change.
|
||||
engineEvents = [...engineEvents, event].slice(-ENGINE_EVENT_LIMIT)
|
||||
notify("engine")
|
||||
},
|
||||
recordEmit(nodeId: string) {
|
||||
emits.set(nodeId, (emits.get(nodeId) ?? 0) + 1)
|
||||
notify(`emit:${nodeId}`)
|
||||
@@ -146,6 +175,10 @@ export const liveStore = {
|
||||
statuses.clear()
|
||||
for (const key of emits.keys()) notify(`emit:${key}`)
|
||||
emits.clear()
|
||||
for (const key of health.keys()) notify(`health:${key}`)
|
||||
health.clear()
|
||||
engineEvents = []
|
||||
notify("engine")
|
||||
logLines = []
|
||||
notify("logs")
|
||||
for (const flow of paused) notify(`paused:${flow}`)
|
||||
@@ -167,6 +200,22 @@ export function useNodeStatus(nodeId: string): LiveStatus | undefined {
|
||||
)
|
||||
}
|
||||
|
||||
/** How the node's connection is doing, once it has said anything about it. */
|
||||
export function useNodeHealth(nodeId: string): NodeHealth | undefined {
|
||||
return useSyncExternalStore(
|
||||
(listener) => subscribeKey(`health:${nodeId}`, listener),
|
||||
() => health.get(nodeId),
|
||||
)
|
||||
}
|
||||
|
||||
/** The last hundred things the engine said about itself, oldest first. */
|
||||
export function useEngineEvents(): EngineEvent[] {
|
||||
return useSyncExternalStore(
|
||||
(listener) => subscribeKey("engine", listener),
|
||||
() => engineEvents,
|
||||
)
|
||||
}
|
||||
|
||||
/** Increments each time the node publishes something. */
|
||||
export function useNodeEmits(nodeId: string): number {
|
||||
return useSyncExternalStore(
|
||||
|
||||
@@ -27,11 +27,48 @@ type FlowEvent =
|
||||
source?: ValueSource
|
||||
}
|
||||
| { type: "node_started"; node: string }
|
||||
| { type: "node_executed"; node: string; outputs: number }
|
||||
| { type: "node_error"; node: string; error: string }
|
||||
| {
|
||||
type: "node_executed"
|
||||
flow?: string
|
||||
node: string
|
||||
outputs: number
|
||||
duration_ms?: number
|
||||
}
|
||||
| {
|
||||
type: "node_error"
|
||||
flow?: string
|
||||
node: string
|
||||
error: string
|
||||
ts?: number
|
||||
}
|
||||
| { type: "node_status"; node: string; status: string; error?: string | null }
|
||||
| ({ type: "node_log" } & LogLine)
|
||||
| { type: "flow_paused"; flow: string; paused: boolean }
|
||||
| {
|
||||
type: "node_health"
|
||||
flow?: string
|
||||
node: string
|
||||
health: "ok" | "down" | "unknown"
|
||||
detail?: string | null
|
||||
ts?: number
|
||||
}
|
||||
| { type: "flow_quarantined"; flow: string; error?: string; ts?: number }
|
||||
| { type: "engine_degraded"; reason?: string; ts?: number }
|
||||
| { type: "engine_fatal"; reason?: string; ts?: number }
|
||||
| {
|
||||
type: "cascade_dropped"
|
||||
flow?: string
|
||||
node?: string
|
||||
deliveries?: number
|
||||
ts?: number
|
||||
}
|
||||
| {
|
||||
type: "queue_unavailable"
|
||||
flow?: string
|
||||
node?: string
|
||||
error?: string
|
||||
ts?: number
|
||||
}
|
||||
| {
|
||||
type: "pipeline_rebuilt"
|
||||
nodes: { id: string; status: string; error?: string | null }[]
|
||||
@@ -110,6 +147,44 @@ export function useFlowSocket(onAuthFailure?: () => void): void {
|
||||
status: "error",
|
||||
error: message.error,
|
||||
})
|
||||
liveStore.recordEngineEvent({
|
||||
type: message.type,
|
||||
flow: message.flow,
|
||||
node: message.node,
|
||||
detail: message.error,
|
||||
ts: message.ts ?? Date.now() / 1000,
|
||||
})
|
||||
break
|
||||
case "node_health":
|
||||
liveStore.setHealth(message.node, {
|
||||
health: message.health,
|
||||
detail: message.detail,
|
||||
})
|
||||
if (message.health === "down") {
|
||||
liveStore.recordEngineEvent({
|
||||
type: message.type,
|
||||
flow: message.flow,
|
||||
node: message.node,
|
||||
detail: message.detail ?? "Reported itself down.",
|
||||
ts: message.ts ?? Date.now() / 1000,
|
||||
})
|
||||
}
|
||||
break
|
||||
case "flow_quarantined":
|
||||
case "engine_degraded":
|
||||
case "engine_fatal":
|
||||
case "cascade_dropped":
|
||||
case "queue_unavailable":
|
||||
liveStore.recordEngineEvent({
|
||||
type: message.type,
|
||||
flow: "flow" in message ? message.flow : undefined,
|
||||
node: "node" in message ? message.node : undefined,
|
||||
detail:
|
||||
("error" in message ? message.error : undefined) ??
|
||||
("reason" in message ? message.reason : undefined) ??
|
||||
"",
|
||||
ts: message.ts ?? Date.now() / 1000,
|
||||
})
|
||||
break
|
||||
case "node_status":
|
||||
liveStore.setStatus(message.node, {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
Activity,
|
||||
Bell,
|
||||
Home,
|
||||
KeyRound,
|
||||
@@ -26,6 +27,7 @@ const baseItems: Item[] = [
|
||||
{ icon: Home, title: "Home", path: "/" },
|
||||
{ icon: Workflow, title: "Flows", path: "/flows" },
|
||||
{ icon: LayoutDashboard, title: "Dashboards", path: "/dashboards" },
|
||||
{ icon: Activity, title: "Health", path: "/health" },
|
||||
// Both are engine-wide operator settings rather than personal ones, so they
|
||||
// sit here and not among the per-user tabs under Settings.
|
||||
{ icon: KeyRound, title: "Secrets", path: "/secrets" },
|
||||
|
||||
@@ -21,6 +21,7 @@ import { Route as OauthAuthorizeRouteImport } from './routes/oauth.authorize'
|
||||
import { Route as LayoutSettingsRouteImport } from './routes/_layout/settings'
|
||||
import { Route as LayoutSecretsRouteImport } from './routes/_layout/secrets'
|
||||
import { Route as LayoutModulesRouteImport } from './routes/_layout/modules'
|
||||
import { Route as LayoutHealthRouteImport } from './routes/_layout/health'
|
||||
import { Route as LayoutAlertsRouteImport } from './routes/_layout/alerts'
|
||||
import { Route as LayoutAdminRouteImport } from './routes/_layout/admin'
|
||||
import { Route as LayoutFlowsIndexRouteImport } from './routes/_layout/flows/index'
|
||||
@@ -86,6 +87,11 @@ const LayoutModulesRoute = LayoutModulesRouteImport.update({
|
||||
path: '/modules',
|
||||
getParentRoute: () => LayoutRoute,
|
||||
} as any)
|
||||
const LayoutHealthRoute = LayoutHealthRouteImport.update({
|
||||
id: '/health',
|
||||
path: '/health',
|
||||
getParentRoute: () => LayoutRoute,
|
||||
} as any)
|
||||
const LayoutAlertsRoute = LayoutAlertsRouteImport.update({
|
||||
id: '/alerts',
|
||||
path: '/alerts',
|
||||
@@ -125,6 +131,7 @@ export interface FileRoutesByFullPath {
|
||||
'/signup': typeof SignupRoute
|
||||
'/admin': typeof LayoutAdminRoute
|
||||
'/alerts': typeof LayoutAlertsRoute
|
||||
'/health': typeof LayoutHealthRoute
|
||||
'/modules': typeof LayoutModulesRoute
|
||||
'/secrets': typeof LayoutSecretsRoute
|
||||
'/settings': typeof LayoutSettingsRoute
|
||||
@@ -143,6 +150,7 @@ export interface FileRoutesByTo {
|
||||
'/signup': typeof SignupRoute
|
||||
'/admin': typeof LayoutAdminRoute
|
||||
'/alerts': typeof LayoutAlertsRoute
|
||||
'/health': typeof LayoutHealthRoute
|
||||
'/modules': typeof LayoutModulesRoute
|
||||
'/secrets': typeof LayoutSecretsRoute
|
||||
'/settings': typeof LayoutSettingsRoute
|
||||
@@ -163,6 +171,7 @@ export interface FileRoutesById {
|
||||
'/signup': typeof SignupRoute
|
||||
'/_layout/admin': typeof LayoutAdminRoute
|
||||
'/_layout/alerts': typeof LayoutAlertsRoute
|
||||
'/_layout/health': typeof LayoutHealthRoute
|
||||
'/_layout/modules': typeof LayoutModulesRoute
|
||||
'/_layout/secrets': typeof LayoutSecretsRoute
|
||||
'/_layout/settings': typeof LayoutSettingsRoute
|
||||
@@ -184,6 +193,7 @@ export interface FileRouteTypes {
|
||||
| '/signup'
|
||||
| '/admin'
|
||||
| '/alerts'
|
||||
| '/health'
|
||||
| '/modules'
|
||||
| '/secrets'
|
||||
| '/settings'
|
||||
@@ -202,6 +212,7 @@ export interface FileRouteTypes {
|
||||
| '/signup'
|
||||
| '/admin'
|
||||
| '/alerts'
|
||||
| '/health'
|
||||
| '/modules'
|
||||
| '/secrets'
|
||||
| '/settings'
|
||||
@@ -221,6 +232,7 @@ export interface FileRouteTypes {
|
||||
| '/signup'
|
||||
| '/_layout/admin'
|
||||
| '/_layout/alerts'
|
||||
| '/_layout/health'
|
||||
| '/_layout/modules'
|
||||
| '/_layout/secrets'
|
||||
| '/_layout/settings'
|
||||
@@ -330,6 +342,13 @@ declare module '@tanstack/react-router' {
|
||||
preLoaderRoute: typeof LayoutModulesRouteImport
|
||||
parentRoute: typeof LayoutRoute
|
||||
}
|
||||
'/_layout/health': {
|
||||
id: '/_layout/health'
|
||||
path: '/health'
|
||||
fullPath: '/health'
|
||||
preLoaderRoute: typeof LayoutHealthRouteImport
|
||||
parentRoute: typeof LayoutRoute
|
||||
}
|
||||
'/_layout/alerts': {
|
||||
id: '/_layout/alerts'
|
||||
path: '/alerts'
|
||||
@@ -391,6 +410,7 @@ const CanvasRouteWithChildren =
|
||||
interface LayoutRouteChildren {
|
||||
LayoutAdminRoute: typeof LayoutAdminRoute
|
||||
LayoutAlertsRoute: typeof LayoutAlertsRoute
|
||||
LayoutHealthRoute: typeof LayoutHealthRoute
|
||||
LayoutModulesRoute: typeof LayoutModulesRoute
|
||||
LayoutSecretsRoute: typeof LayoutSecretsRoute
|
||||
LayoutSettingsRoute: typeof LayoutSettingsRoute
|
||||
@@ -402,6 +422,7 @@ interface LayoutRouteChildren {
|
||||
const LayoutRouteChildren: LayoutRouteChildren = {
|
||||
LayoutAdminRoute: LayoutAdminRoute,
|
||||
LayoutAlertsRoute: LayoutAlertsRoute,
|
||||
LayoutHealthRoute: LayoutHealthRoute,
|
||||
LayoutModulesRoute: LayoutModulesRoute,
|
||||
LayoutSecretsRoute: LayoutSecretsRoute,
|
||||
LayoutSettingsRoute: LayoutSettingsRoute,
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query"
|
||||
import { createFileRoute, Link } from "@tanstack/react-router"
|
||||
import { ChevronDown, ChevronRight } from "lucide-react"
|
||||
import { useEffect, useState } from "react"
|
||||
|
||||
import {
|
||||
type EventRow,
|
||||
type FlowRollup,
|
||||
type HistoryPoint,
|
||||
ObservabilityService,
|
||||
} from "@/client"
|
||||
import { UplotChart } from "@/components/Common/UplotChart"
|
||||
import { useEngineEvents } from "@/components/Flow/liveStore"
|
||||
import { shape } from "@/components/Flow/MessageSparkline"
|
||||
import { PANEL_SECTION } from "@/components/Flow/SidePanel"
|
||||
import { useFlowSocket } from "@/components/Flow/useFlowSocket"
|
||||
import { Badge } from "@/components/ui/badge"
|
||||
|
||||
export const Route = createFileRoute("/_layout/health")({
|
||||
component: Health,
|
||||
head: () => ({
|
||||
meta: [
|
||||
{
|
||||
title: "Health - Fluksio",
|
||||
},
|
||||
],
|
||||
}),
|
||||
})
|
||||
|
||||
const healthKeys = {
|
||||
all: ["observability"] as const,
|
||||
summary: ["observability", "summary"] as const,
|
||||
events: ["observability", "events"] as const,
|
||||
}
|
||||
|
||||
/** The live view; anything older is the collector's rollups. */
|
||||
const HOURS = 24
|
||||
const CARD = "rounded-lg border border-border bg-card p-4 shadow-e1"
|
||||
|
||||
function ago(ts: string | number | null | undefined): string {
|
||||
if (!ts) return "—"
|
||||
const stamp = typeof ts === "number" ? ts * 1000 : Date.parse(ts)
|
||||
const seconds = Math.max(0, (Date.now() - stamp) / 1000)
|
||||
if (seconds < 90) return `${Math.round(seconds)}s ago`
|
||||
if (seconds < 5400) return `${Math.round(seconds / 60)}m ago`
|
||||
if (seconds < 172800) return `${Math.round(seconds / 3600)}h ago`
|
||||
return `${Math.round(seconds / 86400)}d ago`
|
||||
}
|
||||
|
||||
const round = (value: number) =>
|
||||
value >= 100 ? Math.round(value) : +value.toFixed(1)
|
||||
|
||||
function Tile({
|
||||
label,
|
||||
value,
|
||||
note,
|
||||
}: {
|
||||
label: string
|
||||
value: string
|
||||
note?: string
|
||||
}) {
|
||||
return (
|
||||
<div className={CARD}>
|
||||
<div className={PANEL_SECTION}>{label}</div>
|
||||
<div className="mt-1 text-2xl">{value}</div>
|
||||
{note ? (
|
||||
<div className="text-xs text-muted-foreground">{note}</div>
|
||||
) : null}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** A flow's execution trend, drawn from the 60 slices the rollup carries. */
|
||||
function Spark({ counts }: { counts: number[] }) {
|
||||
const points: HistoryPoint[] = counts.map((value, index) => ({
|
||||
ts: index,
|
||||
value,
|
||||
}))
|
||||
if (points.every((point) => point.value === 0)) {
|
||||
return <span className="text-xs text-muted-foreground">nothing yet</span>
|
||||
}
|
||||
const { line } = shape(points)
|
||||
return (
|
||||
<svg
|
||||
viewBox="0 0 100 100"
|
||||
preserveAspectRatio="none"
|
||||
className="h-6 w-24 overflow-visible"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path
|
||||
d={line}
|
||||
fill="none"
|
||||
stroke="var(--chart-1)"
|
||||
strokeWidth="1.5"
|
||||
vectorEffect="non-scaling-stroke"
|
||||
/>
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
function Failure({ event }: { event: EventRow }) {
|
||||
const [open, setOpen] = useState(false)
|
||||
const [first, ...rest] = event.detail.split("\n")
|
||||
return (
|
||||
<div className="border-b border-border py-2 last:border-0">
|
||||
<button
|
||||
type="button"
|
||||
className="flex w-full items-start gap-2 text-left"
|
||||
onClick={() => setOpen(!open)}
|
||||
disabled={rest.length === 0}
|
||||
>
|
||||
{rest.length ? (
|
||||
open ? (
|
||||
<ChevronDown className="mt-0.5 size-4 shrink-0 text-muted-foreground" />
|
||||
) : (
|
||||
<ChevronRight className="mt-0.5 size-4 shrink-0 text-muted-foreground" />
|
||||
)
|
||||
) : (
|
||||
<span className="size-4 shrink-0" />
|
||||
)}
|
||||
<span className="min-w-0 flex-1">
|
||||
<span className="font-mono text-sm">
|
||||
{event.node || event.flow || "engine"}
|
||||
</span>
|
||||
<span className="ml-2 text-sm text-muted-foreground">{first}</span>
|
||||
</span>
|
||||
<span className="shrink-0 text-xs text-muted-foreground">
|
||||
{ago(event.ts)}
|
||||
</span>
|
||||
</button>
|
||||
{open && rest.length ? (
|
||||
<pre className="mt-2 max-h-64 overflow-auto rounded-md border border-border bg-muted/40 p-3 font-mono text-xs">
|
||||
{rest.join("\n")}
|
||||
</pre>
|
||||
) : null}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Health() {
|
||||
// The page is a socket subscriber like the editor: a failure should appear
|
||||
// without waiting for the next poll.
|
||||
useFlowSocket()
|
||||
const live = useEngineEvents()
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
const { data: summary } = useQuery({
|
||||
queryKey: healthKeys.summary,
|
||||
queryFn: () => ObservabilityService.readSummary(),
|
||||
refetchInterval: 10000,
|
||||
})
|
||||
const { data: series } = useQuery({
|
||||
queryKey: ["observability", "timeseries", HOURS],
|
||||
queryFn: () => ObservabilityService.readTimeseries({ hours: HOURS }),
|
||||
refetchInterval: 30000,
|
||||
})
|
||||
const { data: flows } = useQuery({
|
||||
queryKey: ["observability", "flows", HOURS],
|
||||
queryFn: () => ObservabilityService.readFlowRollups({ hours: HOURS }),
|
||||
refetchInterval: 30000,
|
||||
})
|
||||
const { data: runs } = useQuery({
|
||||
queryKey: ["observability", "runs"],
|
||||
queryFn: () => ObservabilityService.readRuns({ limit: 15 }),
|
||||
refetchInterval: 30000,
|
||||
})
|
||||
const { data: failures } = useQuery({
|
||||
queryKey: [...healthKeys.events, "failure"],
|
||||
queryFn: () =>
|
||||
ObservabilityService.readEvents({ kind: "failure", limit: 25 }),
|
||||
refetchInterval: 30000,
|
||||
})
|
||||
const { data: audit } = useQuery({
|
||||
queryKey: [...healthKeys.events, "audit"],
|
||||
queryFn: () =>
|
||||
ObservabilityService.readEvents({ kind: "audit", limit: 15 }),
|
||||
refetchInterval: 30000,
|
||||
})
|
||||
const { data: dead } = useQuery({
|
||||
queryKey: ["observability", "dead-letter"],
|
||||
queryFn: () => ObservabilityService.readDeadLetters({ limit: 20 }),
|
||||
refetchInterval: 30000,
|
||||
})
|
||||
|
||||
// Something just went wrong on the socket. The row for it is written on the
|
||||
// collector's next flush, so the refetch waits that out rather than asking
|
||||
// for a failure the database does not have yet.
|
||||
const seen = live.length
|
||||
useEffect(() => {
|
||||
if (!seen) return
|
||||
const timer = setTimeout(() => {
|
||||
queryClient.invalidateQueries({ queryKey: healthKeys.events })
|
||||
queryClient.invalidateQueries({ queryKey: healthKeys.summary })
|
||||
}, 16000)
|
||||
return () => clearTimeout(timer)
|
||||
}, [seen, queryClient])
|
||||
|
||||
const points = series ?? []
|
||||
const at = (
|
||||
pick: (point: (typeof points)[number]) => number,
|
||||
): HistoryPoint[] =>
|
||||
points.map((point) => ({ ts: point.ts, value: pick(point) }))
|
||||
|
||||
const queue = (summary?.queue ?? {}) as Record<string, number>
|
||||
const degraded = summary?.status === "degraded"
|
||||
|
||||
return (
|
||||
<div className="grid gap-6">
|
||||
<div className="grid gap-1">
|
||||
<div className="flex items-center gap-3">
|
||||
<h1 className="text-2xl">Health</h1>
|
||||
<Badge variant={degraded ? "destructive" : "secondary"}>
|
||||
{degraded ? "Degraded" : "Running normally"}
|
||||
</Badge>
|
||||
</div>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{summary?.problems.length
|
||||
? summary.problems.join(" · ")
|
||||
: "What the engine has been doing over the last day, and what it is doing now."}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<section className="grid gap-3 sm:grid-cols-2 lg:grid-cols-5">
|
||||
<Tile
|
||||
label="Nodes"
|
||||
value={String(summary?.nodes.total ?? 0)}
|
||||
note={
|
||||
summary?.nodes.error
|
||||
? `${summary.nodes.error} failed to load`
|
||||
: "all loaded"
|
||||
}
|
||||
/>
|
||||
<Tile
|
||||
label="Flows running"
|
||||
value={`${summary?.flows.running ?? 0}/${summary?.flows.total ?? 0}`}
|
||||
note={
|
||||
summary?.flows.quarantined
|
||||
? `${summary.flows.quarantined} quarantined`
|
||||
: summary?.flows.paused
|
||||
? `${summary.flows.paused} paused`
|
||||
: "none paused"
|
||||
}
|
||||
/>
|
||||
<Tile
|
||||
label="Failures (24h)"
|
||||
value={String(summary?.failures_24h ?? 0)}
|
||||
note={
|
||||
failures?.length
|
||||
? `latest ${ago(failures[0].ts)}`
|
||||
: "nothing recorded"
|
||||
}
|
||||
/>
|
||||
<Tile
|
||||
label="Queue in flight"
|
||||
value={String(queue.pending ?? 0)}
|
||||
note={`${queue.delayed ?? 0} waiting · ${queue.parked ?? 0} parked`}
|
||||
/>
|
||||
<Tile
|
||||
label="Loop lag"
|
||||
value={`${round(summary?.loop_lag.ewma ?? 0)} ms`}
|
||||
note={`peak ${round(summary?.loop_lag.max_60s ?? 0)} ms in the last minute`}
|
||||
/>
|
||||
</section>
|
||||
|
||||
<section className="grid gap-4 lg:grid-cols-2">
|
||||
<div className={`${CARD} flex h-64 flex-col`}>
|
||||
<h2 className={PANEL_SECTION}>Throughput per minute</h2>
|
||||
<UplotChart
|
||||
labels={["messages", "executions"]}
|
||||
plots={[
|
||||
at((point) => point.messages),
|
||||
at((point) => point.executions),
|
||||
]}
|
||||
empty="Nothing has run yet."
|
||||
/>
|
||||
</div>
|
||||
<div className={`${CARD} flex h-64 flex-col`}>
|
||||
<h2 className={PANEL_SECTION}>Failures and timing</h2>
|
||||
<UplotChart
|
||||
labels={["errors", "avg ms", "avg lag ms"]}
|
||||
plots={[
|
||||
at((point) => point.errors),
|
||||
at((point) => point.avg_ms),
|
||||
at((point) => point.avg_lag_ms),
|
||||
]}
|
||||
empty="Nothing has run yet."
|
||||
/>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="grid gap-3">
|
||||
<h2 className={PANEL_SECTION}>Flows</h2>
|
||||
<div className={`${CARD} overflow-x-auto`}>
|
||||
<table className="w-full text-sm">
|
||||
<thead className="text-xs text-muted-foreground">
|
||||
<tr className="text-left">
|
||||
<th className="pb-2 font-medium">Flow</th>
|
||||
<th className="pb-2 font-medium">Executions</th>
|
||||
<th className="pb-2 font-medium">Errors</th>
|
||||
<th className="pb-2 font-medium">Avg</th>
|
||||
<th className="pb-2 font-medium">Lag</th>
|
||||
<th className="pb-2 font-medium">Trend</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{(flows ?? []).map((row: FlowRollup) => (
|
||||
<tr key={row.flow} className="border-t border-border">
|
||||
<td className="py-2">
|
||||
<Link
|
||||
to="/flows/$flowName"
|
||||
params={{ flowName: row.flow }}
|
||||
className="font-mono hover:underline"
|
||||
>
|
||||
{row.flow || "—"}
|
||||
</Link>
|
||||
</td>
|
||||
<td className="py-2">{row.executions}</td>
|
||||
<td className="py-2">
|
||||
{row.errors ? (
|
||||
<Badge variant="destructive">{row.errors} failed</Badge>
|
||||
) : (
|
||||
<span className="text-muted-foreground">none</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="py-2">{round(row.avg_ms)} ms</td>
|
||||
<td className="py-2">{round(row.avg_lag_ms)} ms</td>
|
||||
<td className="py-2">
|
||||
<Spark counts={row.spark} />
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
{flows?.length === 0 ? (
|
||||
<tr>
|
||||
<td
|
||||
colSpan={6}
|
||||
className="py-6 text-center text-muted-foreground"
|
||||
>
|
||||
No flow has run in the last day.
|
||||
</td>
|
||||
</tr>
|
||||
) : null}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="grid content-start gap-4 lg:grid-cols-2">
|
||||
<div className="grid content-start gap-3">
|
||||
<h2 className={PANEL_SECTION}>Recent runs</h2>
|
||||
<div className={CARD}>
|
||||
{runs?.length ? (
|
||||
runs.map((run) => (
|
||||
<div
|
||||
key={run.id}
|
||||
className="flex items-baseline gap-3 border-b border-border py-2 text-sm last:border-0"
|
||||
>
|
||||
<span className="min-w-0 flex-1 truncate font-mono">
|
||||
{run.flow}
|
||||
</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{run.source}
|
||||
</span>
|
||||
<span
|
||||
className={
|
||||
run.status === "error"
|
||||
? "text-destructive"
|
||||
: run.status === "ok"
|
||||
? "text-status-success"
|
||||
: "text-muted-foreground"
|
||||
}
|
||||
>
|
||||
{run.status}
|
||||
</span>
|
||||
<span className="w-16 text-right text-muted-foreground">
|
||||
{round(run.duration_ms)} ms
|
||||
</span>
|
||||
<span className="w-16 text-right text-xs text-muted-foreground">
|
||||
{ago(run.started_at)}
|
||||
</span>
|
||||
</div>
|
||||
))
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
No runs recorded yet.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid content-start gap-3">
|
||||
<h2 className={PANEL_SECTION}>Failures</h2>
|
||||
<div className={CARD}>
|
||||
{failures?.length ? (
|
||||
failures.map((event) => <Failure key={event.id} event={event} />)
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Nothing has failed in the last day.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{dead?.length ? (
|
||||
<section className="grid gap-3">
|
||||
<h2 className={PANEL_SECTION}>Given up on</h2>
|
||||
<div className={CARD}>
|
||||
{dead.map((item) => (
|
||||
<div
|
||||
key={item.id}
|
||||
className="flex items-baseline gap-3 border-b border-border py-2 text-sm last:border-0"
|
||||
>
|
||||
<span className="min-w-0 flex-1 truncate font-mono">
|
||||
{item.node}
|
||||
</span>
|
||||
<span className="text-muted-foreground">{item.reason}</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{ago(item.ts)}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
) : null}
|
||||
|
||||
<section className="grid gap-3">
|
||||
<h2 className={PANEL_SECTION}>Changes</h2>
|
||||
<div className={CARD}>
|
||||
{audit?.length ? (
|
||||
audit.map((event) => (
|
||||
<div
|
||||
key={event.id}
|
||||
className="flex items-baseline gap-3 border-b border-border py-2 text-sm last:border-0"
|
||||
>
|
||||
<span className="min-w-0 flex-1 truncate">
|
||||
{event.actor} {event.detail}
|
||||
{event.flow ? (
|
||||
<span className="ml-1 font-mono text-muted-foreground">
|
||||
{event.flow}
|
||||
</span>
|
||||
) : null}
|
||||
</span>
|
||||
<span className="shrink-0 text-xs text-muted-foreground">
|
||||
{ago(event.ts)}
|
||||
</span>
|
||||
</div>
|
||||
))
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Nothing has changed yet.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user