diff --git a/frontend/scripts/capture-screenshots.mjs b/frontend/scripts/capture-screenshots.mjs index d66c41f..2e0a4d5 100644 --- a/frontend/scripts/capture-screenshots.mjs +++ b/frontend/scripts/capture-screenshots.mjs @@ -91,10 +91,11 @@ for (const theme of ["light", "dark"]) { await captureDashboards(page, dir) await captureMedia(page, dir) await captureRuns(page, dir) + await captureWorkers(page, dir) await context.close() console.log( - ` wrote ${dir}/{website-hero,app-login,app-dashboard,app-flows,app-flow-panel,app-panel,app-runs,app-run,app-run-context}.png`, + ` wrote ${dir}/{website-hero,app-login,app-dashboard,app-flows,app-flow-panel,app-panel,app-runs,app-run,app-run-context,app-workers}.png`, ) } } @@ -232,6 +233,14 @@ async function captureMedia(page, dir) { * The flow editor, empty-handed if the instance has no flows yet: seeds one * with a node so the canvas and the node panel are both worth looking at. */ +/** Every machine a node can run on, and the sizes it can ask for. */ +async function captureWorkers(page, dir) { + await page.goto(`${APP_URL}/workers`, { waitUntil: "networkidle" }) + await page.getByText(/^Sizes$/).waitFor({ timeout: 15000 }) + await page.waitForTimeout(500) + await page.screenshot({ path: `${dir}/app-workers.png`, fullPage: true }) +} + async function captureFlows(page, dir) { await page.goto(`${APP_URL}/flows`, { waitUntil: "networkidle" }) diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 7ce7f7b..5e38b81 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -722,6 +722,154 @@ export const EventRowSchema = { title: 'EventRow' } as const; +export const FlavorCreateSchema = { + properties: { + cpus: { + type: 'integer', + minimum: 1, + title: 'Cpus', + default: 1 + }, + gpus: { + type: 'integer', + minimum: 0, + title: 'Gpus', + default: 0 + }, + ram: { + type: 'integer', + minimum: 1, + title: 'Ram', + default: 2048 + }, + description: { + type: 'string', + maxLength: 255, + title: 'Description', + default: '' + }, + name: { + type: 'string', + maxLength: 64, + title: 'Name' + } + }, + type: 'object', + required: ['name'], + title: 'FlavorCreate' +} as const; + +export const FlavorPublicSchema = { + properties: { + cpus: { + type: 'integer', + minimum: 1, + title: 'Cpus', + default: 1 + }, + gpus: { + type: 'integer', + minimum: 0, + title: 'Gpus', + default: 0 + }, + ram: { + type: 'integer', + minimum: 1, + title: 'Ram', + default: 2048 + }, + description: { + type: 'string', + maxLength: 255, + title: 'Description', + default: '' + }, + name: { + type: 'string', + title: 'Name' + } + }, + type: 'object', + required: ['name'], + title: 'FlavorPublic' +} as const; + +export const FlavorUpdateSchema = { + properties: { + cpus: { + anyOf: [ + { + type: 'integer', + minimum: 1 + }, + { + type: 'null' + } + ], + title: 'Cpus' + }, + gpus: { + anyOf: [ + { + type: 'integer', + minimum: 0 + }, + { + type: 'null' + } + ], + title: 'Gpus' + }, + ram: { + anyOf: [ + { + type: 'integer', + minimum: 1 + }, + { + type: 'null' + } + ], + title: 'Ram' + }, + description: { + anyOf: [ + { + type: 'string', + maxLength: 255 + }, + { + type: 'null' + } + ], + title: 'Description' + } + }, + type: 'object', + title: 'FlavorUpdate', + description: 'Everything but the name: renaming would orphan the nodes that ask.' +} as const; + +export const FlavorsPublicSchema = { + properties: { + data: { + items: { + '$ref': '#/components/schemas/FlavorPublic' + }, + type: 'array', + title: 'Data' + }, + count: { + type: 'integer', + title: 'Count' + } + }, + type: 'object', + required: ['data', 'count'], + title: 'FlavorsPublic' +} as const; + export const FlowDef_InputSchema = { properties: { name: { @@ -2364,6 +2512,22 @@ export const RemoteUserBodySchema = { title: 'RemoteUserBody' } as const; +export const ResourceLevelSchema = { + properties: { + total: { + type: 'integer', + title: 'Total' + }, + free: { + type: 'integer', + title: 'Free' + } + }, + type: 'object', + required: ['total', 'free'], + title: 'ResourceLevel' +} as const; + export const ResourcesSchema = { properties: { cpus: { @@ -2380,6 +2544,44 @@ export const ResourcesSchema = { description: 'Whole devices held for the whole execution, named to the node through CUDA_VISIBLE_DEVICES. Nothing else is given them while it runs, which is what keeps two preallocating processes apart.', default: 0 }, + ram: { + anyOf: [ + { + type: 'integer', + minimum: 1 + }, + { + type: 'null' + } + ], + title: 'Ram', + description: "Megabytes held for the whole execution; accepts '512M' or '2G'. Counted against machines that said how much they have, and ignored by those that did not — which is a machine with nothing to say about memory, not one with none." + }, + flavor: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Flavor', + description: 'A stored size by name, standing in for cpus, gpus and ram. Read again every time the node is built, so editing the flavor edits what the next run gets.' + }, + duration_s: { + anyOf: [ + { + type: 'integer', + minimum: 1 + }, + { + type: 'null' + } + ], + title: 'Duration S', + description: "How long one execution is expected to take; accepts '30m' or '2h'. A statement about the node for whoever is planning around it, not a limit — the limit is `timeout`." + }, env: { additionalProperties: { type: 'string' @@ -2407,6 +2609,42 @@ arrives. Both are a node saying how much of the machine it takes, which is what this is.` } as const; +export const ResourcesSnapshotSchema = { + properties: { + cpus: { + '$ref': '#/components/schemas/ResourceLevel' + }, + gpus: { + '$ref': '#/components/schemas/ResourceLevel' + }, + waiting: { + items: { + '$ref': '#/components/schemas/WaitingNode' + }, + type: 'array', + title: 'Waiting' + }, + targets: { + items: { + '$ref': '#/components/schemas/TargetResources' + }, + type: 'array', + title: 'Targets' + }, + provisioners: { + items: { + additionalProperties: true, + type: 'object' + }, + type: 'array', + title: 'Provisioners' + } + }, + type: 'object', + required: ['cpus', 'gpus'], + title: 'ResourcesSnapshot' +} as const; + export const RuleSchema = { properties: { events: { @@ -2877,6 +3115,47 @@ export const SweepEntrySchema = { title: 'SweepEntry' } as const; +export const TargetResourcesSchema = { + properties: { + target: { + type: 'string', + title: 'Target' + }, + cpus: { + '$ref': '#/components/schemas/ResourceLevel' + }, + gpus: { + '$ref': '#/components/schemas/ResourceLevel' + }, + ram_mb: { + anyOf: [ + { + '$ref': '#/components/schemas/ResourceLevel' + }, + { + type: 'null' + } + ] + }, + labels: { + items: { + type: 'string' + }, + type: 'array', + title: 'Labels' + }, + in_flight: { + type: 'integer', + title: 'In Flight', + default: 0 + } + }, + type: 'object', + required: ['target', 'cpus', 'gpus'], + title: 'TargetResources', + description: 'One machine: this engine, or a worker attached to it.' +} as const; + export const TokenSchema = { properties: { access_token: { @@ -3326,6 +3605,26 @@ export const ValidationResultSchema = { title: 'ValidationResult' } as const; +export const WaitingNodeSchema = { + properties: { + node: { + type: 'string', + title: 'Node' + }, + reason: { + type: 'string', + title: 'Reason' + }, + seconds: { + type: 'number', + title: 'Seconds' + } + }, + type: 'object', + required: ['node', 'reason', 'seconds'], + title: 'WaitingNode' +} as const; + export const WidgetDefSchema = { properties: { id: { @@ -3427,6 +3726,27 @@ export const WorkerInfoSchema = { type: 'string', title: 'Venv Digest', default: '' + }, + cpus: { + type: 'integer', + title: 'Cpus', + default: 1 + }, + gpus: { + type: 'integer', + title: 'Gpus', + default: 0 + }, + ram_mb: { + anyOf: [ + { + type: 'integer' + }, + { + type: 'null' + } + ], + title: 'Ram Mb' } }, type: 'object', diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index 68f02c8..0cb2aba 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudDisconnectResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsReadRunsData, RunsReadRunsResponse, RunsReadOverviewResponse, RunsReadRunData, RunsReadRunResponse, RunsCancelRunData, RunsCancelRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; +import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudDisconnectResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsReadRunsData, RunsReadRunsResponse, RunsReadOverviewResponse, RunsReadRunData, RunsReadRunResponse, RunsCancelRunData, RunsCancelRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; export class AlertsService { /** @@ -415,6 +415,85 @@ export class DashboardsService { } } +export class FlavorsService { + /** + * Read Flavors + * Every size a node can ask for. + * @returns FlavorsPublic Successful Response + * @throws ApiError + */ + public static readFlavors(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/flavors/' + }); + } + + /** + * Create Flavor + * @param data The data for the request. + * @param data.requestBody + * @returns FlavorPublic Successful Response + * @throws ApiError + */ + public static createFlavor(data: FlavorsCreateFlavorData): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/api/v1/flavors/', + body: data.requestBody, + mediaType: 'application/json', + errors: { + 422: 'Validation Error' + } + }); + } + + /** + * Update Flavor + * Change what a size means. Every node naming it gets the new one. + * @param data The data for the request. + * @param data.name + * @param data.requestBody + * @returns FlavorPublic Successful Response + * @throws ApiError + */ + public static updateFlavor(data: FlavorsUpdateFlavorData): CancelablePromise { + return __request(OpenAPI, { + method: 'PATCH', + url: '/api/v1/flavors/{name}', + path: { + name: data.name + }, + body: data.requestBody, + mediaType: 'application/json', + errors: { + 422: 'Validation Error' + } + }); + } + + /** + * Delete Flavor + * Remove a size, as long as no node still asks for it. + * @param data The data for the request. + * @param data.name + * @returns Message Successful Response + * @throws ApiError + */ + public static deleteFlavor(data: FlavorsDeleteFlavorData): CancelablePromise { + return __request(OpenAPI, { + method: 'DELETE', + url: '/api/v1/flavors/{name}', + path: { + name: data.name + }, + errors: { + 422: 'Validation Error' + } + }); + } +} + export class FlowsService { /** * Read Flows @@ -2263,12 +2342,12 @@ export class WorkersService { /** * Read Resources - * What this machine has free, and which nodes are queued for it. + * Every machine, what is free of it, and which nodes are queued. * * A node waiting its turn looks exactly like a node that has hung — the run * sits at `running` and says nothing — so what is waiting, and for what, has * to be readable somewhere. - * @returns unknown Successful Response + * @returns ResourcesSnapshot Successful Response * @throws ApiError */ public static readResources(): CancelablePromise { diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index f0d9507..888eecd 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -224,6 +224,37 @@ export type EventRow = { actor: string; }; +export type FlavorCreate = { + cpus?: number; + gpus?: number; + ram?: number; + description?: string; + name: string; +}; + +export type FlavorPublic = { + cpus?: number; + gpus?: number; + ram?: number; + description?: string; + name: string; +}; + +export type FlavorsPublic = { + data: Array; + count: number; +}; + +/** + * Everything but the name: renaming would orphan the nodes that ask. + */ +export type FlavorUpdate = { + cpus?: (number | null); + gpus?: (number | null); + ram?: (number | null); + description?: (string | null); +}; + /** * One atomic flow. */ @@ -866,6 +897,11 @@ export type RemoteUserBody = { code: string; }; +export type ResourceLevel = { + total: number; + free: number; +}; + /** * What one execution of a node needs to have to itself. * @@ -890,6 +926,18 @@ export type Resources = { * Whole devices held for the whole execution, named to the node through CUDA_VISIBLE_DEVICES. Nothing else is given them while it runs, which is what keeps two preallocating processes apart. */ gpus?: number; + /** + * Megabytes held for the whole execution; accepts '512M' or '2G'. Counted against machines that said how much they have, and ignored by those that did not — which is a machine with nothing to say about memory, not one with none. + */ + ram?: (number | null); + /** + * A stored size by name, standing in for cpus, gpus and ram. Read again every time the node is built, so editing the flavor edits what the next run gets. + */ + flavor?: (string | null); + /** + * How long one execution is expected to take; accepts '30m' or '2h'. A statement about the node for whoever is planning around it, not a limit — the limit is `timeout`. + */ + duration_s?: (number | null); /** * Extra environment for the worker this node runs in, applied over what the allocation derives. Where a library's own tuning goes — XLA_FLAGS, XLA_PYTHON_CLIENT_MEM_FRACTION — since those are composed strings the engine must not invent. */ @@ -898,6 +946,16 @@ export type Resources = { }; }; +export type ResourcesSnapshot = { + cpus: ResourceLevel; + gpus: ResourceLevel; + waiting?: Array; + targets?: Array; + provisioners?: Array<{ + [key: string]: unknown; + }>; +}; + /** * Which events go to which channels. */ @@ -1033,6 +1091,18 @@ export type SweepEntry = { idempotency_key?: (string | null); }; +/** + * One machine: this engine, or a worker attached to it. + */ +export type TargetResources = { + target: string; + cpus: ResourceLevel; + gpus: ResourceLevel; + ram_mb?: (ResourceLevel | null); + labels?: Array<(string)>; + in_flight?: number; +}; + export type Token = { access_token: string; token_type?: string; @@ -1134,6 +1204,12 @@ export type ValidationResult = { issues?: Array; }; +export type WaitingNode = { + node: string; + reason: string; + seconds: number; +}; + /** * One tile: what it shows or does, and where it sits. * @@ -1184,6 +1260,9 @@ export type WorkerInfo = { last_seen?: number; python?: string; venv_digest?: string; + cpus?: number; + gpus?: number; + ram_mb?: (number | null); }; export type AlertsReadAlertsConfigResponse = (AlertsConfig); @@ -1286,6 +1365,27 @@ export type DashboardsRenameDashboardData = { export type DashboardsRenameDashboardResponse = (DashboardDef_Output); +export type FlavorsReadFlavorsResponse = (FlavorsPublic); + +export type FlavorsCreateFlavorData = { + requestBody: FlavorCreate; +}; + +export type FlavorsCreateFlavorResponse = (FlavorPublic); + +export type FlavorsUpdateFlavorData = { + name: string; + requestBody: FlavorUpdate; +}; + +export type FlavorsUpdateFlavorResponse = (FlavorPublic); + +export type FlavorsDeleteFlavorData = { + name: string; +}; + +export type FlavorsDeleteFlavorResponse = (Message); + export type FlowsReadFlowsResponse = (FlowsPublic); export type FlowsReadNodeTypesResponse = (Array); @@ -1760,7 +1860,7 @@ export type UtilsHealthResponse = ({ export type WorkersReadWorkersResponse = (Array); -export type WorkersReadResourcesResponse = (unknown); +export type WorkersReadResourcesResponse = (ResourcesSnapshot); export type WorkersIssueTokenData = { requestBody: TokenRequest; diff --git a/frontend/src/components/Flow/FlowNode.tsx b/frontend/src/components/Flow/FlowNode.tsx index dde25b3..152b1cf 100644 --- a/frontend/src/components/Flow/FlowNode.tsx +++ b/frontend/src/components/Flow/FlowNode.tsx @@ -60,7 +60,14 @@ const NODE_ICONS = { // One dot says everything about a node's state. Idle nodes carry no dot at all, // so the canvas stays quiet until something happens. +// Queued is neutral on purpose: the three status colours mean "running", +// "went well" and "went wrong", and a node waiting for a machine is none of +// those — it is idle with a reason, which the tooltip gives. const STATUS_STYLES = { + queued: { + dot: "bg-muted-foreground", + label: "Queued — waiting for a machine", + }, running: { dot: "bg-primary animate-pulse", label: "Running" }, success: { dot: "bg-status-success", label: "Last run succeeded" }, error: { dot: "bg-destructive", label: "Something went wrong" }, @@ -305,7 +312,7 @@ function FlowNodeComponent({ data, selected }: NodeProps) { /> - {problem || style.label} + {problem || live?.detail || style.label} ) : null} diff --git a/frontend/src/components/Flow/NodePanel.tsx b/frontend/src/components/Flow/NodePanel.tsx index b8ca7e6..acf995b 100644 --- a/frontend/src/components/Flow/NodePanel.tsx +++ b/frontend/src/components/Flow/NodePanel.tsx @@ -42,6 +42,7 @@ import { cn } from "@/lib/utils" import { DtypeValue } from "./FlowBoundary" import { MessageSparkline } from "./MessageSparkline" import { + flavorsQueryOptions, flowKeys, libraryQueryOptions, nodeSourceQueryOptions, @@ -81,6 +82,27 @@ const RESERVED_PARAMS = new Set(["synchronous"]) /** A setting reaches the function by name, so the name has to be one. */ const IDENTIFIER = /^[A-Za-z_]\w*$/ +/** Sentinels for the resources dropdown, which has two options that are not sizes. */ +const NO_RESOURCES = "__shared__" +const CUSTOM_RESOURCES = "__custom__" + +/** `2h` and `30m` and `90` — the grammar the engine parses, written back. */ +function parseDuration(text: string): number | null { + const match = /^\s*(\d+)\s*([smhd])?\s*$/i.exec(text) + if (!match) return null + const unit = (match[2] ?? "s").toLowerCase() + const scale = { s: 1, m: 60, h: 3600, d: 86400 }[unit] ?? 1 + return Number(match[1]) * scale +} + +function formatDuration(seconds: number | null | undefined): string { + if (!seconds) return "" + if (seconds % 86400 === 0) return `${seconds / 86400}d` + if (seconds % 3600 === 0) return `${seconds / 3600}h` + if (seconds % 60 === 0) return `${seconds / 60}m` + return `${seconds}s` +} + /** * A text field that offers what is already in use elsewhere. * @@ -828,6 +850,128 @@ function ParamsForm({ ) } +/** + * How much of a machine this node takes, and how long it is expected to take. + * + * A named size is the usual answer: what "gpu-small" means is a property of the + * machines this installation has, and those change. The numbers are still there + * for a node that genuinely wants its own. + */ +function ResourcesSection({ + node, + onChange, +}: { + node: NodeDef_Input + onChange: (node: NodeDef_Input) => void +}) { + const { data: flavors } = useQuery(flavorsQueryOptions()) + const resources = node.resources ?? null + const choice = !resources + ? NO_RESOURCES + : (resources.flavor ?? CUSTOM_RESOURCES) + + const edit = (next: NonNullable | null) => + onChange({ ...node, resources: next }) + + const pick = (value: string) => { + if (value === NO_RESOURCES) return edit(null) + // A flavor already says how much, so the numbers go with it — sending both + // is two answers, and the engine refuses it. + const kept = resources?.duration_s + ? { duration_s: resources.duration_s } + : {} + if (value === CUSTOM_RESOURCES) return edit({ cpus: 1, gpus: 0, ...kept }) + return edit({ flavor: value, ...kept }) + } + + return ( +
+ + + + {choice === CUSTOM_RESOURCES ? ( +
+ {( + [ + ["cpus", "cores", 1], + ["gpus", "gpus", 0], + ["ram", "MB", 0], + ] as const + ).map(([field, label, floor]) => ( +
+ { + const raw = event.target.value + const value = + raw === "" ? null : Math.max(floor, Number(raw) || 0) + edit({ + ...resources, + cpus: resources?.cpus ?? 1, + // `ram` may be left unstated; the other two always have one. + [field]: field === "ram" ? value : (value ?? floor), + }) + }} + /> + {label} +
+ ))} +
+ ) : null} + + {resources ? ( + { + const text = event.target.value.trim() + const seconds = text === "" ? null : parseDuration(text) + if (text !== "" && seconds === null) { + event.target.value = formatDuration(resources.duration_s) + return + } + if (seconds !== resources.duration_s) { + edit({ ...resources, duration_s: seconds }) + } + }} + /> + ) : null} + +

+ {resources + ? "Held for the whole execution, and what the node's own libraries are told they may use. The duration is a planning fact, not a limit." + : "Nothing held: the node shares the pool with every other node that says nothing."} +

+
+ ) +} + /** * Sharing a node moves its code to the library, where other flows can point at * it. Each flow keeps its own ports and settings; only the code is common, so @@ -1209,6 +1353,9 @@ function PanelBody({

) : null} + {hasSource ? ( + + ) : null} {hasSource ? ( ) : null} diff --git a/frontend/src/components/Flow/liveStore.ts b/frontend/src/components/Flow/liveStore.ts index 342dc67..52057ae 100644 --- a/frontend/src/components/Flow/liveStore.ts +++ b/frontend/src/components/Flow/liveStore.ts @@ -24,8 +24,10 @@ export type LiveValue = { source?: ValueSource } export type LiveStatus = { - status: "active" | "error" | "running" | "success" + status: "active" | "error" | "queued" | "running" | "success" error?: string | null + /** Why it is queued — what it is waiting for, and on which machine. */ + detail?: string | null } /** How a node's connection is doing, which is not how its last run went. */ export type NodeHealth = { diff --git a/frontend/src/components/Flow/queries.ts b/frontend/src/components/Flow/queries.ts index 2a94e78..993eb3e 100644 --- a/frontend/src/components/Flow/queries.ts +++ b/frontend/src/components/Flow/queries.ts @@ -9,6 +9,7 @@ import { useCallback, useEffect, useRef, useState } from "react" import { ApiError, + FlavorsService, type FlowDef_Input, FlowsService, SecretsService, @@ -44,6 +45,12 @@ export const secretsQueryOptions = () => ({ queryFn: () => SecretsService.readSecrets(), }) +/** The named sizes a node can ask for, offered in the node panel. */ +export const flavorsQueryOptions = () => ({ + queryKey: ["flavors"] as const, + queryFn: () => FlavorsService.readFlavors(), +}) + /** Every flow at once, merged on what its nodes talk to. */ export const graphQueryOptions = () => ({ queryKey: flowKeys.graph, diff --git a/frontend/src/components/Flow/useFlowSocket.ts b/frontend/src/components/Flow/useFlowSocket.ts index d17044b..41677ea 100644 --- a/frontend/src/components/Flow/useFlowSocket.ts +++ b/frontend/src/components/Flow/useFlowSocket.ts @@ -42,6 +42,14 @@ type FlowEvent = source?: ValueSource } | { type: "node_started"; node: string } + | { + type: "node_queued" + flow?: string + node: string + run?: string + detail?: string + ts?: number + } | { type: "node_executed" flow?: string @@ -179,6 +187,14 @@ function connect() { source: message.source, }) break + case "node_queued": + // Waiting for a machine, which looks exactly like hung from outside. + // `node_started` overwrites this, so nothing has to clear it. + liveStore.setStatus(message.node, { + status: "queued", + detail: message.detail, + }) + break case "node_started": liveStore.setStatus(message.node, { status: "running" }) break diff --git a/frontend/src/components/Sidebar/AppSidebar.tsx b/frontend/src/components/Sidebar/AppSidebar.tsx index 4c733f8..b3386f0 100644 --- a/frontend/src/components/Sidebar/AppSidebar.tsx +++ b/frontend/src/components/Sidebar/AppSidebar.tsx @@ -7,6 +7,7 @@ import { LayoutDashboard, LogOut, Package, + Server, Settings, Users, Workflow, @@ -37,6 +38,7 @@ const baseItems: Item[] = [ // sit here and not among the per-user tabs under Settings. { icon: KeyRound, title: "Secrets", path: "/secrets" }, { icon: Package, title: "Modules", path: "/modules" }, + { icon: Server, title: "Workers", path: "/workers" }, { icon: Bell, title: "Alerts", path: "/alerts" }, ] diff --git a/frontend/src/routeTree.gen.ts b/frontend/src/routeTree.gen.ts index e710eaa..49465c5 100644 --- a/frontend/src/routeTree.gen.ts +++ b/frontend/src/routeTree.gen.ts @@ -20,6 +20,7 @@ import { Route as LayoutIndexRouteImport } from './routes/_layout/index' import { Route as ViewNameRouteImport } from './routes/view.$name' import { Route as PanelIdRouteImport } from './routes/panel.$id' import { Route as OauthAuthorizeRouteImport } from './routes/oauth.authorize' +import { Route as LayoutWorkersRouteImport } from './routes/_layout/workers' import { Route as LayoutSettingsRouteImport } from './routes/_layout/settings' import { Route as LayoutSecretsRouteImport } from './routes/_layout/secrets' import { Route as LayoutModulesRouteImport } from './routes/_layout/modules' @@ -85,6 +86,11 @@ const OauthAuthorizeRoute = OauthAuthorizeRouteImport.update({ path: '/oauth/authorize', getParentRoute: () => rootRouteImport, } as any) +const LayoutWorkersRoute = LayoutWorkersRouteImport.update({ + id: '/workers', + path: '/workers', + getParentRoute: () => LayoutRoute, +} as any) const LayoutSettingsRoute = LayoutSettingsRouteImport.update({ id: '/settings', path: '/settings', @@ -152,6 +158,7 @@ export interface FileRoutesByFullPath { '/modules': typeof LayoutModulesRoute '/secrets': typeof LayoutSecretsRoute '/settings': typeof LayoutSettingsRoute + '/workers': typeof LayoutWorkersRoute '/oauth/authorize': typeof OauthAuthorizeRoute '/panel/$id': typeof PanelIdRoute '/view/$name': typeof ViewNameRoute @@ -174,6 +181,7 @@ export interface FileRoutesByTo { '/modules': typeof LayoutModulesRoute '/secrets': typeof LayoutSecretsRoute '/settings': typeof LayoutSettingsRoute + '/workers': typeof LayoutWorkersRoute '/oauth/authorize': typeof OauthAuthorizeRoute '/panel/$id': typeof PanelIdRoute '/view/$name': typeof ViewNameRoute @@ -198,6 +206,7 @@ export interface FileRoutesById { '/_layout/modules': typeof LayoutModulesRoute '/_layout/secrets': typeof LayoutSecretsRoute '/_layout/settings': typeof LayoutSettingsRoute + '/_layout/workers': typeof LayoutWorkersRoute '/oauth/authorize': typeof OauthAuthorizeRoute '/panel/$id': typeof PanelIdRoute '/view/$name': typeof ViewNameRoute @@ -223,6 +232,7 @@ export interface FileRouteTypes { | '/modules' | '/secrets' | '/settings' + | '/workers' | '/oauth/authorize' | '/panel/$id' | '/view/$name' @@ -245,6 +255,7 @@ export interface FileRouteTypes { | '/modules' | '/secrets' | '/settings' + | '/workers' | '/oauth/authorize' | '/panel/$id' | '/view/$name' @@ -268,6 +279,7 @@ export interface FileRouteTypes { | '/_layout/modules' | '/_layout/secrets' | '/_layout/settings' + | '/_layout/workers' | '/oauth/authorize' | '/panel/$id' | '/view/$name' @@ -373,6 +385,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof OauthAuthorizeRouteImport parentRoute: typeof rootRouteImport } + '/_layout/workers': { + id: '/_layout/workers' + path: '/workers' + fullPath: '/workers' + preLoaderRoute: typeof LayoutWorkersRouteImport + parentRoute: typeof LayoutRoute + } '/_layout/settings': { id: '/_layout/settings' path: '/settings' @@ -472,6 +491,7 @@ interface LayoutRouteChildren { LayoutModulesRoute: typeof LayoutModulesRoute LayoutSecretsRoute: typeof LayoutSecretsRoute LayoutSettingsRoute: typeof LayoutSettingsRoute + LayoutWorkersRoute: typeof LayoutWorkersRoute LayoutIndexRoute: typeof LayoutIndexRoute LayoutRunsIdRoute: typeof LayoutRunsIdRoute LayoutDashboardsIndexRoute: typeof LayoutDashboardsIndexRoute @@ -485,6 +505,7 @@ const LayoutRouteChildren: LayoutRouteChildren = { LayoutModulesRoute: LayoutModulesRoute, LayoutSecretsRoute: LayoutSecretsRoute, LayoutSettingsRoute: LayoutSettingsRoute, + LayoutWorkersRoute: LayoutWorkersRoute, LayoutIndexRoute: LayoutIndexRoute, LayoutRunsIdRoute: LayoutRunsIdRoute, LayoutDashboardsIndexRoute: LayoutDashboardsIndexRoute, diff --git a/frontend/src/routes/_layout/workers.tsx b/frontend/src/routes/_layout/workers.tsx new file mode 100644 index 0000000..7a0ca8d --- /dev/null +++ b/frontend/src/routes/_layout/workers.tsx @@ -0,0 +1,439 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" +import { createFileRoute } from "@tanstack/react-router" +import { Cpu, Pencil, Plus, Server, Trash2 } from "lucide-react" +import { useState } from "react" + +import { + type ApiError, + type FlavorPublic, + FlavorsService, + WorkersService, +} from "@/client" +import { Button } from "@/components/ui/button" +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" +import useAuth from "@/hooks/useAuth" +import useCustomToast from "@/hooks/useCustomToast" +import { handleError } from "@/utils" + +export const Route = createFileRoute("/_layout/workers")({ + component: Workers, + head: () => ({ + meta: [{ title: "Workers - Fluksio" }], + }), +}) + +/** The node panel's dropdown reads this too, so an edit refreshes it. */ +const flavorsKey = ["flavors"] + +const emptyFlavor = { name: "", cpus: 1, gpus: 0, ram: 2048, description: "" } + +/** Fresh enough to watch a node take a machine, slow enough to be free. */ +const REFETCH_MS = 5000 + +function ago(seconds: number): string { + const since = Date.now() / 1000 - seconds + if (since < 60) return `${Math.max(0, Math.round(since))}s ago` + if (since < 3600) return `${Math.round(since / 60)}m ago` + return `${Math.round(since / 3600)}h ago` +} + +type Level = { total: number; free: number } + +/** "cpu 2/18" — what is in use, out of what there is. */ +function Used({ label, level }: { label: string; level?: Level | null }) { + if (!level?.total) return null + return ( + + {label} {level.total - level.free}/{level.total} + + ) +} + +function Machines() { + const { data, isError } = useQuery({ + queryKey: ["workers", "resources"], + queryFn: () => WorkersService.readResources(), + refetchInterval: REFETCH_MS, + }) + + if (isError || !data) { + return ( +

+ This engine does not account for resources. +

+ ) + } + + return ( +
+ {(data.targets ?? []).map((target) => ( +
+ + + {target.target} + {(target.labels ?? []).length > 0 ? ( + + {(target.labels ?? []).join(", ")} + + ) : null} + + + + + + +
+ ))} + + {(data.waiting ?? []).length > 0 ? ( +
+ {(data.waiting ?? []).map((node) => ( +

+ {node.node} — {node.reason} ·{" "} + {node.seconds.toFixed(0)}s +

+ ))} +
+ ) : null} + + {(data.provisioners ?? []).map((provisioner) => { + const outstanding = (provisioner.outstanding ?? []) as { + profile: string + job: string + seconds: number + }[] + return ( +

+ {String(provisioner.name)} can start{" "} + {(provisioner.profiles as string[]).join(", ")} + {outstanding.length > 0 + ? ` — asked for ${outstanding + .map((job) => `${job.profile} (job ${job.job})`) + .join(", ")}` + : ""} + {provisioner.last_error + ? ` — last failed: ${String(provisioner.last_error)}` + : ""} +

+ ) + })} +
+ ) +} + +function Attached() { + const { data } = useQuery({ + queryKey: ["workers"], + queryFn: () => WorkersService.readWorkers(), + refetchInterval: REFETCH_MS, + }) + const workers = data ?? [] + + if (workers.length === 0) { + return ( +

+ Nothing attached. A machine runs{" "} + pip install fluksio-worker and dials + in; nodes go to it by label, or because it has room and this engine does + not. +

+ ) + } + + return ( +
+ {workers.map((worker) => ( +
+ + + + {worker.name} + {(worker.labels ?? []).length > 0 ? ( + + {(worker.labels ?? []).join(", ")} + + ) : null} + + + {worker.cpus} cpu + {worker.gpus ? ` · ${worker.gpus} gpu` : ""} + {worker.ram_mb ? ` · ${Math.round(worker.ram_mb / 1024)} GB` : ""}{" "} + · {worker.in_flight}/{worker.max_parallel} in flight + + + + seen {ago(worker.last_seen ?? 0)} + +
+ ))} +
+ ) +} + +function Flavors() { + const { user } = useAuth() + const { data } = useQuery({ + queryKey: flavorsKey, + queryFn: () => FlavorsService.readFlavors(), + }) + const queryClient = useQueryClient() + const { showSuccessToast, showErrorToast } = useCustomToast() + const [editing, setEditing] = useState(null) + const [isNew, setIsNew] = useState(false) + const [pendingDelete, setPendingDelete] = useState(null) + + const refresh = () => queryClient.invalidateQueries({ queryKey: flavorsKey }) + + const save = useMutation({ + mutationFn: (flavor: typeof emptyFlavor) => + isNew + ? FlavorsService.createFlavor({ requestBody: flavor }) + : FlavorsService.updateFlavor({ + name: flavor.name, + requestBody: { + cpus: flavor.cpus, + gpus: flavor.gpus, + ram: flavor.ram, + description: flavor.description, + }, + }), + onSuccess: (_result, flavor) => { + showSuccessToast(`Saved '${flavor.name}'`) + setEditing(null) + }, + onError: handleError.bind(showErrorToast), + onSettled: refresh, + }) + + const remove = useMutation({ + mutationFn: (name: string) => FlavorsService.deleteFlavor({ name }), + onSuccess: (_result, name) => { + showSuccessToast(`Deleted '${name}'`) + setPendingDelete(null) + }, + onError: (error: ApiError) => { + handleError.call(showErrorToast, error) + setPendingDelete(null) + }, + onSettled: refresh, + }) + + const flavors = data?.data ?? [] + const mayEdit = Boolean(user?.is_superuser) + + return ( +
+ {flavors.map((flavor: FlavorPublic) => ( +
+ + {flavor.name} + + {flavor.cpus} cpu · {Math.round((flavor.ram ?? 0) / 1024)} GB + {flavor.gpus ? ` · ${flavor.gpus} gpu` : ""} + {flavor.description ? ` — ${flavor.description}` : ""} + + + {mayEdit ? ( + + + + + ) : null} +
+ ))} + + {mayEdit ? ( + + ) : null} + + !open && setEditing(null)} + > + + + + {isNew ? "New size" : `Edit '${editing?.name}'`} + + + Every node that names this gets what it says here, from its next + run. + + + {editing ? ( +
+ {isNew ? ( +
+ + + setEditing({ ...editing, name: event.target.value }) + } + /> +
+ ) : null} +
+ {( + [ + ["cpus", "Cores", 1], + ["gpus", "GPUs", 0], + ["ram", "MB", 1], + ] as const + ).map(([field, label, floor]) => ( +
+ + + setEditing({ + ...editing, + [field]: Math.max( + floor, + Number(event.target.value) || 0, + ), + }) + } + /> +
+ ))} +
+
+ + + setEditing({ ...editing, description: event.target.value }) + } + /> +
+
+ ) : null} + + + + + + + + !open && setPendingDelete(null)} + > + + + Delete '{pendingDelete}'? + + A node still asking for it keeps this from being deleted, and it + will say which. Nothing is seeded back afterwards. + + + + + + + + +
+ ) +} + +function Workers() { + return ( +
+
+

Workers

+

+ Every machine this engine can run a node on, what is free of each, and + the sizes a node can ask for by name. +

+
+ +
+

Machines

+ +
+ +
+

Attached workers

+ +
+ +
+

Sizes

+ +
+
+ ) +}