Remote workers: a GPU box dials in and runs the nodes bound to it

The engine runs where the automations are and the GPU is somewhere else,
usually behind a different network — so the worker connects out and the engine
answers over the socket it was given. Nothing has to expose Redis, and the
same connection works through the tunnel the hosted access will use.

What travels is the protocol the local pool already speaks, so a node cannot
tell which kind of worker it is on. A node declares device: gpu and
device_policy, the label is resolved per call (a worker attaching later needs
no rebuild), and a run whose labels nothing carries waits in the queue saying
what it waits for rather than failing — submit from the couch, the GPU box
picks it up when it is switched on.

Two things had to move with it. Compiling now happens on the machine that will
run the node: a node importing torch is correct on the GPU box and a missing
module on the engine, so checking it here failed nodes that were fine. And the
artifact endpoint accepts a worker's own credential, because storing a
checkpoint is exactly what that credential is for — and only that.

Verified against the real split: the training ran on this host (its checkpoint
names the machine and a numpy the engine does not have), streamed 40 metric
points back mid-run, and the evaluate node read the checkpoint on the engine.
Cancel kills the remote training; pulling the worker fails the run in six
seconds instead of waiting out its ten-minute timeout.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AD8SfVhzXBG2nAfFcVh3iD

frontend/src/client/schemas.gen.ts

@@ -2458,6 +2458,39 @@ export const TokenSchema = {
     title: 'Token'
 } as const;
 
+export const TokenIssuedSchema = {
+    properties: {
+        name: {
+            type: 'string',
+            title: 'Name'
+        },
+        token: {
+            type: 'string',
+            title: 'Token'
+        },
+        expires_days: {
+            type: 'integer',
+            title: 'Expires Days',
+            default: 365
+        }
+    },
+    type: 'object',
+    required: ['name', 'token'],
+    title: 'TokenIssued'
+} as const;
+
+export const TokenRequestSchema = {
+    properties: {
+        name: {
+            type: 'string',
+            title: 'Name'
+        }
+    },
+    type: 'object',
+    required: ['name'],
+    title: 'TokenRequest'
+} as const;
+
 export const TriggerRequestSchema = {
     properties: {
         values: {
@@ -2879,6 +2912,55 @@ refresh_s, range_s}\`\`: it publishes \`\`{range_s, interval_s}\`\` to
 a flow answers with on \`\`message\`\`.`
 } as const;
 
+export const WorkerInfoSchema = {
+    properties: {
+        name: {
+            type: 'string',
+            title: 'Name'
+        },
+        labels: {
+            items: {
+                type: 'string'
+            },
+            type: 'array',
+            title: 'Labels'
+        },
+        max_parallel: {
+            type: 'integer',
+            title: 'Max Parallel',
+            default: 1
+        },
+        in_flight: {
+            type: 'integer',
+            title: 'In Flight',
+            default: 0
+        },
+        attached_at: {
+            type: 'number',
+            title: 'Attached At',
+            default: 0
+        },
+        last_seen: {
+            type: 'number',
+            title: 'Last Seen',
+            default: 0
+        },
+        python: {
+            type: 'string',
+            title: 'Python',
+            default: ''
+        },
+        venv_digest: {
+            type: 'string',
+            title: 'Venv Digest',
+            default: ''
+        }
+    },
+    type: 'object',
+    required: ['name'],
+    title: 'WorkerInfo'
+} as const;
+
 export const app__api__routes__dashboards__PublishRequestSchema = {
     properties: {
         version: {

frontend/src/client/sdk.gen.ts

@@ -3,7 +3,7 @@
 import type { CancelablePromise } from './core/CancelablePromise';
 import { OpenAPI } from './core/OpenAPI';
 import { request as __request } from './core/request';
-import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsReadRunsData, RunsReadRunsResponse, RunsReadRunData, RunsReadRunResponse, RunsCancelRunData, RunsCancelRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse } from './types.gen';
+import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsReadRunsData, RunsReadRunsResponse, RunsReadRunData, RunsReadRunResponse, RunsCancelRunData, RunsCancelRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersReadRuntimeResponse } from './types.gen';
 
 export class AlertsService {
     /**
@@ -1841,4 +1841,58 @@ export class UtilsService {
             url: '/api/v1/utils/health/'
         });
     }
+}
+
+export class WorkersService {
+    /**
+     * Read Workers
+     * What is attached, and how busy it is.
+     * @returns WorkerInfo Successful Response
+     * @throws ApiError
+     */
+    public static readWorkers(): CancelablePromise<WorkersReadWorkersResponse> {
+        return __request(OpenAPI, {
+            method: 'GET',
+            url: '/api/v1/workers'
+        });
+    }
+    
+    /**
+     * Issue Token
+     * Mint the credential a worker presents when it dials in.
+     *
+     * Shown once. It is signed with the same keypair the agent tokens use, so
+     * rotating that key revokes every worker along with them.
+     * @param data The data for the request.
+     * @param data.requestBody
+     * @returns TokenIssued Successful Response
+     * @throws ApiError
+     */
+    public static issueToken(data: WorkersIssueTokenData): CancelablePromise<WorkersIssueTokenResponse> {
+        return __request(OpenAPI, {
+            method: 'POST',
+            url: '/api/v1/workers/tokens',
+            body: data.requestBody,
+            mediaType: 'application/json',
+            errors: {
+                422: 'Validation Error'
+            }
+        });
+    }
+    
+    /**
+     * Read Runtime
+     * The worker's own code, so a fresh host installs by fetching one file.
+     *
+     * It is the same module the engine's local workers run — deliberately
+     * standard library only, and with nothing of the app importable in it.
+     * @returns string Successful Response
+     * @throws ApiError
+     */
+    public static readRuntime(): CancelablePromise<WorkersReadRuntimeResponse> {
+        return __request(OpenAPI, {
+            method: 'GET',
+            url: '/api/v1/workers/runtime'
+        });
+    }
 }

frontend/src/client/types.gen.ts

@@ -850,6 +850,16 @@ export type Token = {
     token_type?: string;
 };
 
+export type TokenIssued = {
+    name: string;
+    token: string;
+    expires_days?: number;
+};
+
+export type TokenRequest = {
+    name: string;
+};
+
 export type TriggerRequest = {
     values?: {
         [key: string]: unknown;
@@ -955,6 +965,17 @@ export type WidgetDef = {
 
 export type type = 'stat' | 'gauge' | 'chart' | 'markdown' | 'agenda' | 'notification' | 'button' | 'switch' | 'slider' | 'input' | 'dropdown';
 
+export type WorkerInfo = {
+    name: string;
+    labels?: Array<(string)>;
+    max_parallel?: number;
+    in_flight?: number;
+    attached_at?: number;
+    last_seen?: number;
+    python?: string;
+    venv_digest?: string;
+};
+
 export type AlertsReadAlertsConfigResponse = (AlertsConfig);
 
 export type AlertsSaveAlertsConfigData = {
@@ -1445,4 +1466,14 @@ export type UtilsHealthCheckResponse = (boolean);
 
 export type UtilsHealthResponse = ({
     [key: string]: unknown;
-});
+});
+
+export type WorkersReadWorkersResponse = (Array<WorkerInfo>);
+
+export type WorkersIssueTokenData = {
+    requestBody: TokenRequest;
+};
+
+export type WorkersIssueTokenResponse = (TokenIssued);
+
+export type WorkersReadRuntimeResponse = (string);