Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run python nodes out of process, with modules of their own

Browse Source 
User code no longer execs in the engine. A pool of persistent worker
subprocesses speaks one JSON object per line; the controller installs a
proxy as the node's function, so every execution path funnels through it
and the pipeline is untouched. A crash costs one subprocess, a per-node
timeout is a kill, and cancelling from the canvas is that same kill.

The workers run a venv of the user's own on the data volume, filled from
a pip manifest versioned beside the flows. Applying it retires the
workers and rebuilds, so a package lands without restarting the engine.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt
This commit is contained in:
stroblme
2026-08-16 21:43:36 +02:00
co-authored by Claude Fable 5
parent 979c9d3c1f
commit f300c43f3a
27 changed files with 1536 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/client/schemas.gen.ts
+108
View File
@@ -27,6 +27,36 @@ export const AlertsConfigSchema = {
description: 'The whole alerting setup, as stored and as the API sees it.'
} as const;
export const ApplyRequestSchema = {
properties: {
requirements: {
type: 'string',
title: 'Requirements',
default: ''
}
},
type: 'object',
title: 'ApplyRequest',
description: 'A pip manifest, one requirement per line.'
} as const;
export const ApplyResultSchema = {
properties: {
ok: {
type: 'boolean',
title: 'Ok'
},
output: {
type: 'string',
title: 'Output',
default: ''
}
},
type: 'object',
required: ['ok'],
title: 'ApplyResult'
} as const;
export const Body_login_login_access_tokenSchema = {
properties: {
grant_type: {
@@ -890,6 +920,58 @@ export const MessagesPublicSchema = {
title: 'MessagesPublic'
} as const;
export const ModulePackageSchema = {
properties: {
name: {
type: 'string',
title: 'Name'
},
version: {
type: 'string',
title: 'Version'
}
},
type: 'object',
required: ['name', 'version'],
title: 'ModulePackage',
description: 'One package installed in the venv node code runs on.'
} as const;
export const ModulesInfoSchema = {
properties: {
python_version: {
type: 'string',
title: 'Python Version',
default: ''
},
venv_path: {
type: 'string',
title: 'Venv Path',
default: ''
},
requirements: {
type: 'string',
title: 'Requirements',
default: ''
},
packages: {
items: {
'$ref': '#/components/schemas/ModulePackage'
},
type: 'array',
title: 'Packages'
},
applied: {
type: 'boolean',
title: 'Applied',
default: false
}
},
type: 'object',
title: 'ModulesInfo',
description: 'The venv node code imports from, and the manifest that describes it.'
} as const;
export const NewPasswordSchema = {
properties: {
token: {
@@ -960,6 +1042,19 @@ export const NodeDef_InputSchema = {
}
],
title: 'Source Ref'
},
timeout: {
anyOf: [
{
type: 'number',
exclusiveMinimum: 0
},
{
type: 'null'
}
],
title: 'Timeout',
description: "Seconds this node's code may run before it is stopped. Above 60 the engine may deliver its work again while it is still running."
}
},
type: 'object',
@@ -1020,6 +1115,19 @@ export const NodeDef_OutputSchema = {
}
],
title: 'Source Ref'
},
timeout: {
anyOf: [
{
type: 'number',
exclusiveMinimum: 0
},
{
type: 'null'
}
],
title: 'Timeout',
description: "Seconds this node's code may run before it is stopped. Above 60 the engine may deliver its work again while it is still running."
}
},
type: 'object',
frontend/src/client/sdk.gen.ts
+65 -1
View File
@@ -3,7 +3,7 @@
import type { CancelablePromise } from './core/CancelablePromise';
import { OpenAPI } from './core/OpenAPI';
import { request as __request } from './core/request';
import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsTestChannelData, AlertsTestChannelResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, PrivateCreateUserData, PrivateCreateUserResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse } from './types.gen';
import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsTestChannelData, AlertsTestChannelResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, PrivateCreateUserData, PrivateCreateUserResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse } from './types.gen';
export class AlertsService {
/**
@@ -670,6 +670,32 @@ export class FlowsService {
});
}
/**
* Cancel Node
* Stop a node that is running right now, by killing the worker running it.
*
* Idempotent on purpose: by the time a click reaches here the node may well
* have finished, and that is the outcome that was asked for.
* @param data The data for the request.
* @param data.name
* @param data.nodeId
* @returns Message Successful Response
* @throws ApiError
*/
public static cancelNode(data: FlowsCancelNodeData): CancelablePromise<FlowsCancelNodeResponse> {
return __request(OpenAPI, {
method: 'POST',
url: '/api/v1/flows/{name}/nodes/{node_id}/cancel',
path: {
name: data.name,
node_id: data.nodeId
},
errors: {
422: 'Validation Error'
}
});
}
/**
* Read Flow State
* The last value seen on every message of this flow.
@@ -878,6 +904,44 @@ export class MessagesService {
}
}
export class ModulesService {
/**
* Read Modules
* What node code can import, and whether it matches the manifest.
* @returns ModulesInfo Successful Response
* @throws ApiError
*/
public static readModules(): CancelablePromise<ModulesReadModulesResponse> {
return __request(OpenAPI, {
method: 'GET',
url: '/api/v1/modules/'
});
}
/**
* Apply Modules
* Install exactly these requirements, then hand them to the workers.
*
* A manifest that does not resolve changes nothing: the venv is left as it
* was and the stored manifest is only written once the install succeeded.
* @param data The data for the request.
* @param data.requestBody
* @returns ApplyResult Successful Response
* @throws ApiError
*/
public static applyModules(data: ModulesApplyModulesData): CancelablePromise<ModulesApplyModulesResponse> {
return __request(OpenAPI, {
method: 'POST',
url: '/api/v1/modules/apply',
body: data.requestBody,
mediaType: 'application/json',
errors: {
422: 'Validation Error'
}
});
}
}
export class OauthService {
/**
* Register Client
frontend/src/client/types.gen.ts
+54
View File
@@ -43,6 +43,18 @@ export type app__flow__schemas__MessageValue = {
ts?: (number | null);
};
/**
* A pip manifest, one requirement per line.
*/
export type ApplyRequest = {
requirements?: string;
};
export type ApplyResult = {
ok: boolean;
output?: string;
};
export type Body_login_login_access_token = {
grant_type?: (string | null);
username: string;
@@ -308,6 +320,25 @@ export type MessagesPublic = {
count: number;
};
/**
* One package installed in the venv node code runs on.
*/
export type ModulePackage = {
name: string;
version: string;
};
/**
* The venv node code imports from, and the manifest that describes it.
*/
export type ModulesInfo = {
python_version?: string;
venv_path?: string;
requirements?: string;
packages?: Array<ModulePackage>;
applied?: boolean;
};
export type NewPassword = {
token: string;
new_password: string;
@@ -327,6 +358,10 @@ export type NodeDef_Input = {
requires?: Array<MessageSpec>;
provides?: Array<MessageSpec>;
source_ref?: (string | null);
/**
* Seconds this node's code may run before it is stopped. Above 60 the engine may deliver its work again while it is still running.
*/
timeout?: (number | null);
};
/**
@@ -343,6 +378,10 @@ export type NodeDef_Output = {
requires?: Array<MessageSpec>;
provides?: Array<MessageSpec>;
source_ref?: (string | null);
/**
* Seconds this node's code may run before it is stopped. Above 60 the engine may deliver its work again while it is still running.
*/
timeout?: (number | null);
};
/**
@@ -805,6 +844,13 @@ export type FlowsTriggerNodeData = {
export type FlowsTriggerNodeResponse = (FlowStatePublic);
export type FlowsCancelNodeData = {
name: string;
nodeId: string;
};
export type FlowsCancelNodeResponse = (Message);
export type FlowsReadFlowStateData = {
name: string;
};
@@ -859,6 +905,14 @@ export type MessagesReadMessageHistoryData = {
export type MessagesReadMessageHistoryResponse = (MessagePoints);
export type ModulesReadModulesResponse = (ModulesInfo);
export type ModulesApplyModulesData = {
requestBody: ApplyRequest;
};
export type ModulesApplyModulesResponse = (ApplyResult);
export type OauthRegisterClientData = {
requestBody: OAuthClientRegister;
};