Add a Python SDK: flows declared in your own repository

A data scientist keeps their code where it is and decorates it: `@node`
declares a function's ports beside the function, `Flow(name, nodes=[...])`
says which of them make a flow, and `use(fn, wire=..., **settings)` rebinds
one for a single flow. `fluksio sync` uploads the document plus a generated
import shim per node, so the store still holds a complete, runnable,
git-versioned definition while the code it imports stays theirs.

`fluksio login|run|runs` and `flow.submit().wait()` are the client half, over
the run endpoints that already existed. Runs record the user repository's
commit beside the store's, so "what code produced this number" is answerable
on the side that now holds the code.

- `fluksio/sdk/`: ports, decorators, the flow builder and its checks, the shim
  generator, an HTTP client and sync. Standard library only at import, so
  `from fluksio import node` in a training script pulls in no engine.
- `FlowDef.origin` marks a flow code-defined; `Run.origin_commit` carries the
  repository's commit; `POST /modules/refresh` retires the workers without an
  install, which every sync calls — a worker holds the imported package in
  memory, so an edit to it is invisible until the process goes.
- The canvas shows a generated body read-only and names the repository to edit
  instead; a body edited there stops the next sync rather than being discarded.
- The worker's reporter carries inert `Port`, `node`, `use` and `Flow`, since
  the shim imports a module whose first line declares them.
- `examples/myresearch` is the worked example, `make sync-example` uploads it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ue1tkFWB1bcGy3aWhCKpU
This commit is contained in:
2026-08-23 20:16:08 +02:00
co-authored by Claude Fable 5
parent 0a49b6e947
commit 19bc2810cf
35 changed files with 2693 additions and 142 deletions
+72
View File
@@ -764,6 +764,17 @@ export const FlowDef_InputSchema = {
type: 'array',
title: 'Outputs',
description: 'Messages a batch run reports as its result, unqualified. Empty means every message the flow ends up holding.'
},
origin: {
anyOf: [
{
'$ref': '#/components/schemas/FlowOrigin'
},
{
type: 'null'
}
],
description: 'Set when the flow was declared in code elsewhere and uploaded by `fluksio sync`. Absent for a flow drawn on the canvas.'
}
},
type: 'object',
@@ -816,6 +827,17 @@ export const FlowDef_OutputSchema = {
type: 'array',
title: 'Outputs',
description: 'Messages a batch run reports as its result, unqualified. Empty means every message the flow ends up holding.'
},
origin: {
anyOf: [
{
'$ref': '#/components/schemas/FlowOrigin'
},
{
type: 'null'
}
],
description: 'Set when the flow was declared in code elsewhere and uploaded by `fluksio sync`. Absent for a flow drawn on the canvas.'
}
},
type: 'object',
@@ -921,6 +943,46 @@ export const FlowInput_OutputSchema = {
description: 'A message the flow starts with rather than computes.'
} as const;
export const FlowOriginSchema = {
properties: {
kind: {
type: 'string',
const: 'python',
title: 'Kind',
default: 'python'
},
repo: {
type: 'string',
title: 'Repo',
default: ''
},
commit: {
type: 'string',
title: 'Commit',
default: ''
},
dirty: {
type: 'boolean',
title: 'Dirty',
default: false
}
},
type: 'object',
title: 'FlowOrigin',
description: `Where a flow was declared, when that was somewhere other than here.
A flow drawn on the canvas has no origin: the store is where it lives. One
stamped with this was declared with the decorators in somebody's own
repository and put here by \`\`fluksio sync\`\`, so the node bodies below it
are generated imports and the code they run is versioned twice — once here
and once there. Its presence is what makes a flow code-defined.
Deliberately no timestamp. The store commits every change it is given, so
when a flow was last synced is a fact its own history already holds — and
one that would otherwise change on every sync, making an unchanged upload
look like a new version of the flow.`
} as const;
export const FlowRollupSchema = {
properties: {
flow: {
@@ -2369,6 +2431,11 @@ export const RunDetailSchema = {
type: 'string',
title: 'Params Digest'
},
origin_commit: {
type: 'string',
title: 'Origin Commit',
default: ''
},
seed: {
anyOf: [
{
@@ -3475,6 +3542,11 @@ export const fluksio__api__routes__runs__RunRowSchema = {
type: 'string',
title: 'Params Digest'
},
origin_commit: {
type: 'string',
title: 'Origin Commit',
default: ''
},
seed: {
anyOf: [
{
+26 -1
View File
@@ -3,7 +3,7 @@
import type { CancelablePromise } from './core/CancelablePromise';
import { OpenAPI } from './core/OpenAPI';
import { request as __request } from './core/request';
import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudDisconnectResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsReadRunsData, RunsReadRunsResponse, RunsReadRunData, RunsReadRunResponse, RunsCancelRunData, RunsCancelRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersReadRuntimeResponse } from './types.gen';
import type { AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudDisconnectResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsRenameDashboardData, DashboardsRenameDashboardResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsRenameFlowData, FlowsRenameFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, PanelsReadPanelsResponse, PanelsSavePanelsData, PanelsSavePanelsResponse, PanelsStartPairingResponse, PanelsPollPairingData, PanelsPollPairingResponse, PanelsPendingDeviceData, PanelsPendingDeviceResponse, PanelsApprovePairingData, PanelsApprovePairingResponse, PanelsUnpairPanelData, PanelsUnpairPanelResponse, PanelsReadPanelData, PanelsReadPanelResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsReadRunsData, RunsReadRunsResponse, RunsReadRunData, RunsReadRunResponse, RunsCancelRunData, RunsCancelRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersReadRuntimeResponse } from './types.gen';
export class AlertsService {
/**
@@ -1160,6 +1160,12 @@ export class ModulesService {
*
* A manifest that does not resolve changes nothing: the venv is left as it
* was and the stored manifest is only written once the install succeeded.
*
* Only the flows already holding a node that would not load are rebuilt,
* because those are the ones an install is called to fix. A flow that this
* install *breaks* — a package taken back out from under it — is still green
* and fails at call time with the node author's own import error, until
* something rebuilds it.
* @param data The data for the request.
* @param data.requestBody
* @returns ApplyResult Successful Response
@@ -1176,6 +1182,25 @@ export class ModulesService {
}
});
}
/**
* Refresh Modules
* Retire the workers without installing anything.
*
* A node that imports the caller's own package holds it in `sys.modules` for
* as long as the process lives, so editing that package changes nothing a
* running worker can see — recompiling the node would not help either, since
* the import returns the module already there. Retiring the processes is the
* whole of it, and `fluksio sync` asks for it after every upload.
* @returns Message Successful Response
* @throws ApiError
*/
public static refreshModules(): CancelablePromise<ModulesRefreshModulesResponse> {
return __request(OpenAPI, {
method: 'POST',
url: '/api/v1/modules/refresh'
});
}
}
export class OauthService {
+33
View File
@@ -235,6 +235,10 @@ export type FlowDef_Input = {
* Messages a batch run reports as its result, unqualified. Empty means every message the flow ends up holding.
*/
outputs?: Array<(string)>;
/**
* Set when the flow was declared in code elsewhere and uploaded by `fluksio sync`. Absent for a flow drawn on the canvas.
*/
origin?: (FlowOrigin | null);
};
/**
@@ -259,6 +263,10 @@ export type FlowDef_Output = {
* Messages a batch run reports as its result, unqualified. Empty means every message the flow ends up holding.
*/
outputs?: Array<(string)>;
/**
* Set when the flow was declared in code elsewhere and uploaded by `fluksio sync`. Absent for a flow drawn on the canvas.
*/
origin?: (FlowOrigin | null);
};
/**
@@ -294,6 +302,27 @@ export type FlowInput_Output = {
initial?: (unknown | null);
};
/**
* Where a flow was declared, when that was somewhere other than here.
*
* A flow drawn on the canvas has no origin: the store is where it lives. One
* stamped with this was declared with the decorators in somebody's own
* repository and put here by ``fluksio sync``, so the node bodies below it
* are generated imports and the code they run is versioned twice — once here
* and once there. Its presence is what makes a flow code-defined.
*
* Deliberately no timestamp. The store commits every change it is given, so
* when a flow was last synced is a fact its own history already holds — and
* one that would otherwise change on every sync, making an unchanged upload
* look like a new version of the flow.
*/
export type FlowOrigin = {
kind?: "python";
repo?: string;
commit?: string;
dirty?: boolean;
};
export type FlowRollup = {
flow: string;
executions: number;
@@ -385,6 +414,7 @@ export type fluksio__api__routes__runs__RunRow = {
[key: string]: unknown;
};
params_digest: string;
origin_commit?: string;
seed: (number | null);
group_id: (string | null);
labels: Array<(string)>;
@@ -845,6 +875,7 @@ export type RunDetail = {
[key: string]: unknown;
};
params_digest: string;
origin_commit?: string;
seed: (number | null);
group_id: (string | null);
labels: Array<(string)>;
@@ -1407,6 +1438,8 @@ export type ModulesApplyModulesData = {
export type ModulesApplyModulesResponse = (ApplyResult);
export type ModulesRefreshModulesResponse = (Message);
export type OauthRegisterClientData = {
requestBody: OAuthClientRegister;
};
@@ -1123,6 +1123,20 @@ function FlowEditorInner({
<span className="truncate px-3 py-1.5 text-sm font-medium">
{flowDoc.title || flowName}
</span>
{/* Declared in code somewhere else, at a commit that names what
actually ran. A dirty tree says so, because then it does not. */}
{flowDoc.origin ? (
<span
className="shrink-0 rounded-full bg-muted px-2 py-0.5 font-mono text-xs text-muted-foreground"
title={flowDoc.origin.repo || "a repository"}
data-testid="flow-origin"
>
{flowDoc.origin.commit
? flowDoc.origin.commit.slice(0, 7)
: "no commit"}
{flowDoc.origin.dirty ? "*" : ""}
</span>
) : null}
</CanvasTitle>
<FlowDock
@@ -1198,6 +1212,7 @@ function FlowEditorInner({
node={selected}
flow={flowName}
nodeTypes={nodeTypeInfo ?? []}
origin={flowDoc.origin}
suggestions={suggestions}
expanded={editorExpanded}
onToggleExpand={() => setEditorExpanded((wide) => !wide)}
@@ -13,9 +13,12 @@ import { monacoFontFamily, setupMonaco } from "./monacoSetup"
export default function NodeEditor({
value,
onChange,
readOnly = false,
}: {
value: string
onChange: (next: string) => void
/** Generated code: readable, and not this panel's to change. */
readOnly?: boolean
}) {
const { resolvedTheme } = useTheme()
const [ready, setReady] = useState(false)
@@ -37,6 +40,7 @@ export default function NodeEditor({
onChange={(next) => onChange(next ?? "")}
loading={<Skeleton className="h-full w-full rounded-md" />}
options={{
readOnly,
fontFamily,
fontSize: 13,
minimap: { enabled: false },
+22 -1
View File
@@ -11,6 +11,7 @@ import {
import {
type DType,
type FlowOrigin,
FlowsService,
type MessageSpec,
type NodeDef_Input,
@@ -997,6 +998,7 @@ function PanelBody({
node,
flow,
nodeType,
origin,
suggestions,
expanded,
onChange,
@@ -1008,6 +1010,7 @@ function PanelBody({
node: NodeDef_Input
flow: string
nodeType: NodeTypeInfo | undefined
origin: FlowOrigin | null | undefined
suggestions: PortSuggestions
expanded: boolean
onChange: (next: NodeDef_Input) => void
@@ -1060,7 +1063,7 @@ function PanelBody({
const editNode = (next: NodeDef_Input) => {
onChange(next)
const current = code ?? source?.code
if (!hasSource || next.source_ref || current === undefined) return
if (!hasSource || next.source_ref || origin || current === undefined) return
const wanted = scaffoldFor(next)
if (current !== wanted && SCAFFOLD_SHAPE.test(current)) editCode(wanted)
}
@@ -1224,6 +1227,19 @@ function PanelBody({
{expanded ? <Minimize2 /> : <Maximize2 />}
</Button>
</div>
{origin ? (
/* The body below is an import of the real function, and the real
function is somewhere else. Editing it here would be undone by
the next sync, so it is read-only and says where to go. */
<p
className="text-xs text-muted-foreground"
data-testid="node-source-generated"
>
Generated by <span className="font-mono">fluksio sync</span> from{" "}
<span className="font-mono">{origin.repo || "a repository"}</span>{" "}
edit it there and sync again.
</p>
) : null}
<div className="min-h-0 flex-1 overflow-hidden rounded-md border border-border">
<Suspense
fallback={
@@ -1233,6 +1249,7 @@ function PanelBody({
<NodeEditor
value={code ?? source?.code ?? ""}
onChange={editCode}
readOnly={Boolean(origin)}
/>
</Suspense>
</div>
@@ -1253,6 +1270,7 @@ export function NodePanel({
node,
flow,
nodeTypes,
origin,
suggestions,
expanded,
onChange,
@@ -1266,6 +1284,8 @@ export function NodePanel({
node: NodeDef_Input | null
flow: string
nodeTypes: NodeTypeInfo[]
/** Set when the flow was declared in code elsewhere; its bodies are generated. */
origin: FlowOrigin | null | undefined
suggestions: PortSuggestions
expanded: boolean
onChange: (next: NodeDef_Input) => void
@@ -1318,6 +1338,7 @@ export function NodePanel({
node={node}
flow={flow}
nodeType={nodeType}
origin={origin}
suggestions={suggestions}
expanded={expanded}
onChange={onChange}