Put resources where somebody would look for them

`resources` was reachable from the SDK and the API and nowhere else, and
`node_queued` was published to nothing at all — so a node waiting for a machine
looked exactly like a node that had hung, which is the failure the event was
added for.

The node panel gets a Resources section: a size by name, the numbers for a node
that wants its own, and how long it is expected to take. Picking a flavor drops
the numbers, because saying both is two answers and the engine refuses it.

A queued node draws a neutral dot rather than one of the three status colours —
it is not running, it did not go well, and it did not go wrong; it is idle with
a reason, which the tooltip gives.

And a Workers screen, which is the first UI for any of this: every machine the
engine can reach with what is free of each, what is attached, what a cluster
has been asked for, and the sizes, editable. `fluksio status` grew a line of
the same.

The demo's training node already preferred a GPU worker, which was exactly the
declaration that used to be dropped, so it now says how much of that machine it
takes as well.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01A6HeySA27EkGANZN95QySW

frontend/scripts/capture-screenshots.mjs

@@ -91,10 +91,11 @@ for (const theme of ["light", "dark"]) {
     await captureDashboards(page, dir)
     await captureMedia(page, dir)
     await captureRuns(page, dir)
+    await captureWorkers(page, dir)
 
     await context.close()
     console.log(
-      `  wrote ${dir}/{website-hero,app-login,app-dashboard,app-flows,app-flow-panel,app-panel,app-runs,app-run,app-run-context}.png`,
+      `  wrote ${dir}/{website-hero,app-login,app-dashboard,app-flows,app-flow-panel,app-panel,app-runs,app-run,app-run-context,app-workers}.png`,
     )
   }
 }
@@ -232,6 +233,14 @@ async function captureMedia(page, dir) {
  * The flow editor, empty-handed if the instance has no flows yet: seeds one
  * with a node so the canvas and the node panel are both worth looking at.
  */
+/** Every machine a node can run on, and the sizes it can ask for. */
+async function captureWorkers(page, dir) {
+  await page.goto(`${APP_URL}/workers`, { waitUntil: "networkidle" })
+  await page.getByText(/^Sizes$/).waitFor({ timeout: 15000 })
+  await page.waitForTimeout(500)
+  await page.screenshot({ path: `${dir}/app-workers.png`, fullPage: true })
+}
+
 async function captureFlows(page, dir) {
   await page.goto(`${APP_URL}/flows`, { waitUntil: "networkidle" })
 

frontend/src/client/schemas.gen.ts

@@ -722,6 +722,154 @@ export const EventRowSchema = {
     title: 'EventRow'
 } as const;
 
+export const FlavorCreateSchema = {
+    properties: {
+        cpus: {
+            type: 'integer',
+            minimum: 1,
+            title: 'Cpus',
+            default: 1
+        },
+        gpus: {
+            type: 'integer',
+            minimum: 0,
+            title: 'Gpus',
+            default: 0
+        },
+        ram: {
+            type: 'integer',
+            minimum: 1,
+            title: 'Ram',
+            default: 2048
+        },
+        description: {
+            type: 'string',
+            maxLength: 255,
+            title: 'Description',
+            default: ''
+        },
+        name: {
+            type: 'string',
+            maxLength: 64,
+            title: 'Name'
+        }
+    },
+    type: 'object',
+    required: ['name'],
+    title: 'FlavorCreate'
+} as const;
+
+export const FlavorPublicSchema = {
+    properties: {
+        cpus: {
+            type: 'integer',
+            minimum: 1,
+            title: 'Cpus',
+            default: 1
+        },
+        gpus: {
+            type: 'integer',
+            minimum: 0,
+            title: 'Gpus',
+            default: 0
+        },
+        ram: {
+            type: 'integer',
+            minimum: 1,
+            title: 'Ram',
+            default: 2048
+        },
+        description: {
+            type: 'string',
+            maxLength: 255,
+            title: 'Description',
+            default: ''
+        },
+        name: {
+            type: 'string',
+            title: 'Name'
+        }
+    },
+    type: 'object',
+    required: ['name'],
+    title: 'FlavorPublic'
+} as const;
+
+export const FlavorUpdateSchema = {
+    properties: {
+        cpus: {
+            anyOf: [
+                {
+                    type: 'integer',
+                    minimum: 1
+                },
+                {
+                    type: 'null'
+                }
+            ],
+            title: 'Cpus'
+        },
+        gpus: {
+            anyOf: [
+                {
+                    type: 'integer',
+                    minimum: 0
+                },
+                {
+                    type: 'null'
+                }
+            ],
+            title: 'Gpus'
+        },
+        ram: {
+            anyOf: [
+                {
+                    type: 'integer',
+                    minimum: 1
+                },
+                {
+                    type: 'null'
+                }
+            ],
+            title: 'Ram'
+        },
+        description: {
+            anyOf: [
+                {
+                    type: 'string',
+                    maxLength: 255
+                },
+                {
+                    type: 'null'
+                }
+            ],
+            title: 'Description'
+        }
+    },
+    type: 'object',
+    title: 'FlavorUpdate',
+    description: 'Everything but the name: renaming would orphan the nodes that ask.'
+} as const;
+
+export const FlavorsPublicSchema = {
+    properties: {
+        data: {
+            items: {
+                '$ref': '#/components/schemas/FlavorPublic'
+            },
+            type: 'array',
+            title: 'Data'
+        },
+        count: {
+            type: 'integer',
+            title: 'Count'
+        }
+    },
+    type: 'object',
+    required: ['data', 'count'],
+    title: 'FlavorsPublic'
+} as const;
+
 export const FlowDef_InputSchema = {
     properties: {
         name: {
@@ -2364,6 +2512,22 @@ export const RemoteUserBodySchema = {
     title: 'RemoteUserBody'
 } as const;
 
+export const ResourceLevelSchema = {
+    properties: {
+        total: {
+            type: 'integer',
+            title: 'Total'
+        },
+        free: {
+            type: 'integer',
+            title: 'Free'
+        }
+    },
+    type: 'object',
+    required: ['total', 'free'],
+    title: 'ResourceLevel'
+} as const;
+
 export const ResourcesSchema = {
     properties: {
         cpus: {
@@ -2380,6 +2544,44 @@ export const ResourcesSchema = {
             description: 'Whole devices held for the whole execution, named to the node through CUDA_VISIBLE_DEVICES. Nothing else is given them while it runs, which is what keeps two preallocating processes apart.',
             default: 0
         },
+        ram: {
+            anyOf: [
+                {
+                    type: 'integer',
+                    minimum: 1
+                },
+                {
+                    type: 'null'
+                }
+            ],
+            title: 'Ram',
+            description: "Megabytes held for the whole execution; accepts '512M' or '2G'. Counted against machines that said how much they have, and ignored by those that did not — which is a machine with nothing to say about memory, not one with none."
+        },
+        flavor: {
+            anyOf: [
+                {
+                    type: 'string'
+                },
+                {
+                    type: 'null'
+                }
+            ],
+            title: 'Flavor',
+            description: 'A stored size by name, standing in for cpus, gpus and ram. Read again every time the node is built, so editing the flavor edits what the next run gets.'
+        },
+        duration_s: {
+            anyOf: [
+                {
+                    type: 'integer',
+                    minimum: 1
+                },
+                {
+                    type: 'null'
+                }
+            ],
+            title: 'Duration S',
+            description: "How long one execution is expected to take; accepts '30m' or '2h'. A statement about the node for whoever is planning around it, not a limit — the limit is `timeout`."
+        },
         env: {
             additionalProperties: {
                 type: 'string'
@@ -2407,6 +2609,42 @@ arrives. Both are a node saying how much of the machine it takes, which is
 what this is.`
 } as const;
 
+export const ResourcesSnapshotSchema = {
+    properties: {
+        cpus: {
+            '$ref': '#/components/schemas/ResourceLevel'
+        },
+        gpus: {
+            '$ref': '#/components/schemas/ResourceLevel'
+        },
+        waiting: {
+            items: {
+                '$ref': '#/components/schemas/WaitingNode'
+            },
+            type: 'array',
+            title: 'Waiting'
+        },
+        targets: {
+            items: {
+                '$ref': '#/components/schemas/TargetResources'
+            },
+            type: 'array',
+            title: 'Targets'
+        },
+        provisioners: {
+            items: {
+                additionalProperties: true,
+                type: 'object'
+            },
+            type: 'array',
+            title: 'Provisioners'
+        }
+    },
+    type: 'object',
+    required: ['cpus', 'gpus'],
+    title: 'ResourcesSnapshot'
+} as const;
+
 export const RuleSchema = {
     properties: {
         events: {
@@ -2877,6 +3115,47 @@ export const SweepEntrySchema = {
     title: 'SweepEntry'
 } as const;
 
+export const TargetResourcesSchema = {
+    properties: {
+        target: {
+            type: 'string',
+            title: 'Target'
+        },
+        cpus: {
+            '$ref': '#/components/schemas/ResourceLevel'
+        },
+        gpus: {
+            '$ref': '#/components/schemas/ResourceLevel'
+        },
+        ram_mb: {
+            anyOf: [
+                {
+                    '$ref': '#/components/schemas/ResourceLevel'
+                },
+                {
+                    type: 'null'
+                }
+            ]
+        },
+        labels: {
+            items: {
+                type: 'string'
+            },
+            type: 'array',
+            title: 'Labels'
+        },
+        in_flight: {
+            type: 'integer',
+            title: 'In Flight',
+            default: 0
+        }
+    },
+    type: 'object',
+    required: ['target', 'cpus', 'gpus'],
+    title: 'TargetResources',
+    description: 'One machine: this engine, or a worker attached to it.'
+} as const;
+
 export const TokenSchema = {
     properties: {
         access_token: {
@@ -3326,6 +3605,26 @@ export const ValidationResultSchema = {
     title: 'ValidationResult'
 } as const;
 
+export const WaitingNodeSchema = {
+    properties: {
+        node: {
+            type: 'string',
+            title: 'Node'
+        },
+        reason: {
+            type: 'string',
+            title: 'Reason'
+        },
+        seconds: {
+            type: 'number',
+            title: 'Seconds'
+        }
+    },
+    type: 'object',
+    required: ['node', 'reason', 'seconds'],
+    title: 'WaitingNode'
+} as const;
+
 export const WidgetDefSchema = {
     properties: {
         id: {
@@ -3427,6 +3726,27 @@ export const WorkerInfoSchema = {
             type: 'string',
             title: 'Venv Digest',
             default: ''
+        },
+        cpus: {
+            type: 'integer',
+            title: 'Cpus',
+            default: 1
+        },
+        gpus: {
+            type: 'integer',
+            title: 'Gpus',
+            default: 0
+        },
+        ram_mb: {
+            anyOf: [
+                {
+                    type: 'integer'
+                },
+                {
+                    type: 'null'
+                }
+            ],
+            title: 'Ram Mb'
         }
     },
     type: 'object',

frontend/src/client/types.gen.ts

@@ -224,6 +224,37 @@ export type EventRow = {
     actor: string;
 };
 
+export type FlavorCreate = {
+    cpus?: number;
+    gpus?: number;
+    ram?: number;
+    description?: string;
+    name: string;
+};
+
+export type FlavorPublic = {
+    cpus?: number;
+    gpus?: number;
+    ram?: number;
+    description?: string;
+    name: string;
+};
+
+export type FlavorsPublic = {
+    data: Array<FlavorPublic>;
+    count: number;
+};
+
+/**
+ * Everything but the name: renaming would orphan the nodes that ask.
+ */
+export type FlavorUpdate = {
+    cpus?: (number | null);
+    gpus?: (number | null);
+    ram?: (number | null);
+    description?: (string | null);
+};
+
 /**
  * One atomic flow.
  */
@@ -866,6 +897,11 @@ export type RemoteUserBody = {
     code: string;
 };
 
+export type ResourceLevel = {
+    total: number;
+    free: number;
+};
+
 /**
  * What one execution of a node needs to have to itself.
  *
@@ -890,6 +926,18 @@ export type Resources = {
      * Whole devices held for the whole execution, named to the node through CUDA_VISIBLE_DEVICES. Nothing else is given them while it runs, which is what keeps two preallocating processes apart.
      */
     gpus?: number;
+    /**
+     * Megabytes held for the whole execution; accepts '512M' or '2G'. Counted against machines that said how much they have, and ignored by those that did not — which is a machine with nothing to say about memory, not one with none.
+     */
+    ram?: (number | null);
+    /**
+     * A stored size by name, standing in for cpus, gpus and ram. Read again every time the node is built, so editing the flavor edits what the next run gets.
+     */
+    flavor?: (string | null);
+    /**
+     * How long one execution is expected to take; accepts '30m' or '2h'. A statement about the node for whoever is planning around it, not a limit — the limit is `timeout`.
+     */
+    duration_s?: (number | null);
     /**
      * Extra environment for the worker this node runs in, applied over what the allocation derives. Where a library's own tuning goes — XLA_FLAGS, XLA_PYTHON_CLIENT_MEM_FRACTION — since those are composed strings the engine must not invent.
      */
@@ -898,6 +946,16 @@ export type Resources = {
     };
 };
 
+export type ResourcesSnapshot = {
+    cpus: ResourceLevel;
+    gpus: ResourceLevel;
+    waiting?: Array<WaitingNode>;
+    targets?: Array<TargetResources>;
+    provisioners?: Array<{
+        [key: string]: unknown;
+    }>;
+};
+
 /**
  * Which events go to which channels.
  */
@@ -1033,6 +1091,18 @@ export type SweepEntry = {
     idempotency_key?: (string | null);
 };
 
+/**
+ * One machine: this engine, or a worker attached to it.
+ */
+export type TargetResources = {
+    target: string;
+    cpus: ResourceLevel;
+    gpus: ResourceLevel;
+    ram_mb?: (ResourceLevel | null);
+    labels?: Array<(string)>;
+    in_flight?: number;
+};
+
 export type Token = {
     access_token: string;
     token_type?: string;
@@ -1134,6 +1204,12 @@ export type ValidationResult = {
     issues?: Array<ValidationIssue>;
 };
 
+export type WaitingNode = {
+    node: string;
+    reason: string;
+    seconds: number;
+};
+
 /**
  * One tile: what it shows or does, and where it sits.
  *
@@ -1184,6 +1260,9 @@ export type WorkerInfo = {
     last_seen?: number;
     python?: string;
     venv_digest?: string;
+    cpus?: number;
+    gpus?: number;
+    ram_mb?: (number | null);
 };
 
 export type AlertsReadAlertsConfigResponse = (AlertsConfig);
@@ -1286,6 +1365,27 @@ export type DashboardsRenameDashboardData = {
 
 export type DashboardsRenameDashboardResponse = (DashboardDef_Output);
 
+export type FlavorsReadFlavorsResponse = (FlavorsPublic);
+
+export type FlavorsCreateFlavorData = {
+    requestBody: FlavorCreate;
+};
+
+export type FlavorsCreateFlavorResponse = (FlavorPublic);
+
+export type FlavorsUpdateFlavorData = {
+    name: string;
+    requestBody: FlavorUpdate;
+};
+
+export type FlavorsUpdateFlavorResponse = (FlavorPublic);
+
+export type FlavorsDeleteFlavorData = {
+    name: string;
+};
+
+export type FlavorsDeleteFlavorResponse = (Message);
+
 export type FlowsReadFlowsResponse = (FlowsPublic);
 
 export type FlowsReadNodeTypesResponse = (Array<NodeTypeInfo>);
@@ -1760,7 +1860,7 @@ export type UtilsHealthResponse = ({
 
 export type WorkersReadWorkersResponse = (Array<WorkerInfo>);
 
-export type WorkersReadResourcesResponse = (unknown);
+export type WorkersReadResourcesResponse = (ResourcesSnapshot);
 
 export type WorkersIssueTokenData = {
     requestBody: TokenRequest;

frontend/src/components/Flow/FlowNode.tsx

@@ -60,7 +60,14 @@ const NODE_ICONS = {
 
 // One dot says everything about a node's state. Idle nodes carry no dot at all,
 // so the canvas stays quiet until something happens.
+// Queued is neutral on purpose: the three status colours mean "running",
+// "went well" and "went wrong", and a node waiting for a machine is none of
+// those — it is idle with a reason, which the tooltip gives.
 const STATUS_STYLES = {
+  queued: {
+    dot: "bg-muted-foreground",
+    label: "Queued — waiting for a machine",
+  },
   running: { dot: "bg-primary animate-pulse", label: "Running" },
   success: { dot: "bg-status-success", label: "Last run succeeded" },
   error: { dot: "bg-destructive", label: "Something went wrong" },
@@ -305,7 +312,7 @@ function FlowNodeComponent({ data, selected }: NodeProps) {
                   />
                 </TooltipTrigger>
                 <TooltipContent className="max-h-60 max-w-xs overflow-y-auto whitespace-pre-line break-words">
-                  {problem || style.label}
+                  {problem || live?.detail || style.label}
                 </TooltipContent>
               </Tooltip>
             ) : null}

frontend/src/components/Flow/NodePanel.tsx

@@ -42,6 +42,7 @@ import { cn } from "@/lib/utils"
 import { DtypeValue } from "./FlowBoundary"
 import { MessageSparkline } from "./MessageSparkline"
 import {
+  flavorsQueryOptions,
   flowKeys,
   libraryQueryOptions,
   nodeSourceQueryOptions,
@@ -81,6 +82,27 @@ const RESERVED_PARAMS = new Set(["synchronous"])
 /** A setting reaches the function by name, so the name has to be one. */
 const IDENTIFIER = /^[A-Za-z_]\w*$/
 
+/** Sentinels for the resources dropdown, which has two options that are not sizes. */
+const NO_RESOURCES = "__shared__"
+const CUSTOM_RESOURCES = "__custom__"
+
+/** `2h` and `30m` and `90` — the grammar the engine parses, written back. */
+function parseDuration(text: string): number | null {
+  const match = /^\s*(\d+)\s*([smhd])?\s*$/i.exec(text)
+  if (!match) return null
+  const unit = (match[2] ?? "s").toLowerCase()
+  const scale = { s: 1, m: 60, h: 3600, d: 86400 }[unit] ?? 1
+  return Number(match[1]) * scale
+}
+
+function formatDuration(seconds: number | null | undefined): string {
+  if (!seconds) return ""
+  if (seconds % 86400 === 0) return `${seconds / 86400}d`
+  if (seconds % 3600 === 0) return `${seconds / 3600}h`
+  if (seconds % 60 === 0) return `${seconds / 60}m`
+  return `${seconds}s`
+}
+
 /**
  * A text field that offers what is already in use elsewhere.
  *
@@ -828,6 +850,128 @@ function ParamsForm({
   )
 }
 
+/**
+ * How much of a machine this node takes, and how long it is expected to take.
+ *
+ * A named size is the usual answer: what "gpu-small" means is a property of the
+ * machines this installation has, and those change. The numbers are still there
+ * for a node that genuinely wants its own.
+ */
+function ResourcesSection({
+  node,
+  onChange,
+}: {
+  node: NodeDef_Input
+  onChange: (node: NodeDef_Input) => void
+}) {
+  const { data: flavors } = useQuery(flavorsQueryOptions())
+  const resources = node.resources ?? null
+  const choice = !resources
+    ? NO_RESOURCES
+    : (resources.flavor ?? CUSTOM_RESOURCES)
+
+  const edit = (next: NonNullable<NodeDef_Input["resources"]> | null) =>
+    onChange({ ...node, resources: next })
+
+  const pick = (value: string) => {
+    if (value === NO_RESOURCES) return edit(null)
+    // A flavor already says how much, so the numbers go with it — sending both
+    // is two answers, and the engine refuses it.
+    const kept = resources?.duration_s
+      ? { duration_s: resources.duration_s }
+      : {}
+    if (value === CUSTOM_RESOURCES) return edit({ cpus: 1, gpus: 0, ...kept })
+    return edit({ flavor: value, ...kept })
+  }
+
+  return (
+    <div className="grid gap-1.5">
+      <Label htmlFor="node-resources" className={SECTION}>
+        Resources
+      </Label>
+      <Select value={choice} onValueChange={pick}>
+        <SelectTrigger id="node-resources" className="h-8 text-sm">
+          <SelectValue />
+        </SelectTrigger>
+        <SelectContent>
+          <SelectItem value={NO_RESOURCES}>Shared pool</SelectItem>
+          {(flavors?.data ?? []).map((flavor) => (
+            <SelectItem key={flavor.name} value={flavor.name}>
+              {flavor.name} — {flavor.cpus} cpu ·{" "}
+              {Math.round((flavor.ram ?? 0) / 1024)} GB
+              {flavor.gpus ? ` · ${flavor.gpus} gpu` : ""}
+            </SelectItem>
+          ))}
+          <SelectItem value={CUSTOM_RESOURCES}>Custom…</SelectItem>
+        </SelectContent>
+      </Select>
+
+      {choice === CUSTOM_RESOURCES ? (
+        <div className="grid grid-cols-3 gap-1.5">
+          {(
+            [
+              ["cpus", "cores", 1],
+              ["gpus", "gpus", 0],
+              ["ram", "MB", 0],
+            ] as const
+          ).map(([field, label, floor]) => (
+            <div key={field} className="grid gap-1">
+              <Input
+                type="number"
+                min={floor}
+                className="h-8 text-sm"
+                aria-label={label}
+                placeholder={field === "ram" ? "unstated" : String(floor)}
+                value={
+                  resources?.[field] != null ? String(resources[field]) : ""
+                }
+                onChange={(event) => {
+                  const raw = event.target.value
+                  const value =
+                    raw === "" ? null : Math.max(floor, Number(raw) || 0)
+                  edit({
+                    ...resources,
+                    cpus: resources?.cpus ?? 1,
+                    // `ram` may be left unstated; the other two always have one.
+                    [field]: field === "ram" ? value : (value ?? floor),
+                  })
+                }}
+              />
+              <span className="text-xs text-muted-foreground">{label}</span>
+            </div>
+          ))}
+        </div>
+      ) : null}
+
+      {resources ? (
+        <Input
+          className="h-8 text-sm"
+          placeholder="expected duration — 30m, 2h"
+          defaultValue={formatDuration(resources.duration_s)}
+          key={`${choice}-${resources.duration_s ?? ""}`}
+          onBlur={(event) => {
+            const text = event.target.value.trim()
+            const seconds = text === "" ? null : parseDuration(text)
+            if (text !== "" && seconds === null) {
+              event.target.value = formatDuration(resources.duration_s)
+              return
+            }
+            if (seconds !== resources.duration_s) {
+              edit({ ...resources, duration_s: seconds })
+            }
+          }}
+        />
+      ) : null}
+
+      <p className="text-xs text-muted-foreground">
+        {resources
+          ? "Held for the whole execution, and what the node's own libraries are told they may use. The duration is a planning fact, not a limit."
+          : "Nothing held: the node shares the pool with every other node that says nothing."}
+      </p>
+    </div>
+  )
+}
+
 /**
  * Sharing a node moves its code to the library, where other flows can point at
  * it. Each flow keeps its own ports and settings; only the code is common, so
@@ -1209,6 +1353,9 @@ function PanelBody({
             </p>
           </div>
         ) : null}
+        {hasSource ? (
+          <ResourcesSection node={node} onChange={editNode} />
+        ) : null}
         {hasSource ? (
           <SharingSection flow={flow} node={node} onShared={onShared} />
         ) : null}

frontend/src/components/Flow/liveStore.ts

@@ -24,8 +24,10 @@ export type LiveValue = {
   source?: ValueSource
 }
 export type LiveStatus = {
-  status: "active" | "error" | "running" | "success"
+  status: "active" | "error" | "queued" | "running" | "success"
   error?: string | null
+  /** Why it is queued — what it is waiting for, and on which machine. */
+  detail?: string | null
 }
 /** How a node's connection is doing, which is not how its last run went. */
 export type NodeHealth = {

frontend/src/components/Flow/queries.ts

@@ -9,6 +9,7 @@ import { useCallback, useEffect, useRef, useState } from "react"
 
 import {
   ApiError,
+  FlavorsService,
   type FlowDef_Input,
   FlowsService,
   SecretsService,
@@ -44,6 +45,12 @@ export const secretsQueryOptions = () => ({
   queryFn: () => SecretsService.readSecrets(),
 })
 
+/** The named sizes a node can ask for, offered in the node panel. */
+export const flavorsQueryOptions = () => ({
+  queryKey: ["flavors"] as const,
+  queryFn: () => FlavorsService.readFlavors(),
+})
+
 /** Every flow at once, merged on what its nodes talk to. */
 export const graphQueryOptions = () => ({
   queryKey: flowKeys.graph,

frontend/src/components/Flow/useFlowSocket.ts

@@ -42,6 +42,14 @@ type FlowEvent =
       source?: ValueSource
     }
   | { type: "node_started"; node: string }
+  | {
+      type: "node_queued"
+      flow?: string
+      node: string
+      run?: string
+      detail?: string
+      ts?: number
+    }
   | {
       type: "node_executed"
       flow?: string
@@ -179,6 +187,14 @@ function connect() {
           source: message.source,
         })
         break
+      case "node_queued":
+        // Waiting for a machine, which looks exactly like hung from outside.
+        // `node_started` overwrites this, so nothing has to clear it.
+        liveStore.setStatus(message.node, {
+          status: "queued",
+          detail: message.detail,
+        })
+        break
       case "node_started":
         liveStore.setStatus(message.node, { status: "running" })
         break

frontend/src/components/Sidebar/AppSidebar.tsx

@@ -7,6 +7,7 @@ import {
   LayoutDashboard,
   LogOut,
   Package,
+  Server,
   Settings,
   Users,
   Workflow,
@@ -37,6 +38,7 @@ const baseItems: Item[] = [
   // sit here and not among the per-user tabs under Settings.
   { icon: KeyRound, title: "Secrets", path: "/secrets" },
   { icon: Package, title: "Modules", path: "/modules" },
+  { icon: Server, title: "Workers", path: "/workers" },
   { icon: Bell, title: "Alerts", path: "/alerts" },
 ]
 

frontend/src/routeTree.gen.ts

@@ -20,6 +20,7 @@ import { Route as LayoutIndexRouteImport } from './routes/_layout/index'
 import { Route as ViewNameRouteImport } from './routes/view.$name'
 import { Route as PanelIdRouteImport } from './routes/panel.$id'
 import { Route as OauthAuthorizeRouteImport } from './routes/oauth.authorize'
+import { Route as LayoutWorkersRouteImport } from './routes/_layout/workers'
 import { Route as LayoutSettingsRouteImport } from './routes/_layout/settings'
 import { Route as LayoutSecretsRouteImport } from './routes/_layout/secrets'
 import { Route as LayoutModulesRouteImport } from './routes/_layout/modules'
@@ -85,6 +86,11 @@ const OauthAuthorizeRoute = OauthAuthorizeRouteImport.update({
   path: '/oauth/authorize',
   getParentRoute: () => rootRouteImport,
 } as any)
+const LayoutWorkersRoute = LayoutWorkersRouteImport.update({
+  id: '/workers',
+  path: '/workers',
+  getParentRoute: () => LayoutRoute,
+} as any)
 const LayoutSettingsRoute = LayoutSettingsRouteImport.update({
   id: '/settings',
   path: '/settings',
@@ -152,6 +158,7 @@ export interface FileRoutesByFullPath {
   '/modules': typeof LayoutModulesRoute
   '/secrets': typeof LayoutSecretsRoute
   '/settings': typeof LayoutSettingsRoute
+  '/workers': typeof LayoutWorkersRoute
   '/oauth/authorize': typeof OauthAuthorizeRoute
   '/panel/$id': typeof PanelIdRoute
   '/view/$name': typeof ViewNameRoute
@@ -174,6 +181,7 @@ export interface FileRoutesByTo {
   '/modules': typeof LayoutModulesRoute
   '/secrets': typeof LayoutSecretsRoute
   '/settings': typeof LayoutSettingsRoute
+  '/workers': typeof LayoutWorkersRoute
   '/oauth/authorize': typeof OauthAuthorizeRoute
   '/panel/$id': typeof PanelIdRoute
   '/view/$name': typeof ViewNameRoute
@@ -198,6 +206,7 @@ export interface FileRoutesById {
   '/_layout/modules': typeof LayoutModulesRoute
   '/_layout/secrets': typeof LayoutSecretsRoute
   '/_layout/settings': typeof LayoutSettingsRoute
+  '/_layout/workers': typeof LayoutWorkersRoute
   '/oauth/authorize': typeof OauthAuthorizeRoute
   '/panel/$id': typeof PanelIdRoute
   '/view/$name': typeof ViewNameRoute
@@ -223,6 +232,7 @@ export interface FileRouteTypes {
     | '/modules'
     | '/secrets'
     | '/settings'
+    | '/workers'
     | '/oauth/authorize'
     | '/panel/$id'
     | '/view/$name'
@@ -245,6 +255,7 @@ export interface FileRouteTypes {
     | '/modules'
     | '/secrets'
     | '/settings'
+    | '/workers'
     | '/oauth/authorize'
     | '/panel/$id'
     | '/view/$name'
@@ -268,6 +279,7 @@ export interface FileRouteTypes {
     | '/_layout/modules'
     | '/_layout/secrets'
     | '/_layout/settings'
+    | '/_layout/workers'
     | '/oauth/authorize'
     | '/panel/$id'
     | '/view/$name'
@@ -373,6 +385,13 @@ declare module '@tanstack/react-router' {
       preLoaderRoute: typeof OauthAuthorizeRouteImport
       parentRoute: typeof rootRouteImport
     }
+    '/_layout/workers': {
+      id: '/_layout/workers'
+      path: '/workers'
+      fullPath: '/workers'
+      preLoaderRoute: typeof LayoutWorkersRouteImport
+      parentRoute: typeof LayoutRoute
+    }
     '/_layout/settings': {
       id: '/_layout/settings'
       path: '/settings'
@@ -472,6 +491,7 @@ interface LayoutRouteChildren {
   LayoutModulesRoute: typeof LayoutModulesRoute
   LayoutSecretsRoute: typeof LayoutSecretsRoute
   LayoutSettingsRoute: typeof LayoutSettingsRoute
+  LayoutWorkersRoute: typeof LayoutWorkersRoute
   LayoutIndexRoute: typeof LayoutIndexRoute
   LayoutRunsIdRoute: typeof LayoutRunsIdRoute
   LayoutDashboardsIndexRoute: typeof LayoutDashboardsIndexRoute
@@ -485,6 +505,7 @@ const LayoutRouteChildren: LayoutRouteChildren = {
   LayoutModulesRoute: LayoutModulesRoute,
   LayoutSecretsRoute: LayoutSecretsRoute,
   LayoutSettingsRoute: LayoutSettingsRoute,
+  LayoutWorkersRoute: LayoutWorkersRoute,
   LayoutIndexRoute: LayoutIndexRoute,
   LayoutRunsIdRoute: LayoutRunsIdRoute,
   LayoutDashboardsIndexRoute: LayoutDashboardsIndexRoute,

frontend/src/routes/_layout/workers.tsx

@@ -0,0 +1,439 @@
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"
+import { createFileRoute } from "@tanstack/react-router"
+import { Cpu, Pencil, Plus, Server, Trash2 } from "lucide-react"
+import { useState } from "react"
+
+import {
+  type ApiError,
+  type FlavorPublic,
+  FlavorsService,
+  WorkersService,
+} from "@/client"
+import { Button } from "@/components/ui/button"
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+} from "@/components/ui/dialog"
+import { Input } from "@/components/ui/input"
+import { Label } from "@/components/ui/label"
+import useAuth from "@/hooks/useAuth"
+import useCustomToast from "@/hooks/useCustomToast"
+import { handleError } from "@/utils"
+
+export const Route = createFileRoute("/_layout/workers")({
+  component: Workers,
+  head: () => ({
+    meta: [{ title: "Workers - Fluksio" }],
+  }),
+})
+
+/** The node panel's dropdown reads this too, so an edit refreshes it. */
+const flavorsKey = ["flavors"]
+
+const emptyFlavor = { name: "", cpus: 1, gpus: 0, ram: 2048, description: "" }
+
+/** Fresh enough to watch a node take a machine, slow enough to be free. */
+const REFETCH_MS = 5000
+
+function ago(seconds: number): string {
+  const since = Date.now() / 1000 - seconds
+  if (since < 60) return `${Math.max(0, Math.round(since))}s ago`
+  if (since < 3600) return `${Math.round(since / 60)}m ago`
+  return `${Math.round(since / 3600)}h ago`
+}
+
+type Level = { total: number; free: number }
+
+/** "cpu 2/18" — what is in use, out of what there is. */
+function Used({ label, level }: { label: string; level?: Level | null }) {
+  if (!level?.total) return null
+  return (
+    <span className="font-mono text-xs text-muted-foreground">
+      {label} {level.total - level.free}/{level.total}
+    </span>
+  )
+}
+
+function Machines() {
+  const { data, isError } = useQuery({
+    queryKey: ["workers", "resources"],
+    queryFn: () => WorkersService.readResources(),
+    refetchInterval: REFETCH_MS,
+  })
+
+  if (isError || !data) {
+    return (
+      <p className="text-sm text-muted-foreground">
+        This engine does not account for resources.
+      </p>
+    )
+  }
+
+  return (
+    <div className="grid gap-2">
+      {(data.targets ?? []).map((target) => (
+        <div
+          key={target.target}
+          className="flex flex-wrap items-center justify-between gap-3 rounded-lg border border-border bg-card p-3 shadow-e1"
+        >
+          <span className="flex items-center gap-2 text-sm">
+            <Cpu className="size-4 text-muted-foreground" />
+            {target.target}
+            {(target.labels ?? []).length > 0 ? (
+              <span className="font-mono text-xs text-muted-foreground">
+                {(target.labels ?? []).join(", ")}
+              </span>
+            ) : null}
+          </span>
+          <span className="flex items-center gap-3">
+            <Used label="cpu" level={target.cpus} />
+            <Used label="gpu" level={target.gpus} />
+            <Used label="MB" level={target.ram_mb} />
+          </span>
+        </div>
+      ))}
+
+      {(data.waiting ?? []).length > 0 ? (
+        <div className="grid gap-1 pt-1">
+          {(data.waiting ?? []).map((node) => (
+            <p key={node.node} className="text-xs text-muted-foreground">
+              <span className="font-mono">{node.node}</span> — {node.reason} ·{" "}
+              {node.seconds.toFixed(0)}s
+            </p>
+          ))}
+        </div>
+      ) : null}
+
+      {(data.provisioners ?? []).map((provisioner) => {
+        const outstanding = (provisioner.outstanding ?? []) as {
+          profile: string
+          job: string
+          seconds: number
+        }[]
+        return (
+          <p
+            key={String(provisioner.name)}
+            className="text-xs text-muted-foreground"
+          >
+            {String(provisioner.name)} can start{" "}
+            {(provisioner.profiles as string[]).join(", ")}
+            {outstanding.length > 0
+              ? ` — asked for ${outstanding
+                  .map((job) => `${job.profile} (job ${job.job})`)
+                  .join(", ")}`
+              : ""}
+            {provisioner.last_error
+              ? ` — last failed: ${String(provisioner.last_error)}`
+              : ""}
+          </p>
+        )
+      })}
+    </div>
+  )
+}
+
+function Attached() {
+  const { data } = useQuery({
+    queryKey: ["workers"],
+    queryFn: () => WorkersService.readWorkers(),
+    refetchInterval: REFETCH_MS,
+  })
+  const workers = data ?? []
+
+  if (workers.length === 0) {
+    return (
+      <p className="text-sm text-muted-foreground">
+        Nothing attached. A machine runs{" "}
+        <code className="font-mono">pip install fluksio-worker</code> and dials
+        in; nodes go to it by label, or because it has room and this engine does
+        not.
+      </p>
+    )
+  }
+
+  return (
+    <div className="grid gap-2">
+      {workers.map((worker) => (
+        <div
+          key={worker.name}
+          className="flex flex-wrap items-center justify-between gap-3 rounded-lg border border-border bg-card p-3 shadow-e1"
+        >
+          <span className="grid gap-0.5">
+            <span className="flex items-center gap-2 text-sm">
+              <Server className="size-4 text-muted-foreground" />
+              {worker.name}
+              {(worker.labels ?? []).length > 0 ? (
+                <span className="font-mono text-xs text-muted-foreground">
+                  {(worker.labels ?? []).join(", ")}
+                </span>
+              ) : null}
+            </span>
+            <span className="pl-6 font-mono text-xs text-muted-foreground">
+              {worker.cpus} cpu
+              {worker.gpus ? ` · ${worker.gpus} gpu` : ""}
+              {worker.ram_mb ? ` · ${Math.round(worker.ram_mb / 1024)} GB` : ""}{" "}
+              · {worker.in_flight}/{worker.max_parallel} in flight
+            </span>
+          </span>
+          <span className="text-xs text-muted-foreground">
+            seen {ago(worker.last_seen ?? 0)}
+          </span>
+        </div>
+      ))}
+    </div>
+  )
+}
+
+function Flavors() {
+  const { user } = useAuth()
+  const { data } = useQuery({
+    queryKey: flavorsKey,
+    queryFn: () => FlavorsService.readFlavors(),
+  })
+  const queryClient = useQueryClient()
+  const { showSuccessToast, showErrorToast } = useCustomToast()
+  const [editing, setEditing] = useState<typeof emptyFlavor | null>(null)
+  const [isNew, setIsNew] = useState(false)
+  const [pendingDelete, setPendingDelete] = useState<string | null>(null)
+
+  const refresh = () => queryClient.invalidateQueries({ queryKey: flavorsKey })
+
+  const save = useMutation({
+    mutationFn: (flavor: typeof emptyFlavor) =>
+      isNew
+        ? FlavorsService.createFlavor({ requestBody: flavor })
+        : FlavorsService.updateFlavor({
+            name: flavor.name,
+            requestBody: {
+              cpus: flavor.cpus,
+              gpus: flavor.gpus,
+              ram: flavor.ram,
+              description: flavor.description,
+            },
+          }),
+    onSuccess: (_result, flavor) => {
+      showSuccessToast(`Saved '${flavor.name}'`)
+      setEditing(null)
+    },
+    onError: handleError.bind(showErrorToast),
+    onSettled: refresh,
+  })
+
+  const remove = useMutation({
+    mutationFn: (name: string) => FlavorsService.deleteFlavor({ name }),
+    onSuccess: (_result, name) => {
+      showSuccessToast(`Deleted '${name}'`)
+      setPendingDelete(null)
+    },
+    onError: (error: ApiError) => {
+      handleError.call(showErrorToast, error)
+      setPendingDelete(null)
+    },
+    onSettled: refresh,
+  })
+
+  const flavors = data?.data ?? []
+  const mayEdit = Boolean(user?.is_superuser)
+
+  return (
+    <div className="grid gap-2">
+      {flavors.map((flavor: FlavorPublic) => (
+        <div
+          key={flavor.name}
+          className="flex flex-wrap items-center justify-between gap-3 rounded-lg border border-border bg-card p-3 shadow-e1"
+        >
+          <span className="grid gap-0.5">
+            <span className="text-sm">{flavor.name}</span>
+            <span className="font-mono text-xs text-muted-foreground">
+              {flavor.cpus} cpu · {Math.round((flavor.ram ?? 0) / 1024)} GB
+              {flavor.gpus ? ` · ${flavor.gpus} gpu` : ""}
+              {flavor.description ? ` — ${flavor.description}` : ""}
+            </span>
+          </span>
+          {mayEdit ? (
+            <span className="flex items-center gap-1">
+              <Button
+                variant="ghost"
+                size="icon-sm"
+                aria-label={`Edit ${flavor.name}`}
+                onClick={() => {
+                  setIsNew(false)
+                  setEditing({ ...emptyFlavor, ...flavor })
+                }}
+              >
+                <Pencil />
+              </Button>
+              <Button
+                variant="ghost"
+                size="icon-sm"
+                aria-label={`Delete ${flavor.name}`}
+                className="text-destructive hover:text-destructive"
+                onClick={() => setPendingDelete(flavor.name)}
+              >
+                <Trash2 />
+              </Button>
+            </span>
+          ) : null}
+        </div>
+      ))}
+
+      {mayEdit ? (
+        <Button
+          variant="secondary"
+          className="justify-self-start"
+          onClick={() => {
+            setIsNew(true)
+            setEditing({ ...emptyFlavor })
+          }}
+        >
+          <Plus />
+          Add a size
+        </Button>
+      ) : null}
+
+      <Dialog
+        open={editing !== null}
+        onOpenChange={(open) => !open && setEditing(null)}
+      >
+        <DialogContent>
+          <DialogHeader>
+            <DialogTitle>
+              {isNew ? "New size" : `Edit '${editing?.name}'`}
+            </DialogTitle>
+            <DialogDescription>
+              Every node that names this gets what it says here, from its next
+              run.
+            </DialogDescription>
+          </DialogHeader>
+          {editing ? (
+            <div className="grid gap-3">
+              {isNew ? (
+                <div className="grid gap-1.5">
+                  <Label htmlFor="flavor-name">Name</Label>
+                  <Input
+                    id="flavor-name"
+                    value={editing.name}
+                    placeholder="gpu-large"
+                    onChange={(event) =>
+                      setEditing({ ...editing, name: event.target.value })
+                    }
+                  />
+                </div>
+              ) : null}
+              <div className="grid grid-cols-3 gap-2">
+                {(
+                  [
+                    ["cpus", "Cores", 1],
+                    ["gpus", "GPUs", 0],
+                    ["ram", "MB", 1],
+                  ] as const
+                ).map(([field, label, floor]) => (
+                  <div key={field} className="grid gap-1.5">
+                    <Label htmlFor={`flavor-${field}`}>{label}</Label>
+                    <Input
+                      id={`flavor-${field}`}
+                      type="number"
+                      min={floor}
+                      value={String(editing[field])}
+                      onChange={(event) =>
+                        setEditing({
+                          ...editing,
+                          [field]: Math.max(
+                            floor,
+                            Number(event.target.value) || 0,
+                          ),
+                        })
+                      }
+                    />
+                  </div>
+                ))}
+              </div>
+              <div className="grid gap-1.5">
+                <Label htmlFor="flavor-description">Description</Label>
+                <Input
+                  id="flavor-description"
+                  value={editing.description}
+                  placeholder="What this size is for"
+                  onChange={(event) =>
+                    setEditing({ ...editing, description: event.target.value })
+                  }
+                />
+              </div>
+            </div>
+          ) : null}
+          <DialogFooter>
+            <Button variant="outline" onClick={() => setEditing(null)}>
+              Cancel
+            </Button>
+            <Button
+              disabled={!editing?.name.trim() || save.isPending}
+              onClick={() => editing && save.mutate(editing)}
+            >
+              Save
+            </Button>
+          </DialogFooter>
+        </DialogContent>
+      </Dialog>
+
+      <Dialog
+        open={pendingDelete !== null}
+        onOpenChange={(open) => !open && setPendingDelete(null)}
+      >
+        <DialogContent>
+          <DialogHeader>
+            <DialogTitle>Delete '{pendingDelete}'?</DialogTitle>
+            <DialogDescription>
+              A node still asking for it keeps this from being deleted, and it
+              will say which. Nothing is seeded back afterwards.
+            </DialogDescription>
+          </DialogHeader>
+          <DialogFooter>
+            <Button variant="outline" onClick={() => setPendingDelete(null)}>
+              Keep it
+            </Button>
+            <Button
+              variant="destructive"
+              disabled={remove.isPending}
+              onClick={() => pendingDelete && remove.mutate(pendingDelete)}
+            >
+              Delete
+            </Button>
+          </DialogFooter>
+        </DialogContent>
+      </Dialog>
+    </div>
+  )
+}
+
+function Workers() {
+  return (
+    <div className="grid gap-6">
+      <div className="grid gap-1">
+        <h1 className="text-2xl">Workers</h1>
+        <p className="text-sm text-muted-foreground">
+          Every machine this engine can run a node on, what is free of each, and
+          the sizes a node can ask for by name.
+        </p>
+      </div>
+
+      <section className="grid gap-2">
+        <h2 className="text-sm text-muted-foreground">Machines</h2>
+        <Machines />
+      </section>
+
+      <section className="grid gap-2">
+        <h2 className="text-sm text-muted-foreground">Attached workers</h2>
+        <Attached />
+      </section>
+
+      <section className="grid gap-2">
+        <h2 className="text-sm text-muted-foreground">Sizes</h2>
+        <Flavors />
+      </section>
+    </div>
+  )
+}