Keep a flow's runs when the flow goes

A run record is a record of what ran, so deleting a flow no longer sweeps its
history: `_forget_runs` is gone from the flow-delete path, and `DELETE /runs/{id}`
is the only thing that removes a run one at a time. The in-flight guard stays —
that is about work, not history.

`DELETE /runs?flow=` is the counterpart to the list's flow filter and what a
reseed needs, sharing the four statements with the single-run delete and
refusing the same way while a run of that flow is still going.

A run whose flow is gone reads as one: `useFlowInputs` reports the 404 rather
than an empty declaration set, so the run page says "flow deleted", explains it,
and disables Retry, which the route would refuse anyway.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CL9zvnnvcp1mvA8o7impxk
This commit is contained in:
2026-09-06 14:53:23 +02:00
co-authored by Claude Opus 5
parent 062a2aac60
commit 15c3dd5838
12 changed files with 276 additions and 80 deletions
+5 -23
View File
@@ -16,7 +16,6 @@ from fastapi import (
from fastapi.concurrency import run_in_threadpool
from jwt.exceptions import InvalidTokenError
from pydantic import BaseModel
from sqlalchemy import delete
from sqlmodel import Session, col, select
from fluksio.api.deps import (
@@ -59,7 +58,7 @@ from fluksio.flow.store import (
LibNotFound,
StaleVersion,
)
from fluksio.models import Flavor, Message, Run, RunArtifact, RunMetric, RunNode
from fluksio.models import Flavor, Message, Run
router = APIRouter(
prefix="/flows", tags=["flows"], dependencies=[Depends(get_current_user)]
@@ -242,25 +241,6 @@ def _audit(action: str, flow: str, user: CurrentUser) -> None:
)
def _forget_runs(session: Session, flow: str) -> None:
"""A deleted flow's runs, and everything hanging off them.
Here rather than in ``FlowController.forget_flow`` because renaming a flow
calls that too, and a rename must keep its experiment history.
Only the run tables: ``flow_run``, ``metric_minute`` and ``engine_event``
are the observability rollups, deliberately kept as a record of what ran
and already pruned at OBS_RETENTION_DAYS.
"""
# A subquery, not a materialised list of ids: a demo can hold thousands.
runs = select(col(Run.id)).where(col(Run.flow) == flow)
session.execute(delete(RunNode).where(col(RunNode.run_id).in_(runs)))
session.execute(delete(RunMetric).where(col(RunMetric.run_id).in_(runs)))
session.execute(delete(RunArtifact).where(col(RunArtifact.run_id).in_(runs)))
session.execute(delete(Run).where(col(Run.flow) == flow))
session.commit()
def _source_ref(definition: FlowDef, node_id: str) -> str | None:
"""The library source this node runs, if it is a shared one."""
node = next((n for n in definition.nodes if n.id == node_id), None)
@@ -483,7 +463,7 @@ async def discard_draft(name: str, controller: FlowControllerDep) -> Any:
async def delete_flow(
name: str, controller: FlowControllerDep, user: CurrentUser, session: SessionDep
) -> Any:
"""Delete a flow and everything in it."""
"""Delete a flow and everything in it, except the record of what it ran."""
live = session.exec(
select(Run.id).where(
col(Run.flow) == name, col(Run.status).in_(("running", "queued"))
@@ -503,8 +483,10 @@ async def delete_flow(
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
_audit("deleted", name, user)
# Its files are gone; its values and queued work would otherwise linger.
# Its runs stay: a run record is a record of what ran, and surviving the
# flow it belonged to is the point of keeping one. `DELETE /runs?flow=`
# is what clears them.
await run_in_threadpool(controller.forget_flow, name)
await run_in_threadpool(_forget_runs, session, name)
await controller.reload_flow(name)
return Message(message=f"Deleted flow '{name}'")
+78 -20
View File
@@ -28,7 +28,7 @@ from fluksio.flow.events import event_bus
from fluksio.flow.messages import requalify
from fluksio.flow.runs import RunRejected, RunService, new_run_id
from fluksio.flow.store import FlowNotFound
from fluksio.models import Run, RunArtifact, RunMetric, RunNode
from fluksio.models import Message, Run, RunArtifact, RunMetric, RunNode
router = APIRouter(
prefix="/runs", tags=["runs"], dependencies=[Depends(get_current_user)]
@@ -703,27 +703,88 @@ async def retry_run(run_id: str, request: Request, user: CurrentUser) -> Any:
raise HTTPException(status_code=status, detail=detail) from exc
@router.delete("/{run_id}", status_code=204)
def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response:
"""Forget a run and everything hanging off it.
def _drop_runs(session: Session, chosen: Any) -> None:
"""A selection of runs and everything hanging off them.
The same four statements ``_forget_runs`` uses when a flow goes: the run
tables carry a plain string ``run_id`` and no foreign key, so nothing
cascades on its own. ``flow_run``, ``metric_minute`` and ``engine_event``
stay — they are the observability record and are pruned on their own window.
A live run is refused rather than raced: the driver writes its nodes back
when it finishes, and those rows would arrive for a run that no longer
exists. Cancel it first.
The run tables carry a plain string ``run_id`` and no foreign key, so
nothing cascades on its own and these four statements are the whole of it.
``flow_run``, ``metric_minute`` and ``engine_event`` stay — they are the
observability record and are pruned on their own window.
Two things this costs, both deliberate. ``RunNode.outputs`` *is* the stage
cache, so a later run loses hits this one would have served. And a node
restored from this run points here through ``cached_from`` — ``_series``
cache, so a later run loses hits these would have served. And a node
restored from one of them points here through ``cached_from`` — ``_series``
already reads a missing source as an empty curve, which is what ``NO_CURVE``
explains on the screen. The artifact bytes need no help: ``sweep_artifacts``
keeps whatever a ``run_artifact`` row or a live message still names, so
dropping the rows is enough and the hourly sweep reclaims the blobs.
"""
session.execute(delete(RunNode).where(col(RunNode.run_id).in_(chosen)))
session.execute(delete(RunMetric).where(col(RunMetric.run_id).in_(chosen)))
session.execute(delete(RunArtifact).where(col(RunArtifact.run_id).in_(chosen)))
session.execute(delete(Run).where(col(Run.id).in_(chosen)))
def _live_run(session: Session, flow: str) -> str | None:
"""A run of this flow the driver is still writing to, if there is one."""
return session.exec(
select(Run.id).where(
col(Run.flow) == flow, col(Run.status).in_(("running", "queued"))
)
).first()
@router.delete("", response_model=Message)
def delete_runs(
session: SessionDep,
user: CurrentUser,
flow: Annotated[str, Query(min_length=1)],
) -> Any:
"""Every run of one flow at once.
The counterpart to the list's ``flow`` filter, and what a reseed needs:
deleting a flow leaves its runs standing on purpose, so this is the only
way to clear a history without a call per run. ``flow`` is required —
there is no "delete every run of everything" here, by design.
"""
with writing(session):
live = _live_run(session, flow)
if live is not None:
raise HTTPException(
status_code=409,
detail=(
f"Flow '{flow}' has a run in progress ({live}). Cancel it, or "
"wait for it to finish, before deleting its runs."
),
)
# Counted before the delete, and a subquery rather than a materialised
# list of ids on the way in: a demo can hold thousands.
chosen = select(col(Run.id)).where(col(Run.flow) == flow)
gone = session.exec(
select(func.count()).select_from(Run).where(col(Run.flow) == flow)
).one()
_drop_runs(session, chosen)
session.commit()
event_bus.publish(
{
"type": "audit",
"action": f"deleted {gone} run(s)",
"flow": flow,
"user": user.email,
"ts": time.time(),
}
)
return Message(message=f"Deleted {gone} run(s) of '{flow}'")
@router.delete("/{run_id}", status_code=204)
def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response:
"""Forget a run and everything hanging off it.
A live run is refused rather than raced: the driver writes its nodes back
when it finishes, and those rows would arrive for a run that no longer
exists. Cancel it first.
"""
# Reads the run, then decides whether to delete it — so it takes the write
# lock up front rather than upgrading and losing to whichever flush
# committed in between.
@@ -740,10 +801,7 @@ def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response:
),
)
flow = run.flow
session.execute(delete(RunNode).where(col(RunNode.run_id) == run_id))
session.execute(delete(RunMetric).where(col(RunMetric.run_id) == run_id))
session.execute(delete(RunArtifact).where(col(RunArtifact.run_id) == run_id))
session.execute(delete(Run).where(col(Run.id) == run_id))
_drop_runs(session, [run_id])
session.commit()
event_bus.publish(
{
@@ -792,8 +850,8 @@ def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
for node_row in restored:
source_flow = sources.get(node_row.cached_from)
if source_flow is None:
# The run it came from is gone — deleted with its flow. The
# outputs are still on this run; the curve is not recoverable.
# The run it came from was deleted. The outputs are still on
# this run; the curve is not recoverable.
continue
key = (
node_row.cached_from,
+9 -2
View File
@@ -299,10 +299,17 @@ def test_delete_flow(
client.get(f"{PREFIX}/demo", headers=superuser_token_headers).status_code == 404
)
# Deleting the flow takes its runs with it, so a reseeded demo starts clean.
# The run outlives the flow: a run record is a record of what ran, and
# `DELETE /runs?flow=` is the only thing that clears one.
db.expire_all()
for model in (Run, RunNode, RunMetric, RunArtifact):
assert db.exec(select(func.count()).select_from(model)).one() == 0
assert db.exec(select(func.count()).select_from(model)).one() == 1
assert (
client.get(
f"{settings.API_V1_STR}/runs/run-1", headers=superuser_token_headers
).status_code
== 200
)
def test_delete_flow_is_refused_while_a_run_is_live(
+44 -1
View File
@@ -774,6 +774,49 @@ def test_a_running_run_is_refused_rather_than_raced(client, superuser_token_head
session.commit()
def test_deleting_a_flows_runs_clears_that_flow_only(client, superuser_token_headers):
"""The counterpart to the list's `flow` filter — what a reseed needs.
Deleting the flow itself leaves the history standing, so this is the tool
that clears one. It refuses while a run of that flow is still going, for
the same reason deleting a single run does.
"""
made = datetime.now(UTC)
url = f"{settings.API_V1_STR}/runs"
with Session(db_engine) as session:
session.add(Run(id="bulk-1", flow="swept", status="ok", created_at=made))
session.add(RunNode(run_id="bulk-1", node="swept.a", status="ok"))
session.add(RunMetric(run_id="bulk-1", name="swept.loss", step=0, value=1.0))
session.add(Run(id="bulk-2", flow="swept", status="running", created_at=made))
session.add(Run(id="keep-1", flow="other", status="ok", created_at=made))
session.commit()
busy = client.delete(url, params={"flow": "swept"}, headers=superuser_token_headers)
assert busy.status_code == 409
assert "bulk-2" in busy.json()["detail"]
with Session(db_engine) as session:
session.get(Run, "bulk-2").status = "cancelled"
session.commit()
answer = client.delete(
url, params={"flow": "swept"}, headers=superuser_token_headers
)
assert answer.status_code == 200
assert "2" in answer.json()["message"]
with Session(db_engine) as session:
assert session.exec(select(Run).where(col(Run.flow) == "swept")).all() == []
for table in (RunNode, RunMetric):
assert (
session.exec(select(table).where(col(table.run_id) == "bulk-1")).all()
== []
)
assert session.get(Run, "keep-1") is not None
session.delete(session.get(Run, "keep-1"))
session.commit()
# -----------------------------------------------------------------------------
# A cached node's curve
#
@@ -847,7 +890,7 @@ def test_a_cached_node_answers_with_the_curve_it_was_restored_from(
def test_a_curve_whose_run_is_gone_is_empty_rather_than_an_error(
client, superuser_token_headers
):
"""Deleting a flow deletes its runs; what pointed at one is left holding it."""
"""Delete the run a cache hit points at and the reusing run holds nothing."""
with Session(db_engine) as session:
session.delete(session.get(Run, "src-1"))
session.commit()
+3 -1
View File
@@ -46,7 +46,7 @@ Agents authenticate differently; see [Agents over MCP](agents.md).
| `POST` | `/flows/{name}/publish` | put the draft live |
| `POST` | `/flows/{name}/discard-draft` | throw the draft away |
| `POST` | `/flows/{name}/rename` | rename it |
| `DELETE` | `/flows/{name}` | delete it |
| `DELETE` | `/flows/{name}` | delete it. Its runs stay — a run is the record of what ran, and only `DELETE /runs` clears one. Refused with 409 while a run of it is `running` or `queued` |
| `GET` | `/flows/node-types` | every node type and its parameter schema |
| `GET` | `/flows/graph` | every flow as one graph — what Home draws |
@@ -104,12 +104,14 @@ published to. Flows own the namespace; everything else is a client of it.
| `POST` | `/runs/flows/{name}` | queue one run — `{"params": {...}, "seed": 7, "draft": false, "no_cache": false}`. `"cause"` says where it came from — `api` (the default), `cli` or `sdk` |
| `POST` | `/runs/flows/{name}/sweep` | queue many, sharing a `group_id` |
| `GET` | `/runs` | the queryable history: `?flow=`, `?status=`, `?group=`, `?digest=`, `?since=`, `?before=`, `?limit=`, `?offset=` |
| `DELETE` | `/runs?flow=` | every run of one flow, and everything hanging off them. `flow` is required. Refused with 409 while one of them is still going |
| `GET` | `/runs/overview` | one row per flow that has runs, with how many are running or queued |
| `GET` | `/runs/export/metrics?…&name=&stride=&format=` | every selected run's series as one long table: `run, name, step, ts, value` |
| `GET` | `/runs/export/runs?…&params=&metrics=&format=` | one row per run: its inputs as columns, its final numbers, its status and provenance |
| `GET` | `/runs/metrics/names?…` | every metric name the selected runs recorded, distinct; takes the export's own filters |
| `GET` | `/runs/{id}` | one run in full: params, result, per-node record, artifacts |
| `POST` | `/runs/{id}/cancel` | stop it |
| `DELETE` | `/runs/{id}` | forget it, with its nodes, series and artifact rows. Cancel a live one first |
| `POST` | `/runs/{id}/retry` | run the same thing again, as a new run naming this one |
| `GET` | `/runs/{id}/metrics?name=&stride=` | one metric's series, in step order; every series of the run without `name` |
| `GET` | `/runs/series/compare?ids=a,b,c&metric=&x=` | that metric across several runs. `x` is what to plot against: nothing or `step`, `time` (seconds since each run's own first reading), or another metric's name, joined on the step the two share |
+13 -5
View File
@@ -233,8 +233,9 @@ execution that is not happening this time, so its series is not rewritten
either. The run it was restored from is recorded instead, and that is where the
curve is read back from: asking the reusing run for its metrics answers with
the same points, under its own flow's names. The one way to be left with a
result and no curve is for that earlier run to have been deleted, which
deleting its flow does.
result and no curve is for that earlier run to have been deleted. Deleting a
flow does not do that — its runs stay — so this only happens when the run
itself was deleted.
## Objects that cannot be serialized
@@ -346,6 +347,14 @@ the artifacts it made, its metrics and its result. **Retry** submits it again
as a run of its own, keeping the flow, the inputs, the seed and the sweep it
belonged to.
A run outlives the flow it ran. Deleting a flow leaves its history standing —
being the record of what ran is the point of keeping one — so the Runs screen
still lists it, and opening it says **flow deleted** with Retry disabled,
because there is nothing left to run it against. Clearing that history is a
decision of its own: a run at a time on the screen, or
`DELETE /api/v1/runs?flow=<name>` for a whole flow's worth, which is what a
reseed wants.
### Taking it into a dataframe
An analysis wants a table rather than a screen, and there are two it usually
@@ -407,9 +416,8 @@ sweep, or specific runs. It re-reads on its own and whenever a run finishes.
### When a run draws nothing
A node restored from the [stage cache](#stage-caching) has its curve read back
from the run that recorded it. Delete that run, which deleting its flow does,
and the reusing run is left with a result and an empty curve, and the chart says so
rather than looking broken.
from the run that recorded it. Delete that run and the reusing run is left with
a result and an empty curve, and the chart says so rather than looking broken.
## What this costs, compared
+1 -1
View File
@@ -48,7 +48,7 @@ export const OpenAPI: OpenAPIConfig = {
PASSWORD: undefined,
TOKEN: undefined,
USERNAME: undefined,
VERSION: '0.1.4+dev',
VERSION: '0.1.5+dev',
WITH_CREDENTIALS: false,
interceptors: {
request: new Interceptors(),
File diff suppressed because one or more lines are too long
+17
View File
@@ -1830,6 +1830,12 @@ export type RunsReadRunsData = {
export type RunsReadRunsResponse = (Array<fluksio__api__routes__runs__RunRow>);
export type RunsDeleteRunsData = {
flow: string;
};
export type RunsDeleteRunsResponse = (Message);
export type RunsReadOverviewResponse = (Array<FlowRunsRow>);
export type RunsExportMetricsData = {
@@ -1860,6 +1866,17 @@ export type RunsExportRunsData = {
export type RunsExportRunsResponse = (unknown);
export type RunsReadMetricNamesData = {
flow?: (string | null);
group?: (string | null);
ids?: string;
since?: (string | null);
status?: (string | null);
until?: (string | null);
};
export type RunsReadMetricNamesResponse = (Array<(string)>);
export type RunsReadRunData = {
runId: string;
};
+16 -3
View File
@@ -23,6 +23,7 @@ import { OpenInDashboard } from "./OpenInDashboard"
import {
CARD,
downloadArtifact,
FLOW_GONE,
isLive,
NO_CURVE,
paramText,
@@ -42,7 +43,7 @@ export function RunDetail({ id }: { id: string }) {
const cancel = useCancelRun()
const retry = useRetryRun()
const names = useMetricNames(id)
const declared = useFlowInputs(run?.flow)
const { declared, gone } = useFlowInputs(run?.flow)
const [metric, setMetric] = useState("")
if (isPending || !run) return <Skeleton className="h-96 w-full rounded-lg" />
@@ -96,6 +97,14 @@ export function RunDetail({ id }: { id: string }) {
ran the working copy
</span>
)}
{gone && (
<span
className="rounded-full border border-border px-2 py-0.5 text-muted-foreground text-xs"
data-testid="flow-gone"
>
flow deleted
</span>
)}
<div className="ml-auto flex items-center gap-2">
<OpenInDashboard flow={run.flow} ids={[run.id]} />
@@ -115,7 +124,8 @@ export function RunDetail({ id }: { id: string }) {
size="sm"
className="h-8"
onClick={() => retry.mutate(run.id)}
disabled={retry.isPending}
// There is no flow left to run it against; the route would 404.
disabled={retry.isPending || gone}
data-testid="retry-run"
>
<RotateCcw className="size-3.5" />
@@ -126,6 +136,7 @@ export function RunDetail({ id }: { id: string }) {
</header>
{reason && <p className="text-muted-foreground text-sm">{reason}</p>}
{gone && <p className="text-muted-foreground text-sm">{FLOW_GONE}</p>}
<section className={cn(CARD, "grid gap-4 sm:grid-cols-2 lg:grid-cols-4")}>
<Fact label="Submitted">{ago(String(run.created_at ?? ""))}</Fact>
@@ -157,7 +168,9 @@ export function RunDetail({ id }: { id: string }) {
<h2 className="font-medium text-sm">Inputs</h2>
{fed.length === 0 ? (
<p className="text-muted-foreground text-sm">
This flow declares no inputs, so there was nothing to choose.
{gone
? "The run carried no parameters of its own, and its flow is gone, so what it took from the flow cannot be read back."
: "This flow declares no inputs, so there was nothing to choose."}
</p>
) : (
<dl className="grid gap-x-6 gap-y-1 sm:grid-cols-2 lg:grid-cols-3">
+1 -1
View File
@@ -721,7 +721,7 @@ function ParamDiff({ rows }: { rows: RunRow[] }) {
// ponytail: only when the picks share one flow — spanning flows would mean a
// declaration lookup per flow, and a sweep comparison never does.
const one = rows.every((run) => run.flow === rows[0].flow)
const declared = useFlowInputs(one ? rows[0].flow : undefined)
const { declared } = useFlowInputs(one ? rows[0].flow : undefined)
const varying = varyingKeys(rows)
// Not a parameter, but it is part of what produced the number, and in a
// sweep it is often the only thing that moved.
+23 -8
View File
@@ -2,7 +2,7 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"
import { useNavigate } from "@tanstack/react-router"
import { useMemo } from "react"
import { OpenAPI, RunsService } from "@/client"
import { ApiError, OpenAPI, RunsService } from "@/client"
import { flowQueryOptions } from "@/components/Flow/queries"
import { apiToken } from "@/lib/portal"
@@ -176,8 +176,12 @@ export async function downloadArtifact(digest: string, name: string) {
URL.revokeObjectURL(url)
}
/** The server saying a thing is not there, rather than failing to answer. */
const isMissing = (error: unknown) =>
error instanceof ApiError && error.status === 404
/**
* What a flow declares it can be given, by input name.
* What a flow declares it can be given, by input name — and whether it is gone.
*
* A batch flow's inputs *are* its parameters — a run supplies values for the
* ones it names and takes the flow's own for the rest — so this is what turns
@@ -185,15 +189,21 @@ export async function downloadArtifact(digest: string, name: string) {
*
* The declarations are the flow's *current* ones, while a run carries the
* `flow_version` it was submitted against. An input added since is shown on an
* older run as a default it never actually received.
* older run as a default it never actually received. And the flow may not be
* there at all: deleting one leaves its runs standing, so `gone` is an ordinary
* answer here rather than a fault, and the screen says so instead of drawing a
* run that declares nothing.
*/
export function useFlowInputs(flow: string | undefined) {
const { data } = useQuery({
const { data, error } = useQuery({
...flowQueryOptions(flow ?? ""),
enabled: Boolean(flow),
// A flow that is gone stays gone; asking three more times only delays it.
retry: (count: number, failure: unknown) =>
!isMissing(failure) && count < 3,
})
const inputs = data?.definition.inputs
return useMemo(
const declared = useMemo(
() =>
new Map<string, unknown>(
(inputs ?? [])
@@ -202,6 +212,7 @@ export function useFlowInputs(flow: string | undefined) {
),
[inputs],
)
return { declared, gone: isMissing(error) }
}
/**
@@ -267,13 +278,17 @@ export async function cancelThenDelete(runId: string) {
* Why a finished run can have nothing to draw.
*
* A cache hit replays no emissions, so a restored node's curve is read back
* from the run that recorded it. Deleting that run — deleting its flow does —
* takes the curve with it, and then a reused run has a result and nothing to
* draw. Said out loud rather than left as an empty chart, which reads as a fault.
* from the run that recorded it. Deleting that run takes the curve with it, and
* then a reused run has a result and nothing to draw. Said out loud rather than
* left as an empty chart, which reads as a fault.
*/
export const NO_CURVE =
"No curve was recorded. A node restored from the cache is read back from the run that produced it, so this draws nothing once that run has been deleted — its outputs are still on the result."
/** Why a run can name a flow that is not there any more. */
export const FLOW_GONE =
"The flow this ran has been deleted. The run is kept as the record of what ran, but there is nothing left to retry it against."
/** A run id, short enough for a table cell. The tail is the random half. */
export const shortId = (id: string) => id.slice(-8)