diff --git a/backend/fluksio/api/routes/runs.py b/backend/fluksio/api/routes/runs.py index b7c52e0..123355e 100644 --- a/backend/fluksio/api/routes/runs.py +++ b/backend/fluksio/api/routes/runs.py @@ -146,6 +146,9 @@ class RunNodeRow(BaseModel): #: Which run it was restored from, when it was. That run is also where this #: node's series was recorded. cached_from: str = "" + #: That run has since been deleted, and this node's curve went with it. Its + #: outputs are still on this run's result. + curve_lost: bool = False class ArtifactRow(BaseModel): @@ -781,6 +784,16 @@ def read_run(run_id: str, session: SessionDep) -> Any: ).all() detail = RunDetail.model_validate(run, from_attributes=True) detail.nodes = [RunNodeRow.model_validate(n, from_attributes=True) for n in nodes] + # Worked out here, not stored: a source run is deleted long after the + # nodes restored from it were written. + sources = {n.cached_from for n in nodes if n.cached_from} + kept = ( + set(session.exec(select(col(Run.id)).where(col(Run.id).in_(sources)))) + if sources + else set() + ) + for row in detail.nodes: + row.curve_lost = bool(row.cached_from) and row.cached_from not in kept detail.artifacts = [ ArtifactRow.model_validate(a, from_attributes=True) for a in artifacts ] @@ -824,8 +837,8 @@ def _drop_runs(session: Session, chosen: Any) -> None: Two things this costs, both deliberate. ``RunNode.outputs`` *is* the stage cache, so a later run loses hits these would have served. And a node restored from one of them points here through ``cached_from`` — ``_series`` - already reads a missing source as an empty curve, which is what ``NO_CURVE`` - explains on the screen. The artifact bytes need no help: ``sweep_artifacts`` + already reads a missing source as an empty curve, and ``read_run`` marks + such a node ``curve_lost``. The artifact bytes need no help: ``sweep_artifacts`` keeps whatever a ``run_artifact`` row or a live message still names, so dropping the rows is enough and the hourly sweep reclaims the blobs. """ diff --git a/backend/tests/api/routes/test_runs.py b/backend/tests/api/routes/test_runs.py index c543591..c982b9c 100644 --- a/backend/tests/api/routes/test_runs.py +++ b/backend/tests/api/routes/test_runs.py @@ -1108,6 +1108,11 @@ def test_a_curve_whose_run_is_gone_is_empty_rather_than_an_error( ) assert answer.status_code == 200 assert answer.json() == [] + # And the run says why, rather than leaving an empty chart to explain itself. + (node,) = client.get( + f"{settings.API_V1_STR}/runs/reuse-1", headers=superuser_token_headers + ).json()["nodes"] + assert node["curve_lost"] is True def _metric(run_id: str, name: str, step: int, value: float, ts: float) -> RunMetric: diff --git a/docs/concepts/runs.md b/docs/concepts/runs.md index 7624362..75125d2 100644 --- a/docs/concepts/runs.md +++ b/docs/concepts/runs.md @@ -143,7 +143,8 @@ missing artifact bytes make an entry a miss. A cache hit does not replay emissions. Its metrics are read from the original run, so comparisons still show the recorded curve. Deleting that original run -leaves cached results available but removes their curve. +leaves cached results available but removes their curve. The run's page then +names the deleted run each lost curve went with. ## Objects that cannot be serialized diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 2851e61..1c3611c 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -3569,6 +3569,11 @@ export const RunNodeRowSchema = { type: 'string', title: 'Cached From', default: '' + }, + curve_lost: { + type: 'boolean', + title: 'Curve Lost', + default: false } }, type: 'object', diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 270b8d5..b197d55 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -1206,6 +1206,7 @@ export type RunNodeRow = { logs: string; cache_key?: string; cached_from?: string; + curve_lost?: boolean; }; export type RunPage = { diff --git a/frontend/src/components/Runs/RunDetail.tsx b/frontend/src/components/Runs/RunDetail.tsx index ac9f061..6a8b5c4 100644 --- a/frontend/src/components/Runs/RunDetail.tsx +++ b/frontend/src/components/Runs/RunDetail.tsx @@ -22,10 +22,10 @@ import { MetricPicker, RunMetricChart, useMetricNames } from "./MetricChart" import { OpenInDashboard } from "./OpenInDashboard" import { CARD, + curveLost, downloadArtifact, FLOW_GONE, isLive, - NO_CURVE, paramText, runQueryOptions, shortCommit, @@ -53,10 +53,9 @@ export function RunDetail({ id }: { id: string }) { const nodes = run.nodes ?? [] const artifacts = run.artifacts ?? [] const reason = statusReason(run) - // A restored node's curve comes from the run it was restored from, so an - // empty chart here means that run is gone rather than that this one failed. - // Said once, where it applies. - const cached = nodes.some((node) => node.status === "cached") + // A restored node's curve comes from the run it was restored from, so it + // goes when that run is deleted. Said once, where it applies. + const lost = nodes.filter((node) => node.curve_lost) // What the flow declared when this run was submitted. A run from before // that was kept reads against the flow's current declarations. @@ -206,8 +205,10 @@ export function RunDetail({ id }: { id: string }) { )} - {names.length === 0 && cached && ( -
{NO_CURVE}
+ {lost.length > 0 && ( ++ {curveLost(lost)} +
)} {Object.keys(run.result ?? {}).length > 0 && ( diff --git a/frontend/src/components/Runs/queries.ts b/frontend/src/components/Runs/queries.ts index 4592682..eb186ec 100644 --- a/frontend/src/components/Runs/queries.ts +++ b/frontend/src/components/Runs/queries.ts @@ -285,6 +285,20 @@ export async function cancelThenDelete(runId: string) { export const NO_CURVE = "No curve was recorded. A node restored from the cache is read back from the run that produced it, so this draws nothing once that run has been deleted — its outputs are still on the result." +/** + * Which restored nodes lost their curve, and with which deleted run. + * + * Named, since a chart that is merely short a few lines says nothing. + */ +export function curveLost(nodes: { node: string; cached_from?: string }[]) { + const runs = [...new Set(nodes.map((one) => shortId(one.cached_from ?? "")))] + const which = + runs.length === 1 + ? `run ${runs[0]}, which was` + : `runs ${runs.join(", ")}, which were` + return `The curve of ${nodes.map((one) => one.node).join(", ")} was lost with ${which} deleted. A node restored from the cache reads its curve from the run that recorded it. Its outputs are still on the result.` +} + /** Why a run can name a flow that is not there any more. */ export const FLOW_GONE = "The flow this ran has been deleted. The run is kept as the record of what ran, but there is nothing left to retry it against."