A restored node whose source run is gone says its curve was lost
The run detail works out curve_lost per node when read, from whether its cached_from run still exists. The run page names the deleted run in place of the generic NO_CURVE note. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx
This commit is contained in:
@@ -146,6 +146,9 @@ class RunNodeRow(BaseModel):
|
||||
#: Which run it was restored from, when it was. That run is also where this
|
||||
#: node's series was recorded.
|
||||
cached_from: str = ""
|
||||
#: That run has since been deleted, and this node's curve went with it. Its
|
||||
#: outputs are still on this run's result.
|
||||
curve_lost: bool = False
|
||||
|
||||
|
||||
class ArtifactRow(BaseModel):
|
||||
@@ -781,6 +784,16 @@ def read_run(run_id: str, session: SessionDep) -> Any:
|
||||
).all()
|
||||
detail = RunDetail.model_validate(run, from_attributes=True)
|
||||
detail.nodes = [RunNodeRow.model_validate(n, from_attributes=True) for n in nodes]
|
||||
# Worked out here, not stored: a source run is deleted long after the
|
||||
# nodes restored from it were written.
|
||||
sources = {n.cached_from for n in nodes if n.cached_from}
|
||||
kept = (
|
||||
set(session.exec(select(col(Run.id)).where(col(Run.id).in_(sources))))
|
||||
if sources
|
||||
else set()
|
||||
)
|
||||
for row in detail.nodes:
|
||||
row.curve_lost = bool(row.cached_from) and row.cached_from not in kept
|
||||
detail.artifacts = [
|
||||
ArtifactRow.model_validate(a, from_attributes=True) for a in artifacts
|
||||
]
|
||||
@@ -824,8 +837,8 @@ def _drop_runs(session: Session, chosen: Any) -> None:
|
||||
Two things this costs, both deliberate. ``RunNode.outputs`` *is* the stage
|
||||
cache, so a later run loses hits these would have served. And a node
|
||||
restored from one of them points here through ``cached_from`` — ``_series``
|
||||
already reads a missing source as an empty curve, which is what ``NO_CURVE``
|
||||
explains on the screen. The artifact bytes need no help: ``sweep_artifacts``
|
||||
already reads a missing source as an empty curve, and ``read_run`` marks
|
||||
such a node ``curve_lost``. The artifact bytes need no help: ``sweep_artifacts``
|
||||
keeps whatever a ``run_artifact`` row or a live message still names, so
|
||||
dropping the rows is enough and the hourly sweep reclaims the blobs.
|
||||
"""
|
||||
|
||||
@@ -1108,6 +1108,11 @@ def test_a_curve_whose_run_is_gone_is_empty_rather_than_an_error(
|
||||
)
|
||||
assert answer.status_code == 200
|
||||
assert answer.json() == []
|
||||
# And the run says why, rather than leaving an empty chart to explain itself.
|
||||
(node,) = client.get(
|
||||
f"{settings.API_V1_STR}/runs/reuse-1", headers=superuser_token_headers
|
||||
).json()["nodes"]
|
||||
assert node["curve_lost"] is True
|
||||
|
||||
|
||||
def _metric(run_id: str, name: str, step: int, value: float, ts: float) -> RunMetric:
|
||||
|
||||
@@ -143,7 +143,8 @@ missing artifact bytes make an entry a miss.
|
||||
|
||||
A cache hit does not replay emissions. Its metrics are read from the original
|
||||
run, so comparisons still show the recorded curve. Deleting that original run
|
||||
leaves cached results available but removes their curve.
|
||||
leaves cached results available but removes their curve. The run's page then
|
||||
names the deleted run each lost curve went with.
|
||||
|
||||
## Objects that cannot be serialized
|
||||
|
||||
|
||||
@@ -3569,6 +3569,11 @@ export const RunNodeRowSchema = {
|
||||
type: 'string',
|
||||
title: 'Cached From',
|
||||
default: ''
|
||||
},
|
||||
curve_lost: {
|
||||
type: 'boolean',
|
||||
title: 'Curve Lost',
|
||||
default: false
|
||||
}
|
||||
},
|
||||
type: 'object',
|
||||
|
||||
@@ -1206,6 +1206,7 @@ export type RunNodeRow = {
|
||||
logs: string;
|
||||
cache_key?: string;
|
||||
cached_from?: string;
|
||||
curve_lost?: boolean;
|
||||
};
|
||||
|
||||
export type RunPage = {
|
||||
|
||||
@@ -22,10 +22,10 @@ import { MetricPicker, RunMetricChart, useMetricNames } from "./MetricChart"
|
||||
import { OpenInDashboard } from "./OpenInDashboard"
|
||||
import {
|
||||
CARD,
|
||||
curveLost,
|
||||
downloadArtifact,
|
||||
FLOW_GONE,
|
||||
isLive,
|
||||
NO_CURVE,
|
||||
paramText,
|
||||
runQueryOptions,
|
||||
shortCommit,
|
||||
@@ -53,10 +53,9 @@ export function RunDetail({ id }: { id: string }) {
|
||||
const nodes = run.nodes ?? []
|
||||
const artifacts = run.artifacts ?? []
|
||||
const reason = statusReason(run)
|
||||
// A restored node's curve comes from the run it was restored from, so an
|
||||
// empty chart here means that run is gone rather than that this one failed.
|
||||
// Said once, where it applies.
|
||||
const cached = nodes.some((node) => node.status === "cached")
|
||||
// A restored node's curve comes from the run it was restored from, so it
|
||||
// goes when that run is deleted. Said once, where it applies.
|
||||
const lost = nodes.filter((node) => node.curve_lost)
|
||||
|
||||
// What the flow declared when this run was submitted. A run from before
|
||||
// that was kept reads against the flow's current declarations.
|
||||
@@ -206,8 +205,10 @@ export function RunDetail({ id }: { id: string }) {
|
||||
</section>
|
||||
)}
|
||||
|
||||
{names.length === 0 && cached && (
|
||||
<p className={cn(CARD, "text-muted-foreground text-sm")}>{NO_CURVE}</p>
|
||||
{lost.length > 0 && (
|
||||
<p className={cn(CARD, "text-muted-foreground text-sm")}>
|
||||
{curveLost(lost)}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{Object.keys(run.result ?? {}).length > 0 && (
|
||||
|
||||
@@ -285,6 +285,20 @@ export async function cancelThenDelete(runId: string) {
|
||||
export const NO_CURVE =
|
||||
"No curve was recorded. A node restored from the cache is read back from the run that produced it, so this draws nothing once that run has been deleted — its outputs are still on the result."
|
||||
|
||||
/**
|
||||
* Which restored nodes lost their curve, and with which deleted run.
|
||||
*
|
||||
* Named, since a chart that is merely short a few lines says nothing.
|
||||
*/
|
||||
export function curveLost(nodes: { node: string; cached_from?: string }[]) {
|
||||
const runs = [...new Set(nodes.map((one) => shortId(one.cached_from ?? "")))]
|
||||
const which =
|
||||
runs.length === 1
|
||||
? `run ${runs[0]}, which was`
|
||||
: `runs ${runs.join(", ")}, which were`
|
||||
return `The curve of ${nodes.map((one) => one.node).join(", ")} was lost with ${which} deleted. A node restored from the cache reads its curve from the run that recorded it. Its outputs are still on the result.`
|
||||
}
|
||||
|
||||
/** Why a run can name a flow that is not there any more. */
|
||||
export const FLOW_GONE =
|
||||
"The flow this ran has been deleted. The run is kept as the record of what ran, but there is nothing left to retry it against."
|
||||
|
||||
Reference in New Issue
Block a user