A restored node whose source run is gone says its curve was lost
The run detail works out curve_lost per node when read, from whether its cached_from run still exists. The run page names the deleted run in place of the generic NO_CURVE note. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx
This commit is contained in:
@@ -146,6 +146,9 @@ class RunNodeRow(BaseModel):
|
|||||||
#: Which run it was restored from, when it was. That run is also where this
|
#: Which run it was restored from, when it was. That run is also where this
|
||||||
#: node's series was recorded.
|
#: node's series was recorded.
|
||||||
cached_from: str = ""
|
cached_from: str = ""
|
||||||
|
#: That run has since been deleted, and this node's curve went with it. Its
|
||||||
|
#: outputs are still on this run's result.
|
||||||
|
curve_lost: bool = False
|
||||||
|
|
||||||
|
|
||||||
class ArtifactRow(BaseModel):
|
class ArtifactRow(BaseModel):
|
||||||
@@ -781,6 +784,16 @@ def read_run(run_id: str, session: SessionDep) -> Any:
|
|||||||
).all()
|
).all()
|
||||||
detail = RunDetail.model_validate(run, from_attributes=True)
|
detail = RunDetail.model_validate(run, from_attributes=True)
|
||||||
detail.nodes = [RunNodeRow.model_validate(n, from_attributes=True) for n in nodes]
|
detail.nodes = [RunNodeRow.model_validate(n, from_attributes=True) for n in nodes]
|
||||||
|
# Worked out here, not stored: a source run is deleted long after the
|
||||||
|
# nodes restored from it were written.
|
||||||
|
sources = {n.cached_from for n in nodes if n.cached_from}
|
||||||
|
kept = (
|
||||||
|
set(session.exec(select(col(Run.id)).where(col(Run.id).in_(sources))))
|
||||||
|
if sources
|
||||||
|
else set()
|
||||||
|
)
|
||||||
|
for row in detail.nodes:
|
||||||
|
row.curve_lost = bool(row.cached_from) and row.cached_from not in kept
|
||||||
detail.artifacts = [
|
detail.artifacts = [
|
||||||
ArtifactRow.model_validate(a, from_attributes=True) for a in artifacts
|
ArtifactRow.model_validate(a, from_attributes=True) for a in artifacts
|
||||||
]
|
]
|
||||||
@@ -824,8 +837,8 @@ def _drop_runs(session: Session, chosen: Any) -> None:
|
|||||||
Two things this costs, both deliberate. ``RunNode.outputs`` *is* the stage
|
Two things this costs, both deliberate. ``RunNode.outputs`` *is* the stage
|
||||||
cache, so a later run loses hits these would have served. And a node
|
cache, so a later run loses hits these would have served. And a node
|
||||||
restored from one of them points here through ``cached_from`` — ``_series``
|
restored from one of them points here through ``cached_from`` — ``_series``
|
||||||
already reads a missing source as an empty curve, which is what ``NO_CURVE``
|
already reads a missing source as an empty curve, and ``read_run`` marks
|
||||||
explains on the screen. The artifact bytes need no help: ``sweep_artifacts``
|
such a node ``curve_lost``. The artifact bytes need no help: ``sweep_artifacts``
|
||||||
keeps whatever a ``run_artifact`` row or a live message still names, so
|
keeps whatever a ``run_artifact`` row or a live message still names, so
|
||||||
dropping the rows is enough and the hourly sweep reclaims the blobs.
|
dropping the rows is enough and the hourly sweep reclaims the blobs.
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -1108,6 +1108,11 @@ def test_a_curve_whose_run_is_gone_is_empty_rather_than_an_error(
|
|||||||
)
|
)
|
||||||
assert answer.status_code == 200
|
assert answer.status_code == 200
|
||||||
assert answer.json() == []
|
assert answer.json() == []
|
||||||
|
# And the run says why, rather than leaving an empty chart to explain itself.
|
||||||
|
(node,) = client.get(
|
||||||
|
f"{settings.API_V1_STR}/runs/reuse-1", headers=superuser_token_headers
|
||||||
|
).json()["nodes"]
|
||||||
|
assert node["curve_lost"] is True
|
||||||
|
|
||||||
|
|
||||||
def _metric(run_id: str, name: str, step: int, value: float, ts: float) -> RunMetric:
|
def _metric(run_id: str, name: str, step: int, value: float, ts: float) -> RunMetric:
|
||||||
|
|||||||
@@ -143,7 +143,8 @@ missing artifact bytes make an entry a miss.
|
|||||||
|
|
||||||
A cache hit does not replay emissions. Its metrics are read from the original
|
A cache hit does not replay emissions. Its metrics are read from the original
|
||||||
run, so comparisons still show the recorded curve. Deleting that original run
|
run, so comparisons still show the recorded curve. Deleting that original run
|
||||||
leaves cached results available but removes their curve.
|
leaves cached results available but removes their curve. The run's page then
|
||||||
|
names the deleted run each lost curve went with.
|
||||||
|
|
||||||
## Objects that cannot be serialized
|
## Objects that cannot be serialized
|
||||||
|
|
||||||
|
|||||||
@@ -3569,6 +3569,11 @@ export const RunNodeRowSchema = {
|
|||||||
type: 'string',
|
type: 'string',
|
||||||
title: 'Cached From',
|
title: 'Cached From',
|
||||||
default: ''
|
default: ''
|
||||||
|
},
|
||||||
|
curve_lost: {
|
||||||
|
type: 'boolean',
|
||||||
|
title: 'Curve Lost',
|
||||||
|
default: false
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
type: 'object',
|
type: 'object',
|
||||||
|
|||||||
@@ -1206,6 +1206,7 @@ export type RunNodeRow = {
|
|||||||
logs: string;
|
logs: string;
|
||||||
cache_key?: string;
|
cache_key?: string;
|
||||||
cached_from?: string;
|
cached_from?: string;
|
||||||
|
curve_lost?: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type RunPage = {
|
export type RunPage = {
|
||||||
|
|||||||
@@ -22,10 +22,10 @@ import { MetricPicker, RunMetricChart, useMetricNames } from "./MetricChart"
|
|||||||
import { OpenInDashboard } from "./OpenInDashboard"
|
import { OpenInDashboard } from "./OpenInDashboard"
|
||||||
import {
|
import {
|
||||||
CARD,
|
CARD,
|
||||||
|
curveLost,
|
||||||
downloadArtifact,
|
downloadArtifact,
|
||||||
FLOW_GONE,
|
FLOW_GONE,
|
||||||
isLive,
|
isLive,
|
||||||
NO_CURVE,
|
|
||||||
paramText,
|
paramText,
|
||||||
runQueryOptions,
|
runQueryOptions,
|
||||||
shortCommit,
|
shortCommit,
|
||||||
@@ -53,10 +53,9 @@ export function RunDetail({ id }: { id: string }) {
|
|||||||
const nodes = run.nodes ?? []
|
const nodes = run.nodes ?? []
|
||||||
const artifacts = run.artifacts ?? []
|
const artifacts = run.artifacts ?? []
|
||||||
const reason = statusReason(run)
|
const reason = statusReason(run)
|
||||||
// A restored node's curve comes from the run it was restored from, so an
|
// A restored node's curve comes from the run it was restored from, so it
|
||||||
// empty chart here means that run is gone rather than that this one failed.
|
// goes when that run is deleted. Said once, where it applies.
|
||||||
// Said once, where it applies.
|
const lost = nodes.filter((node) => node.curve_lost)
|
||||||
const cached = nodes.some((node) => node.status === "cached")
|
|
||||||
|
|
||||||
// What the flow declared when this run was submitted. A run from before
|
// What the flow declared when this run was submitted. A run from before
|
||||||
// that was kept reads against the flow's current declarations.
|
// that was kept reads against the flow's current declarations.
|
||||||
@@ -206,8 +205,10 @@ export function RunDetail({ id }: { id: string }) {
|
|||||||
</section>
|
</section>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{names.length === 0 && cached && (
|
{lost.length > 0 && (
|
||||||
<p className={cn(CARD, "text-muted-foreground text-sm")}>{NO_CURVE}</p>
|
<p className={cn(CARD, "text-muted-foreground text-sm")}>
|
||||||
|
{curveLost(lost)}
|
||||||
|
</p>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{Object.keys(run.result ?? {}).length > 0 && (
|
{Object.keys(run.result ?? {}).length > 0 && (
|
||||||
|
|||||||
@@ -285,6 +285,20 @@ export async function cancelThenDelete(runId: string) {
|
|||||||
export const NO_CURVE =
|
export const NO_CURVE =
|
||||||
"No curve was recorded. A node restored from the cache is read back from the run that produced it, so this draws nothing once that run has been deleted — its outputs are still on the result."
|
"No curve was recorded. A node restored from the cache is read back from the run that produced it, so this draws nothing once that run has been deleted — its outputs are still on the result."
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Which restored nodes lost their curve, and with which deleted run.
|
||||||
|
*
|
||||||
|
* Named, since a chart that is merely short a few lines says nothing.
|
||||||
|
*/
|
||||||
|
export function curveLost(nodes: { node: string; cached_from?: string }[]) {
|
||||||
|
const runs = [...new Set(nodes.map((one) => shortId(one.cached_from ?? "")))]
|
||||||
|
const which =
|
||||||
|
runs.length === 1
|
||||||
|
? `run ${runs[0]}, which was`
|
||||||
|
: `runs ${runs.join(", ")}, which were`
|
||||||
|
return `The curve of ${nodes.map((one) => one.node).join(", ")} was lost with ${which} deleted. A node restored from the cache reads its curve from the run that recorded it. Its outputs are still on the result.`
|
||||||
|
}
|
||||||
|
|
||||||
/** Why a run can name a flow that is not there any more. */
|
/** Why a run can name a flow that is not there any more. */
|
||||||
export const FLOW_GONE =
|
export const FLOW_GONE =
|
||||||
"The flow this ran has been deleted. The run is kept as the record of what ran, but there is nothing left to retry it against."
|
"The flow this ran has been deleted. The run is kept as the record of what ran, but there is nothing left to retry it against."
|
||||||
|
|||||||
Reference in New Issue
Block a user