A cached node keeps its curve, and any input can name a run's output
Docs / docs (push) Successful in 22s
Playwright Tests / test-playwright (1, 2) (push) Failing after 2m42s
Playwright Tests / test-playwright (2, 2) (push) Failing after 1m41s
pre-commit / pre-commit (push) Failing after 2m53s
Test Backend / test-backend (push) Successful in 2m21s
Compose Smoke Test / test-compose (push) Successful in 31s
Playwright Tests / merge-reports (push) Failing after 1m6s
Docs / docs (push) Successful in 22s
Playwright Tests / test-playwright (1, 2) (push) Failing after 2m42s
Playwright Tests / test-playwright (2, 2) (push) Failing after 1m41s
pre-commit / pre-commit (push) Failing after 2m53s
Test Backend / test-backend (push) Successful in 2m21s
Compose Smoke Test / test-compose (push) Successful in 31s
Playwright Tests / merge-reports (push) Failing after 1m6s
A cache hit still replays no emissions — those values were the story of an
execution that is not happening — but the run they were recorded in is now
written on the row (`run_node.cached_from`), and the metrics endpoints read the
series back from there. So a reused run answers `run.metrics("train.loss")`
with the same points the run that trained did, rather than looking like a run
that produced no numbers at all. Pointed at rather than copied: a sweep of 500
reusing one frozen node would otherwise duplicate its curve 500 times.
That needed the cross-flow restore fixed first. The cache key has no flow in
it while the stored outputs are named for the flow that produced them, so
`quick.prepare` getting a hit from `train` wrote `train.dataset` into `quick`'s
state and the next node was called without its argument. One rule now covers
both halves: `requalify` reads a name owned by one flow as the same name in
another, applied to the restored outputs, to the node id behind the pointer,
and to the series names on the way out. Reuse across flows is kept.
Also: `@run:<id>.<output>` and a bare `sha256:` digest resolve on every input,
not only artifacts. Chaining a run's json config into the next one from a shell
meant pasting the whole object inline, and the CLI could not even send the
spelling — `_coerce` died in `json.loads` before the engine saw it. Both
spellings are reserved on every input now, `str` included, and `_from_run`
returns whatever the run's result holds rather than only a reference.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Dp9L6gakMVro1K2C5zdtBE
This commit is contained in:
@@ -12,9 +12,10 @@ from fastapi.concurrency import run_in_threadpool
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy import select as sa_select
|
||||
from sqlmodel import col, select
|
||||
from sqlmodel import Session, col, select
|
||||
|
||||
from fluksio.api.deps import CurrentUser, SessionDep, get_current_user
|
||||
from fluksio.flow.messages import requalify
|
||||
from fluksio.flow.runs import RunRejected, RunService, new_run_id
|
||||
from fluksio.flow.store import FlowNotFound
|
||||
from fluksio.models import Run, RunArtifact, RunMetric, RunNode
|
||||
@@ -58,6 +59,9 @@ class RunNodeRow(BaseModel):
|
||||
#: What this node's result was looked up by. Empty when it may not be
|
||||
#: reused; `status` is "cached" when it was.
|
||||
cache_key: str = ""
|
||||
#: Which run it was restored from, when it was. That run is also where this
|
||||
#: node's series was recorded.
|
||||
cached_from: str = ""
|
||||
|
||||
|
||||
class ArtifactRow(BaseModel):
|
||||
@@ -284,6 +288,59 @@ async def cancel_run(run_id: str, request: Request, session: SessionDep) -> Any:
|
||||
return run
|
||||
|
||||
|
||||
def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
|
||||
"""A run's numbers, including the ones a cached node points at.
|
||||
|
||||
A cache hit replays no emissions, so a node restored from an earlier run has
|
||||
no rows of its own — it carries that run's id instead, and its series is read
|
||||
from there. Names are re-qualified on the way out, because the same node
|
||||
reached through two flows publishes under two names and the caller asked for
|
||||
this run's.
|
||||
"""
|
||||
statement = select(RunMetric).where(col(RunMetric.run_id) == run_id)
|
||||
if name:
|
||||
statement = statement.where(col(RunMetric.name) == name)
|
||||
rows = list(session.exec(statement))
|
||||
|
||||
restored = session.exec(
|
||||
select(RunNode).where(
|
||||
col(RunNode.run_id) == run_id, col(RunNode.cached_from) != ""
|
||||
)
|
||||
).all()
|
||||
if restored:
|
||||
run = session.get(Run, run_id)
|
||||
flow = run.flow if run is not None else ""
|
||||
for node_row in restored:
|
||||
source = session.get(Run, node_row.cached_from)
|
||||
if source is None:
|
||||
# The run it came from is gone — deleted with its flow. The
|
||||
# outputs are still on this run; the curve is not recoverable.
|
||||
continue
|
||||
source_node = requalify(node_row.node, flow, source.flow)
|
||||
for row in session.exec(
|
||||
select(RunMetric).where(
|
||||
col(RunMetric.run_id) == node_row.cached_from,
|
||||
col(RunMetric.node) == source_node,
|
||||
)
|
||||
):
|
||||
renamed = requalify(row.name, source.flow, flow)
|
||||
if name and renamed != name:
|
||||
continue
|
||||
rows.append(
|
||||
RunMetric(
|
||||
run_id=run_id,
|
||||
name=renamed,
|
||||
step=row.step,
|
||||
node=node_row.node,
|
||||
ts=row.ts,
|
||||
value=row.value,
|
||||
)
|
||||
)
|
||||
|
||||
rows.sort(key=lambda row: (row.name, row.step))
|
||||
return rows
|
||||
|
||||
|
||||
@router.get("/{run_id}/metrics", response_model=list[MetricPoint])
|
||||
def read_metrics(
|
||||
run_id: str, session: SessionDep, name: str = "", stride: int = 1
|
||||
@@ -293,11 +350,7 @@ def read_metrics(
|
||||
``stride`` thins a long curve down: 3000 steps drawn on a 400-pixel chart
|
||||
is 3000 points nobody can see.
|
||||
"""
|
||||
statement = select(RunMetric).where(col(RunMetric.run_id) == run_id)
|
||||
if name:
|
||||
statement = statement.where(col(RunMetric.name) == name)
|
||||
statement = statement.order_by(col(RunMetric.name), col(RunMetric.step))
|
||||
rows = list(session.exec(statement))
|
||||
rows = _series(session, run_id, name)
|
||||
if stride > 1:
|
||||
rows = rows[:: max(1, stride)]
|
||||
return rows
|
||||
@@ -323,11 +376,7 @@ def compare_metric(session: SessionDep, ids: str, metric: str) -> Any:
|
||||
run = runs.get(run_id)
|
||||
if run is None:
|
||||
continue
|
||||
rows = session.exec(
|
||||
select(RunMetric)
|
||||
.where(col(RunMetric.run_id) == run_id, col(RunMetric.name) == metric)
|
||||
.order_by(col(RunMetric.step))
|
||||
).all()
|
||||
rows = _series(session, run_id, metric)
|
||||
label = run_id
|
||||
if run.seed is not None:
|
||||
label = f"{run_id} (seed {run.seed})"
|
||||
|
||||
Reference in New Issue
Block a user