stroblme
4350916bd8
Docs / docs (push) Successful in 35s
Playwright Tests / test-playwright (1, 2) (push) Failing after 3m14s
Playwright Tests / test-playwright (2, 2) (push) Failing after 1m44s
pre-commit / pre-commit (push) Failing after 3m54s
Test Backend / test-backend (push) Successful in 3m12s
Compose Smoke Test / test-compose (push) Successful in 32s
Playwright Tests / merge-reports (push) Failing after 1m28s
414 lines
14 KiB
Python
414 lines
14 KiB
Python
"""Runs over the API: submit one, watch it, read what it made.
|
|
|
|
Submitting returns immediately with a queued run — a training run is measured
|
|
in hours, so nothing here waits for one. The way to follow a run is to poll it
|
|
or to listen on the flow socket, which carries its start and finish.
|
|
"""
|
|
|
|
from typing import Any
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
|
from fastapi.concurrency import run_in_threadpool
|
|
from pydantic import BaseModel, Field
|
|
from sqlalchemy import func
|
|
from sqlalchemy import select as sa_select
|
|
from sqlmodel import Session, col, select
|
|
|
|
from fluksio.api.deps import CurrentUser, SessionDep, get_current_user
|
|
from fluksio.flow.messages import requalify
|
|
from fluksio.flow.runs import RunRejected, RunService, new_run_id
|
|
from fluksio.flow.store import FlowNotFound
|
|
from fluksio.models import Run, RunArtifact, RunMetric, RunNode
|
|
|
|
router = APIRouter(
|
|
prefix="/runs", tags=["runs"], dependencies=[Depends(get_current_user)]
|
|
)
|
|
|
|
#: A sweep bigger than this is almost always a mistake in a loop.
|
|
MAX_SWEEP = 1000
|
|
|
|
|
|
class RunCreate(BaseModel):
|
|
params: dict[str, Any] = Field(default_factory=dict)
|
|
seed: int | None = None
|
|
#: Run the unpublished draft instead of what is published.
|
|
draft: bool = False
|
|
#: Execute every node, whatever an earlier run already worked out.
|
|
no_cache: bool = False
|
|
|
|
|
|
class SweepEntry(BaseModel):
|
|
params: dict[str, Any] = Field(default_factory=dict)
|
|
seed: int | None = None
|
|
|
|
|
|
class SweepCreate(BaseModel):
|
|
runs: list[SweepEntry] = Field(default_factory=list)
|
|
draft: bool = False
|
|
no_cache: bool = False
|
|
|
|
|
|
class RunNodeRow(BaseModel):
|
|
node: str
|
|
status: str
|
|
attempt: int
|
|
duration_ms: float
|
|
worker: str
|
|
error: str
|
|
logs: str
|
|
#: What this node's result was looked up by. Empty when it may not be
|
|
#: reused; `status` is "cached" when it was.
|
|
cache_key: str = ""
|
|
#: Which run it was restored from, when it was. That run is also where this
|
|
#: node's series was recorded.
|
|
cached_from: str = ""
|
|
|
|
|
|
class ArtifactRow(BaseModel):
|
|
name: str
|
|
node: str
|
|
digest: str
|
|
size: int
|
|
media_type: str
|
|
|
|
|
|
class RunRow(BaseModel):
|
|
"""A run without its result, which is the part that can be large."""
|
|
|
|
id: str
|
|
flow: str
|
|
status: str
|
|
status_reason: str
|
|
cause: str
|
|
params: dict[str, Any]
|
|
params_digest: str
|
|
#: The user repository's commit, for a flow declared in code with the
|
|
#: decorators. Empty for one drawn on the canvas, where `commit` is the
|
|
#: whole answer to what produced the number.
|
|
origin_commit: str = ""
|
|
#: The flow store's own commit. Short, unlike `result`, so the list
|
|
#: carries it: "what code produced this" is a question asked of a table.
|
|
commit: str = ""
|
|
seed: int | None
|
|
group_id: str | None
|
|
labels: list[str]
|
|
created_at: Any
|
|
started_at: Any = None
|
|
finished_at: Any = None
|
|
duration_ms: float
|
|
actor: str
|
|
|
|
|
|
class RunDetail(RunRow):
|
|
result: dict[str, Any] = Field(default_factory=dict)
|
|
flow_version: int = 1
|
|
nodes: list[RunNodeRow] = Field(default_factory=list)
|
|
artifacts: list[ArtifactRow] = Field(default_factory=list)
|
|
|
|
|
|
class FlowRunsRow(BaseModel):
|
|
"""How much a flow has been run, for the screen's list of flows."""
|
|
|
|
flow: str
|
|
runs: int
|
|
running: int
|
|
queued: int
|
|
last_created_at: Any = None
|
|
|
|
|
|
class MetricPoint(BaseModel):
|
|
step: int
|
|
ts: float
|
|
value: float
|
|
name: str = ""
|
|
|
|
|
|
class MetricSeries(BaseModel):
|
|
"""The shape a chart widget already draws, so comparing runs is a binding."""
|
|
|
|
label: str
|
|
points: list[list[float]] = Field(default_factory=list)
|
|
|
|
|
|
class SeriesAnswer(BaseModel):
|
|
metric: str
|
|
#: What the x values are: "step", "time" (seconds since this run's first
|
|
#: reading), or the name of another metric this one was plotted against.
|
|
x: str = "step"
|
|
lines: list[MetricSeries] = Field(default_factory=list)
|
|
|
|
|
|
def _service(request: Request) -> RunService:
|
|
service: RunService | None = getattr(request.app.state, "run_service", None)
|
|
if service is None:
|
|
raise HTTPException(status_code=503, detail="Runs are not available")
|
|
return service
|
|
|
|
|
|
@router.post("/flows/{name}", response_model=RunRow, status_code=202)
|
|
async def create_run(
|
|
name: str, body: RunCreate, request: Request, user: CurrentUser
|
|
) -> Any:
|
|
"""Queue one run of a flow."""
|
|
service = _service(request)
|
|
try:
|
|
return await run_in_threadpool(
|
|
service.submit,
|
|
name,
|
|
params=body.params,
|
|
seed=body.seed,
|
|
cause="api",
|
|
actor=user.email,
|
|
draft=body.draft,
|
|
no_cache=body.no_cache,
|
|
)
|
|
except FlowNotFound as exc:
|
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
|
except RunRejected as exc:
|
|
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
|
|
|
|
|
@router.post("/flows/{name}/sweep", response_model=list[RunRow], status_code=202)
|
|
async def create_sweep(
|
|
name: str, body: SweepCreate, request: Request, user: CurrentUser
|
|
) -> Any:
|
|
"""Queue many runs of one flow under a shared group.
|
|
|
|
An ensemble is this with the same parameters and different seeds; a grid
|
|
search is this with the parameters spread out. Either way the caller
|
|
builds the list — the engine does not own a sweep grammar.
|
|
"""
|
|
if not body.runs:
|
|
raise HTTPException(status_code=422, detail="A sweep needs at least one run")
|
|
if len(body.runs) > MAX_SWEEP:
|
|
raise HTTPException(
|
|
status_code=422, detail=f"A sweep is capped at {MAX_SWEEP} runs"
|
|
)
|
|
service = _service(request)
|
|
group = new_run_id()
|
|
|
|
def submit_all() -> list[Run]:
|
|
return [
|
|
service.submit(
|
|
name,
|
|
params=entry.params,
|
|
seed=entry.seed,
|
|
group_id=group,
|
|
cause="sweep",
|
|
actor=user.email,
|
|
draft=body.draft,
|
|
no_cache=body.no_cache,
|
|
)
|
|
for entry in body.runs
|
|
]
|
|
|
|
try:
|
|
return await run_in_threadpool(submit_all)
|
|
except FlowNotFound as exc:
|
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
|
except RunRejected as exc:
|
|
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
|
|
|
|
|
@router.get("", response_model=list[RunRow])
|
|
def read_runs(
|
|
session: SessionDep,
|
|
flow: str | None = None,
|
|
status: str | None = None,
|
|
group: str | None = None,
|
|
digest: str | None = None,
|
|
limit: int = 50,
|
|
offset: int = 0,
|
|
) -> Any:
|
|
"""Runs, newest first. The queryable table an experiment log needs."""
|
|
statement = select(Run).order_by(col(Run.created_at).desc())
|
|
if flow:
|
|
statement = statement.where(col(Run.flow) == flow)
|
|
if status:
|
|
statement = statement.where(col(Run.status) == status)
|
|
if group:
|
|
statement = statement.where(col(Run.group_id) == group)
|
|
if digest:
|
|
statement = statement.where(col(Run.params_digest) == digest)
|
|
statement = statement.offset(max(0, offset)).limit(min(limit, 500))
|
|
return list(session.exec(statement))
|
|
|
|
|
|
@router.get("/overview", response_model=list[FlowRunsRow])
|
|
def read_overview(session: SessionDep) -> Any:
|
|
"""One row per flow that has ever run, busiest-recent first.
|
|
|
|
The list caps at 500 newest runs, so counting flows on the client goes
|
|
wrong the moment a history outgrows one page. The database counts instead.
|
|
"""
|
|
statement = sa_select(
|
|
col(Run.flow),
|
|
col(Run.status),
|
|
func.count(col(Run.id)),
|
|
func.max(col(Run.created_at)),
|
|
).group_by(col(Run.flow), col(Run.status))
|
|
|
|
rows: dict[str, FlowRunsRow] = {}
|
|
for flow, status, count, latest in session.execute(statement):
|
|
row = rows.setdefault(flow, FlowRunsRow(flow=flow, runs=0, running=0, queued=0))
|
|
row.runs += count
|
|
if status == "running":
|
|
row.running += count
|
|
elif status == "queued":
|
|
row.queued += count
|
|
if row.last_created_at is None or latest > row.last_created_at:
|
|
row.last_created_at = latest
|
|
return sorted(rows.values(), key=lambda row: row.last_created_at, reverse=True)
|
|
|
|
|
|
@router.get("/{run_id}", response_model=RunDetail)
|
|
def read_run(run_id: str, session: SessionDep) -> Any:
|
|
"""One run in full: what it was asked, what each node did, what it made."""
|
|
run = session.get(Run, run_id)
|
|
if run is None:
|
|
raise HTTPException(status_code=404, detail="No such run")
|
|
nodes = session.exec(select(RunNode).where(col(RunNode.run_id) == run_id)).all()
|
|
artifacts = session.exec(
|
|
select(RunArtifact).where(col(RunArtifact.run_id) == run_id)
|
|
).all()
|
|
detail = RunDetail.model_validate(run, from_attributes=True)
|
|
detail.nodes = [RunNodeRow.model_validate(n, from_attributes=True) for n in nodes]
|
|
detail.artifacts = [
|
|
ArtifactRow.model_validate(a, from_attributes=True) for a in artifacts
|
|
]
|
|
return detail
|
|
|
|
|
|
@router.post("/{run_id}/cancel", response_model=RunRow)
|
|
async def cancel_run(run_id: str, request: Request, session: SessionDep) -> Any:
|
|
"""Stop a run. One already past its last node is left as it finished."""
|
|
run = session.get(Run, run_id)
|
|
if run is None:
|
|
raise HTTPException(status_code=404, detail="No such run")
|
|
service = _service(request)
|
|
await run_in_threadpool(service.cancel, run_id)
|
|
session.refresh(run)
|
|
return run
|
|
|
|
|
|
def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
|
|
"""A run's numbers, including the ones a cached node points at.
|
|
|
|
A cache hit replays no emissions, so a node restored from an earlier run has
|
|
no rows of its own — it carries that run's id instead, and its series is read
|
|
from there. Names are re-qualified on the way out, because the same node
|
|
reached through two flows publishes under two names and the caller asked for
|
|
this run's.
|
|
"""
|
|
statement = select(RunMetric).where(col(RunMetric.run_id) == run_id)
|
|
if name:
|
|
statement = statement.where(col(RunMetric.name) == name)
|
|
rows = list(session.exec(statement))
|
|
|
|
restored = session.exec(
|
|
select(RunNode).where(
|
|
col(RunNode.run_id) == run_id, col(RunNode.cached_from) != ""
|
|
)
|
|
).all()
|
|
if restored:
|
|
run = session.get(Run, run_id)
|
|
flow = run.flow if run is not None else ""
|
|
for node_row in restored:
|
|
source = session.get(Run, node_row.cached_from)
|
|
if source is None:
|
|
# The run it came from is gone — deleted with its flow. The
|
|
# outputs are still on this run; the curve is not recoverable.
|
|
continue
|
|
source_node = requalify(node_row.node, flow, source.flow)
|
|
for row in session.exec(
|
|
select(RunMetric).where(
|
|
col(RunMetric.run_id) == node_row.cached_from,
|
|
col(RunMetric.node) == source_node,
|
|
)
|
|
):
|
|
renamed = requalify(row.name, source.flow, flow)
|
|
if name and renamed != name:
|
|
continue
|
|
rows.append(
|
|
RunMetric(
|
|
run_id=run_id,
|
|
name=renamed,
|
|
step=row.step,
|
|
node=node_row.node,
|
|
ts=row.ts,
|
|
value=row.value,
|
|
)
|
|
)
|
|
|
|
rows.sort(key=lambda row: (row.name, row.step))
|
|
return rows
|
|
|
|
|
|
@router.get("/{run_id}/metrics", response_model=list[MetricPoint])
|
|
def read_metrics(
|
|
run_id: str, session: SessionDep, name: str = "", stride: int = 1
|
|
) -> Any:
|
|
"""One metric's series, in step order — or every one of them, unnamed.
|
|
|
|
``stride`` thins a long curve down: 3000 steps drawn on a 400-pixel chart
|
|
is 3000 points nobody can see.
|
|
"""
|
|
rows = _series(session, run_id, name)
|
|
if stride > 1:
|
|
rows = rows[:: max(1, stride)]
|
|
return rows
|
|
|
|
|
|
def _points(session: Session, run_id: str, metric: str, x: str) -> list[list[float]]:
|
|
"""One run's readings of ``metric``, against whichever x was asked for.
|
|
|
|
The step is the default because it is what every run has. Time answers
|
|
"which one got there sooner", and is measured from this run's own first
|
|
reading so that runs started hours apart still lie on top of each other.
|
|
Another metric answers "against what the loop was actually counting" — an
|
|
epoch, or samples seen — and is joined on the step the two share, which is
|
|
the only thing they have in common.
|
|
"""
|
|
rows = _series(session, run_id, metric)
|
|
if x == "time":
|
|
if not rows:
|
|
return []
|
|
start = min(row.ts for row in rows)
|
|
return [[row.ts - start, row.value] for row in rows]
|
|
if x and x != "step":
|
|
against = {row.step: row.value for row in _series(session, run_id, x)}
|
|
return [[against[row.step], row.value] for row in rows if row.step in against]
|
|
return [[float(row.step), row.value] for row in rows]
|
|
|
|
|
|
@router.get("/series/compare", response_model=SeriesAnswer)
|
|
def compare_metric(session: SessionDep, ids: str, metric: str, x: str = "") -> Any:
|
|
"""One metric across several runs, as the chart widget's series shape.
|
|
|
|
This is the comparison view: it answers in the same shape a flow answers a
|
|
chart's query with, so putting three training curves beside each other is
|
|
a widget binding rather than a screen of its own.
|
|
|
|
``x`` names what to plot against — nothing or "step", "time", or another
|
|
metric of the same runs.
|
|
"""
|
|
run_ids = [part for part in ids.split(",") if part]
|
|
if not run_ids:
|
|
raise HTTPException(status_code=422, detail="Name at least one run")
|
|
runs = {
|
|
run.id: run
|
|
for run in session.exec(select(Run).where(col(Run.id).in_(run_ids))).all()
|
|
}
|
|
lines: list[MetricSeries] = []
|
|
for run_id in run_ids:
|
|
run = runs.get(run_id)
|
|
if run is None:
|
|
continue
|
|
label = run_id
|
|
if run.seed is not None:
|
|
label = f"{run_id} (seed {run.seed})"
|
|
lines.append(
|
|
MetricSeries(label=label, points=_points(session, run_id, metric, x))
|
|
)
|
|
return SeriesAnswer(metric=metric, x=x or "step", lines=lines)
|