History is kept by flow id: runs, cascades, rollups and events file the id in their flow column and nodes as <flow id>.<node>, the API answers flow plus flow_name and shows and takes nodes as <key>.<node>; events carry flow_id from the moment they are published; run queue items, the stage cache and cached curves keep ids and keys apart; endpoints, the brain graph, device bindings and search name the id to open

This commit is contained in:
2026-11-02 19:01:19 +01:00
parent e6027af8fe
commit d4a1b1db47
34 changed files with 1374 additions and 329 deletions
+164 -69
View File
@@ -22,13 +22,19 @@ from sqlalchemy import delete, func
from sqlalchemy import select as sa_select
from sqlmodel import Session, col, select
from fluksio.api.deps import CurrentUser, SessionDep, get_current_user
from fluksio.api.deps import (
CurrentUser,
FlowControllerDep,
SessionDep,
get_current_user,
)
from fluksio.core.config import settings
from fluksio.core.db import writing
from fluksio.flow.events import event_bus
from fluksio.flow.messages import requalify
from fluksio.flow.runs import OUTPUT_CAP, RunRejected, RunService, new_run_id
from fluksio.flow.store import FlowNotFound
from fluksio.flow.store import FlowNotFound, FlowStore
from fluksio.flow.tombstones import name_of, names_of, shown_nodes, stored_node
from fluksio.models import FlowRun, Message, Run, RunArtifact, RunMetric, RunNode
router = APIRouter(
@@ -105,6 +111,7 @@ class ImportArtifact(BaseModel):
class ImportEntry(BaseModel):
"""One run executed somewhere else, as it ended there."""
#: The flow's id.
flow: str
#: What it was called where it ran. Importing it again is a no-op.
external_id: str = Field(min_length=1, max_length=255)
@@ -133,6 +140,7 @@ class Imported(BaseModel):
class RunNodeRow(BaseModel):
#: ``<key>.<node>``, the way the engine names it.
node: str
status: str
attempt: int
@@ -153,6 +161,7 @@ class RunNodeRow(BaseModel):
class ArtifactRow(BaseModel):
name: str
#: ``<key>.<node>``.
node: str
digest: str
size: int
@@ -167,7 +176,11 @@ class RunRow(BaseModel):
"""A run without its result, which is the part that can be large."""
id: str
#: The flow's id, which the history is kept under.
flow: str
#: Its key: what it is called now, or what it was last called once it is
#: deleted. The id where nothing remembers either.
flow_name: str = ""
status: str
status_reason: str
cause: str
@@ -224,7 +237,9 @@ class RunDetail(RunRow):
class FlowRunsRow(BaseModel):
"""How much a flow has been run, for the screen's list of flows."""
#: The flow's id, and its key as `RunRow` has it.
flow: str
flow_name: str = ""
runs: int
running: int
queued: int
@@ -260,16 +275,36 @@ def _service(request: Request) -> RunService:
return service
@router.post("/flows/{name}", response_model=RunRow, status_code=202)
def _named[Row: RunRow](
model: type[Row], runs: list[Run], store: FlowStore
) -> list[Row]:
"""Runs as rows, ``flow`` their flow's id and ``flow_name`` its key.
One lookup for the lot, however many flows the page holds.
"""
names = names_of({run.flow_id for run in runs}, store)
return [
model.model_validate(
{**run.model_dump(), "flow": run.flow_id, "flow_name": names[run.flow_id]}
)
for run in runs
]
@router.post("/flows/{flow_id}", response_model=RunRow, status_code=202)
async def create_run(
name: str, body: RunCreate, request: Request, user: CurrentUser
flow_id: str,
body: RunCreate,
request: Request,
user: CurrentUser,
controller: FlowControllerDep,
) -> Any:
"""Queue one run of a flow."""
service = _service(request)
try:
return await run_in_threadpool(
run = await run_in_threadpool(
service.submit,
name,
flow_id,
params=body.params,
seed=body.seed,
cause=body.cause,
@@ -282,11 +317,16 @@ async def create_run(
raise HTTPException(status_code=404, detail=str(exc)) from exc
except RunRejected as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
return _named(RunRow, [run], controller.store)[0]
@router.post("/flows/{name}/sweep", response_model=list[RunRow], status_code=202)
@router.post("/flows/{flow_id}/sweep", response_model=list[RunRow], status_code=202)
async def create_sweep(
name: str, body: SweepCreate, request: Request, user: CurrentUser
flow_id: str,
body: SweepCreate,
request: Request,
user: CurrentUser,
controller: FlowControllerDep,
) -> Any:
"""Queue many runs of one flow under a shared group.
@@ -306,7 +346,7 @@ async def create_sweep(
def submit_all() -> list[Run]:
return [
service.submit(
name,
flow_id,
params=entry.params,
seed=entry.seed,
group_id=group,
@@ -320,11 +360,12 @@ async def create_sweep(
]
try:
return await run_in_threadpool(submit_all)
runs = await run_in_threadpool(submit_all)
except FlowNotFound as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except RunRejected as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
return _named(RunRow, runs, controller.store)
@router.post("/import", response_model=list[Imported], status_code=202)
@@ -369,6 +410,7 @@ async def import_runs(body: ImportCreate, request: Request, user: CurrentUser) -
@router.get("", response_model=list[RunRow])
def read_runs(
session: SessionDep,
controller: FlowControllerDep,
flow: str | None = None,
status: str | None = None,
group: str | None = None,
@@ -384,11 +426,11 @@ def read_runs(
first, so handing back the last row's ``created_at`` reads the next page
whatever landed meanwhile — which ``offset`` cannot, since a run submitted
between two pages shifts every row down one. ``since`` bounds the other
end and is inclusive.
end and is inclusive. ``flow`` is a flow id.
"""
statement = select(Run).order_by(col(Run.created_at).desc())
if flow:
statement = statement.where(col(Run.flow) == flow)
statement = statement.where(col(Run.flow_id) == flow)
if status:
statement = statement.where(col(Run.status) == status)
if group:
@@ -400,11 +442,11 @@ def read_runs(
if before:
statement = statement.where(col(Run.created_at) < _aware(before))
statement = statement.offset(max(0, offset)).limit(min(limit, 500))
return list(session.exec(statement))
return _named(RunRow, list(session.exec(statement)), controller.store)
@router.get("/overview", response_model=list[FlowRunsRow])
def read_overview(session: SessionDep) -> Any:
def read_overview(session: SessionDep, controller: FlowControllerDep) -> Any:
"""One row per flow that has ever run, busiest-recent first.
The list caps at 500 newest runs, so counting flows on the client goes
@@ -416,11 +458,11 @@ def read_overview(session: SessionDep) -> Any:
# rather than making it cheaper to find. A rollup is the answer if the
# scan ever measures.
statement = sa_select(
col(Run.flow),
col(Run.flow_id),
col(Run.status),
func.count(col(Run.id)),
func.max(col(Run.created_at)),
).group_by(col(Run.flow), col(Run.status))
).group_by(col(Run.flow_id), col(Run.status))
rows: dict[str, FlowRunsRow] = {}
for flow, status, count, latest in session.execute(statement):
@@ -432,6 +474,9 @@ def read_overview(session: SessionDep) -> Any:
row.queued += count
if row.last_created_at is None or latest > row.last_created_at:
row.last_created_at = latest
names = names_of(rows, controller.store)
for flow_id, row in rows.items():
row.flow_name = names[flow_id]
return sorted(rows.values(), key=lambda row: row.last_created_at, reverse=True)
@@ -463,7 +508,9 @@ EXPORT_CAP = 10_000
RUN_COLUMNS = (
"id",
#: The flow's id, and its key as `RunRow` has it.
"flow",
"flow_name",
"status",
"created_at",
"started_at",
@@ -489,14 +536,14 @@ def _selection(
since: datetime | None,
until: datetime | None,
) -> Any:
"""The query behind a selection of runs, newest first.
"""The query behind a selection of runs, newest first. ``flow`` is an id.
Separate from reading it, because the names route asks the same question
as a subquery rather than for the rows.
"""
statement = select(Run).order_by(col(Run.created_at).desc())
if flow:
statement = statement.where(col(Run.flow) == flow)
statement = statement.where(col(Run.flow_id) == flow)
if status:
statement = statement.where(col(Run.status) == status)
if group:
@@ -655,6 +702,7 @@ def _stream(
@router.get("/export/metrics")
def export_metrics(
session: SessionDep,
controller: FlowControllerDep,
flow: str | None = None,
status: str | None = None,
group: str | None = None,
@@ -679,7 +727,7 @@ def export_metrics(
for run in runs:
rows: list[dict[str, Any]] = []
for series, points in groupby(
_series(session, run.id), key=lambda row: row.name
_series(session, controller.store, run.id), key=lambda row: row.name
):
if wanted and series not in wanted:
continue
@@ -701,6 +749,7 @@ def export_metrics(
@router.get("/export/runs")
def export_runs(
session: SessionDep,
controller: FlowControllerDep,
flow: str | None = None,
status: str | None = None,
group: str | None = None,
@@ -734,9 +783,15 @@ def export_runs(
*(f"metric.{key}" for key in scores),
]
names = names_of({run.flow_id for run in runs}, controller.store)
def chunks() -> Iterator[list[dict[str, Any]]]:
for run in runs:
row = {column: _cell(getattr(run, column)) for column in RUN_COLUMNS}
own = {"flow": run.flow_id, "flow_name": names[run.flow_id]}
row = {
column: _cell(own[column] if column in own else getattr(run, column))
for column in RUN_COLUMNS
}
row.update((f"param.{k}", _cell(_dig(run.params, k))) for k in inputs)
row.update((f"metric.{k}", _cell(_dig(run.result, k))) for k in scores)
yield [row]
@@ -773,7 +828,7 @@ def read_metric_names(
@router.get("/{run_id}", response_model=RunDetail)
def read_run(run_id: str, session: SessionDep) -> Any:
def read_run(run_id: str, session: SessionDep, controller: FlowControllerDep) -> Any:
"""One run in full: what it was asked, what each node did, what it made."""
run = session.get(Run, run_id)
if run is None:
@@ -782,8 +837,16 @@ def read_run(run_id: str, session: SessionDep) -> Any:
artifacts = session.exec(
select(RunArtifact).where(col(RunArtifact.run_id) == run_id)
).all()
detail = RunDetail.model_validate(run, from_attributes=True)
detail.nodes = [RunNodeRow.model_validate(n, from_attributes=True) for n in nodes]
store = controller.store
detail = _named(RunDetail, [run], store)[0]
# Stored as `<flow id>.<node>`, shown as the engine names them.
shown = shown_nodes({n.node for n in nodes} | {a.node for a in artifacts}, store)
detail.nodes = [
RunNodeRow.model_validate(n, from_attributes=True).model_copy(
update={"node": shown[n.node]}
)
for n in nodes
]
# Worked out here, not stored: a source run is deleted long after the
# nodes restored from it were written.
sources = {n.cached_from for n in nodes if n.cached_from}
@@ -795,19 +858,29 @@ def read_run(run_id: str, session: SessionDep) -> Any:
for row in detail.nodes:
row.curve_lost = bool(row.cached_from) and row.cached_from not in kept
detail.artifacts = [
ArtifactRow.model_validate(a, from_attributes=True) for a in artifacts
ArtifactRow.model_validate(a, from_attributes=True).model_copy(
update={"node": shown[a.node]}
)
for a in artifacts
]
return detail
@router.get("/{run_id}/nodes/{node}/outputs", response_model=dict[str, Any] | None)
def read_node_outputs(run_id: str, node: str, session: SessionDep) -> Any:
def read_node_outputs(
run_id: str, node: str, session: SessionDep, controller: FlowControllerDep
) -> Any:
"""What one node returned, by message, as the stage cache stored it.
Kept only for a node that succeeded and may be reused, and only up to
256 kB of JSON. Null when the node published nothing.
The node is ``<key>.<node>``, as the run shows it. Kept only for a node
that succeeded and may be reused, and only up to 256 kB of JSON. Null when
the node published nothing.
"""
row = session.get(RunNode, {"run_id": run_id, "node": node})
run = session.get(Run, run_id)
if run is None:
raise HTTPException(status_code=404, detail="No such run")
stored = stored_node(node, controller.store, run.flow_id)
row = session.get(RunNode, {"run_id": run_id, "node": stored})
if row is None:
raise HTTPException(status_code=404, detail=f"No node '{node}' in this run")
if row.outputs is None:
@@ -823,7 +896,9 @@ def read_node_outputs(run_id: str, node: str, session: SessionDep) -> Any:
@router.post("/{run_id}/cancel", response_model=RunRow)
async def cancel_run(run_id: str, request: Request, session: SessionDep) -> Any:
async def cancel_run(
run_id: str, request: Request, session: SessionDep, controller: FlowControllerDep
) -> Any:
"""Stop a run. One already past its last node is left as it finished."""
run = session.get(Run, run_id)
if run is None:
@@ -831,21 +906,24 @@ async def cancel_run(run_id: str, request: Request, session: SessionDep) -> Any:
service = _service(request)
await run_in_threadpool(service.cancel, run_id)
session.refresh(run)
return run
return _named(RunRow, [run], controller.store)[0]
@router.post("/{run_id}/retry", response_model=RunRow, status_code=202)
async def retry_run(run_id: str, request: Request, user: CurrentUser) -> Any:
async def retry_run(
run_id: str, request: Request, user: CurrentUser, controller: FlowControllerDep
) -> Any:
"""Run the same thing again, as a new run pointing back at this one."""
service = _service(request)
try:
return await run_in_threadpool(service.retry, run_id, user.email)
run = await run_in_threadpool(service.retry, run_id, user.email)
except FlowNotFound as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except RunRejected as exc:
detail = str(exc)
status = 404 if detail.startswith("No run ") else 422
raise HTTPException(status_code=status, detail=detail) from exc
return _named(RunRow, [run], controller.store)[0]
def _drop_runs(session: Session, chosen: Any) -> None:
@@ -867,23 +945,38 @@ def _drop_runs(session: Session, chosen: Any) -> None:
session.execute(delete(Run).where(col(Run.id).in_(chosen)))
def _live_run(session: Session, flow: str) -> str | None:
def _live_run(session: Session, flow_id: str) -> str | None:
"""A run of this flow the driver is still writing to, if there is one."""
return session.exec(
select(Run.id).where(
col(Run.flow) == flow, col(Run.status).in_(("running", "queued"))
col(Run.flow_id) == flow_id, col(Run.status).in_(("running", "queued"))
)
).first()
def _audit(action: str, flow_id: str, user: CurrentUser, store: FlowStore) -> None:
"""Record who deleted runs. On the bus a flow goes by its key and id."""
event_bus.publish(
{
"type": "audit",
"action": action,
"flow": name_of(flow_id, store),
"flow_id": flow_id,
"user": user.email,
"ts": time.time(),
}
)
@router.delete("", response_model=Message)
def delete_runs(
session: SessionDep,
user: CurrentUser,
controller: FlowControllerDep,
flow: Annotated[str, Query(min_length=1)],
history: bool = False,
) -> Any:
"""Every run of one flow at once.
"""Every run of one flow at once, ``flow`` being its id.
The counterpart to the list's ``flow`` filter, and what a reseed needs:
deleting a flow leaves its runs standing on purpose, so this is the only
@@ -906,28 +999,22 @@ def delete_runs(
)
# Counted before the delete, and a subquery rather than a materialised
# list of ids on the way in: a demo can hold thousands.
chosen = select(col(Run.id)).where(col(Run.flow) == flow)
chosen = select(col(Run.id)).where(col(Run.flow_id) == flow)
gone = session.exec(
select(func.count()).select_from(Run).where(col(Run.flow) == flow)
select(func.count()).select_from(Run).where(col(Run.flow_id) == flow)
).one()
_drop_runs(session, chosen)
if history:
session.execute(delete(FlowRun).where(col(FlowRun.flow) == flow))
session.execute(delete(FlowRun).where(col(FlowRun.flow_id) == flow))
session.commit()
event_bus.publish(
{
"type": "audit",
"action": f"deleted {gone} run(s)",
"flow": flow,
"user": user.email,
"ts": time.time(),
}
)
_audit(f"deleted {gone} run(s)", flow, user, controller.store)
return Message(message=f"Deleted {gone} run(s) of '{flow}'")
@router.delete("/{run_id}", status_code=204)
def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response:
def delete_run(
run_id: str, session: SessionDep, user: CurrentUser, controller: FlowControllerDep
) -> Response:
"""Forget a run and everything hanging off it.
A live run is refused rather than raced: the driver writes its nodes back
@@ -949,29 +1036,24 @@ def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response:
"finish, before deleting it."
),
)
flow = run.flow
flow_id = run.flow_id
_drop_runs(session, [run_id])
session.commit()
event_bus.publish(
{
"type": "audit",
"action": f"deleted run {run_id}",
"flow": flow,
"user": user.email,
"ts": time.time(),
}
)
_audit(f"deleted run {run_id}", flow_id, user, controller.store)
return Response(status_code=204)
def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
def _series(
session: Session, store: FlowStore, run_id: str, name: str = ""
) -> list[RunMetric]:
"""A run's numbers, including the ones a cached node points at.
A cache hit replays no emissions, so a node restored from an earlier run has
no rows of its own — it carries that run's id instead, and its series is read
from there. Names are re-qualified on the way out, because the same node
reached through two flows publishes under two names and the caller asked for
this run's.
this run's. Nodes are found by flow id, which is how rows name them, and
names are moved between the two flows' keys, which is how they are named.
"""
statement = select(RunMetric).where(col(RunMetric.run_id) == run_id)
if name:
@@ -985,16 +1067,17 @@ def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
).all()
if restored:
run = session.get(Run, run_id)
flow = run.flow if run is not None else ""
flow = run.flow_id if run is not None else ""
# Two queries for the lot, rather than a `Run` lookup and a `RunMetric`
# query per restored node. A comparison of twenty runs calls this
# twenty times, so the per-node round trips multiplied.
sources = {
row.id: row.flow
row.id: row.flow_id
for row in session.exec(
select(Run).where(col(Run.id).in_({n.cached_from for n in restored}))
)
}
keys = names_of({flow, *sources.values()}, store)
wanted: dict[tuple[str, str], list[RunNode]] = {}
for node_row in restored:
source_flow = sources.get(node_row.cached_from)
@@ -1017,7 +1100,7 @@ def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
for row in cached:
for node_row in wanted.get((row.run_id, row.node), ()):
source_flow = sources[node_row.cached_from]
renamed = requalify(row.name, source_flow, flow)
renamed = requalify(row.name, keys[source_flow], keys[flow])
if name and renamed != name:
continue
rows.append(
@@ -1039,6 +1122,7 @@ def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
def read_metrics(
run_id: str,
session: SessionDep,
controller: FlowControllerDep,
name: str = "",
stride: Annotated[int, Query(ge=1, le=100_000)] = 1,
) -> Any:
@@ -1047,13 +1131,15 @@ def read_metrics(
``stride`` thins a long curve down: 3000 steps drawn on a 400-pixel chart
is 3000 points nobody can see.
"""
rows = _series(session, run_id, name)
rows = _series(session, controller.store, run_id, name)
if stride > 1:
rows = rows[:: max(1, stride)]
return rows
def _points(session: Session, run_id: str, metric: str, x: str) -> list[list[float]]:
def _points(
session: Session, store: FlowStore, run_id: str, metric: str, x: str
) -> list[list[float]]:
"""One run's readings of ``metric``, against whichever x was asked for.
The step is the default because it is what every run has. Time answers
@@ -1063,20 +1149,26 @@ def _points(session: Session, run_id: str, metric: str, x: str) -> list[list[flo
epoch, or samples seen — and is joined on the step the two share, which is
the only thing they have in common.
"""
rows = _series(session, run_id, metric)
rows = _series(session, store, run_id, metric)
if x == "time":
if not rows:
return []
start = min(row.ts for row in rows)
return [[row.ts - start, row.value] for row in rows]
if x and x != "step":
against = {row.step: row.value for row in _series(session, run_id, x)}
against = {row.step: row.value for row in _series(session, store, run_id, x)}
return [[against[row.step], row.value] for row in rows if row.step in against]
return [[float(row.step), row.value] for row in rows]
@router.get("/series/compare", response_model=SeriesAnswer)
def compare_metric(session: SessionDep, ids: str, metric: str, x: str = "") -> Any:
def compare_metric(
session: SessionDep,
controller: FlowControllerDep,
ids: str,
metric: str,
x: str = "",
) -> Any:
"""One metric across several runs, as the chart widget's series shape.
This is the comparison view: it answers in the same shape a flow answers a
@@ -1102,6 +1194,9 @@ def compare_metric(session: SessionDep, ids: str, metric: str, x: str = "") -> A
if run.seed is not None:
label = f"{run_id} (seed {run.seed})"
lines.append(
MetricSeries(label=label, points=_points(session, run_id, metric, x))
MetricSeries(
label=label,
points=_points(session, controller.store, run_id, metric, x),
)
)
return SeriesAnswer(metric=metric, x=x or "step", lines=lines)