History is kept by flow id: runs, cascades, rollups and events file the id in their flow column and nodes as <flow id>.<node>, the API answers flow plus flow_name and shows and takes nodes as <key>.<node>; events carry flow_id from the moment they are published; run queue items, the stage cache and cached curves keep ids and keys apart; endpoints, the brain graph, device bindings and search name the id to open

This commit is contained in:
2026-11-02 19:01:19 +01:00
parent e6027af8fe
commit d4a1b1db47
34 changed files with 1374 additions and 329 deletions
+6 -5
View File
@@ -110,19 +110,20 @@ async def delete_flavor(
def _usages(controller: FlowControllerDep, name: str) -> list[str]:
"""Which nodes name this flavor, published or in a draft."""
"""Which nodes name this flavor, published or in a draft, as
``<key>.<node>``."""
used: list[str] = []
for flow_name in controller.store.list_flows():
for flow_id in controller.store.list_flows():
seen = set()
for draft in (False, True):
if draft and not controller.store.has_draft(flow_name):
if draft and not controller.store.has_draft(flow_id):
continue
try:
flow = controller.store.read_flow(flow_name, draft=draft)
flow = controller.store.read_flow(flow_id, draft=draft)
except Exception:
continue
for node in flow.nodes:
if node.resources is not None and node.resources.flavor == name:
seen.add(f"{flow_name}.{node.id}")
seen.add(f"{flow.name}.{node.id}")
used.extend(sorted(seen))
return used
+42 -22
View File
@@ -119,6 +119,9 @@ class Endpoint(BaseModel):
kind: str
#: Stable within its kind, and used as the canvas node id.
id: str
#: The id of the dashboard or flow it sits in, which is what opening it
#: goes by. Never read it off ``id``, whose flow part is a key.
owner_id: str = ""
label: str
#: What sort of widget, or the node type in the other flow.
detail: str = ""
@@ -194,24 +197,31 @@ def _endpoints(controller: FlowController, definition: FlowDef) -> list[Endpoint
found[f"dashboard:{binding['dashboard']}:{binding['widget']}"] = Endpoint(
kind="dashboard",
id=f"dashboard:{binding['dashboard']}:{binding['widget']}",
owner_id=binding["dashboard"],
label=binding["title"],
detail=binding["type"],
provides=[binding["provides"]] if binding["provides"] else [],
requires=binding["requires"],
)
def _flow_endpoint(key: str, detail: str) -> Endpoint:
def _flow_endpoint(key: str, detail: str, owner_id: str = "") -> Endpoint:
"""One label per node on the far side, however many names reach it."""
return found.setdefault(
f"flow:{key}",
Endpoint(kind="flow", id=f"flow:{key}", label=key, detail=detail),
Endpoint(
kind="flow",
id=f"flow:{key}",
owner_id=owner_id,
label=key,
detail=detail,
),
)
# An outsider reaching into this flow.
for other, node_id, provides, requires in controller.cross_flow_nodes(flow):
endpoint = _flow_endpoint(f"{other}.{node_id}", "flow")
endpoint.provides += provides
endpoint.requires += requires
for other in controller.cross_flow_nodes(flow):
endpoint = _flow_endpoint(f"{other.flow}.{other.node}", "flow", other.flow_id)
endpoint.provides += other.provides
endpoint.requires += other.requires
# And this flow reaching out: its own ports bound to a message of another
# flow. Read from the working document rather than from the store, so a
@@ -235,6 +245,7 @@ def _endpoints(controller: FlowController, definition: FlowDef) -> list[Endpoint
endpoint = _flow_endpoint(
f"{far.flow}.{far.node}" if far else message,
far.type if far else "flow",
far.flow_id if far else "",
)
# An endpoint publishing into this flow is what this flow reads.
side = endpoint.requires if ours_publishes else endpoint.provides
@@ -289,14 +300,15 @@ def _read_flow(controller: FlowController, flow_id: str) -> FlowDef:
raise _missing(flow_id)
def _audit(action: str, flow: str, user: CurrentUser) -> None:
"""Record who changed what, by the flow's key. The collector writes it
down; the bus carries it."""
def _audit(action: str, key: str, flow_id: str, user: CurrentUser) -> None:
"""Record who changed what. The collector writes it down under the flow's
id; the bus carries the key with it, as it does for every event."""
event_bus.publish(
{
"type": "audit",
"action": action,
"flow": flow,
"flow": key,
"flow_id": flow_id,
"user": user.email,
"ts": time.time(),
}
@@ -415,13 +427,12 @@ async def delete_shared_node(
first, so the source can be retired while flows still use it.
"""
if detach:
running = await run_in_threadpool(controller.store.usages, lib_name, False)
running = await run_in_threadpool(controller.store.lib_users, lib_name)
try:
await run_in_threadpool(controller.store.detach_lib, lib_name)
except LibNotFound:
raise HTTPException(status_code=404, detail=f"No shared node '{lib_name}'")
# `usages` qualifies with the flow's id.
for flow_id in sorted({usage.split(".", 1)[0] for usage in running}):
for flow_id in running:
await controller.reload_flow(flow_id)
return Message(message=f"Deleted shared node '{lib_name}'")
# The published flows too: a draft that unshared a node still runs it.
@@ -643,7 +654,7 @@ async def publish_flow(
status_code=409,
detail={"message": str(exc), "current_version": exc.current},
)
_audit("published", definition.name, user)
_audit("published", definition.name, flow_id, user)
await controller.reload_flow(flow_id)
return await run_in_threadpool(_detail, controller, published)
@@ -680,7 +691,7 @@ async def delete_flow(
"""
live = session.exec(
select(Run.id).where(
col(Run.flow) == flow_id, col(Run.status).in_(("running", "queued"))
col(Run.flow_id) == flow_id, col(Run.status).in_(("running", "queued"))
)
).first()
if live is not None:
@@ -695,7 +706,7 @@ async def delete_flow(
key = await run_in_threadpool(controller.store.delete_flow, flow_id)
except FlowNotFound:
raise _missing(flow_id)
_audit("deleted", key, user)
_audit("deleted", key, flow_id, user)
# Its nodes first, or they would put values and work straight back.
# Its runs stay: a run record is a record of what ran, and surviving the
# flow it belonged to is the point of keeping one. `DELETE /runs?flow=`
@@ -824,7 +835,7 @@ async def save_node_source(
controller.store.write_lib_source, ref, source.code
)
if changed:
_audit(f"edited shared node '{ref}'", name, user)
_audit(f"edited shared node '{ref}'", name, flow_id, user)
await controller.reload_lib_users(ref)
else:
await run_in_threadpool(
@@ -905,7 +916,7 @@ async def start_flow(
"""Let the engine run this flow again."""
definition = _read_flow(controller, flow_id)
await controller.set_enabled(flow_id, definition.name, True)
_audit("started", definition.name, user)
_audit("started", definition.name, flow_id, user)
return await run_in_threadpool(_detail, controller, definition)
@@ -916,7 +927,7 @@ async def stop_flow(
"""Take this flow off the engine: no subscriptions, schedules or webhooks."""
definition = _read_flow(controller, flow_id)
await controller.set_enabled(flow_id, definition.name, False)
_audit("stopped", definition.name, user)
_audit("stopped", definition.name, flow_id, user)
return await run_in_threadpool(_detail, controller, definition)
@@ -1123,9 +1134,14 @@ async def train_node(
if service is None:
raise HTTPException(status_code=503, detail="Runs are not available")
try:
# Registered by `fluksio sync`, which gives it whatever id it gets: the
# built-in flow is known by its key.
train_id = controller.store.key_index().get(TRAIN_FLOW)
if train_id is None:
raise FlowNotFound(TRAIN_FLOW)
run = await run_in_threadpool(
service.submit,
TRAIN_FLOW,
train_id,
params={"installation": orjson.dumps(wren.installation()).decode()},
cause="train",
actor=user.email,
@@ -1153,11 +1169,15 @@ def read_training(
controller: FlowControllerDep,
session: SessionDep,
) -> Any:
"""A wren node's latest training run, or null when it has none."""
"""A wren node's latest training run, or null when it has none.
Found by the node it trains alone: only a training run names one, and
the training flow may have been registered again under another id since.
"""
_node_def(controller, flow_id, node_id)
run = session.exec(
select(Run)
.where(col(Run.flow) == TRAIN_FLOW, col(Run.for_node) == f"{flow_id}.{node_id}")
.where(col(Run.for_node) == f"{flow_id}.{node_id}")
.order_by(col(Run.created_at).desc())
).first()
if run is None:
+59 -14
View File
@@ -28,6 +28,7 @@ from fluksio.api.deps import (
from fluksio.api.routes.runs import elapsed_ms
from fluksio.core.config import settings
from fluksio.flow.controller import NodeStatus
from fluksio.flow.tombstones import names_of, shown_nodes, stored_node
from fluksio.models import EngineEvent, FlowRun, Message, MetricBucket
router = APIRouter(
@@ -77,7 +78,9 @@ class SeriesPoint(BaseModel):
class FlowRollup(BaseModel):
#: The flow's id, and its key: what it is called now, or was last called.
flow: str
flow_name: str = ""
executions: int
errors: int
messages: int
@@ -100,7 +103,9 @@ class NodeTypeTiming(BaseModel):
class RunRow(BaseModel):
id: str
#: The flow's id, and its key as `FlowRollup` has it.
flow: str
flow_name: str = ""
source: str
#: The run this one continues: for a flush, the run whose value it released.
parent: str = ""
@@ -130,7 +135,11 @@ class EventRow(BaseModel):
id: int
ts: datetime
type: str
#: The flow's id, and its key as `FlowRollup` has it. Both empty for the
#: engine as a whole.
flow: str
flow_name: str = ""
#: ``<key>.<node>``, or empty.
node: str
detail: str
actor: str
@@ -138,6 +147,8 @@ class EventRow(BaseModel):
class DeadLetter(BaseModel):
"""A work item given up on, named as the engine runs it: by key."""
id: str
ts: float
flow: str
@@ -317,6 +328,7 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
@router.get("/timeseries", response_model=list[SeriesPoint])
def read_timeseries(
session: SessionDep,
controller: FlowControllerDep,
flow: str | None = None,
node: str | None = None,
hours: Annotated[int, Query(ge=0, le=8760)] = 24,
@@ -331,6 +343,8 @@ def read_timeseries(
the same pair ``/runs`` and ``/events`` take — so a chart dragged to a
span can be re-fetched at that span's own resolution rather than
magnifying the buckets it already has.
``flow`` is a flow id, ``node`` a ``<key>.<node>`` as the engine names it.
"""
hours = _window_hours(hours)
# The database does the fold: a week of minute rows per node used to cross
@@ -356,9 +370,10 @@ def read_timeseries(
col(MetricBucket.bucket) < (_aware(until) if until else _closed_before(stride)),
)
if flow:
statement = statement.where(col(MetricBucket.flow) == flow)
statement = statement.where(col(MetricBucket.flow_id) == flow)
if node:
statement = statement.where(col(MetricBucket.node) == node)
stored = stored_node(node, controller.store, flow or "")
statement = statement.where(col(MetricBucket.node) == stored)
return [
SeriesPoint(
@@ -376,7 +391,9 @@ def read_timeseries(
@router.get("/flows", response_model=list[FlowRollup])
def read_flow_rollups(
session: SessionDep, hours: Annotated[int, Query(ge=0, le=8760)] = 24
session: SessionDep,
controller: FlowControllerDep,
hours: Annotated[int, Query(ge=0, le=8760)] = 24,
) -> Any:
"""One row per flow, with a coarse trend of how much it ran."""
hours = _window_hours(hours)
@@ -399,7 +416,7 @@ def read_flow_rollups(
).label("slot")
statement = (
sa_select(
col(MetricBucket.flow),
col(MetricBucket.flow_id),
slot,
func.sum(col(MetricBucket.executions)).label("executions"),
func.sum(col(MetricBucket.errors)).label("errors"),
@@ -412,13 +429,13 @@ def read_flow_rollups(
col(MetricBucket.bucket) >= since,
col(MetricBucket.bucket) < datetime.fromtimestamp(float(end), UTC),
)
.group_by(col(MetricBucket.flow), slot)
.group_by(col(MetricBucket.flow_id), slot)
)
rollups: dict[str, dict[str, Any]] = {}
for row in session.execute(statement):
entry = rollups.setdefault(
row.flow,
row.flow_id,
{
"executions": 0,
"errors": 0,
@@ -448,15 +465,17 @@ def read_flow_rollups(
# mapping and subscript it.
last_errors = dict(
session.exec(
select(col(EngineEvent.flow), func.max(col(EngineEvent.ts)))
select(col(EngineEvent.flow_id), func.max(col(EngineEvent.ts)))
.where(col(EngineEvent.type) == "node_error", col(EngineEvent.ts) >= since)
.group_by(col(EngineEvent.flow))
.group_by(col(EngineEvent.flow_id))
).all()
)
names = names_of(rollups, controller.store)
return [
FlowRollup(
flow=flow,
flow_name=names[flow],
executions=entry["executions"],
errors=entry["errors"],
messages=entry["messages"],
@@ -483,8 +502,9 @@ def read_node_type_timings(
flows. A node deleted since it ran has no type left to count under.
"""
hours = _window_hours(hours)
# By `<flow id>.<node>`, the way the rollups name a node.
types = {
f"{flow.name}.{node.id}": node.type
f"{flow.id}.{node.id}": node.type
for flow in controller.store.read_all()
for node in flow.nodes
}
@@ -526,6 +546,7 @@ def read_node_type_timings(
@router.get("/runs", response_model=RunPage)
def read_runs(
session: SessionDep,
controller: FlowControllerDep,
flow: str | None = None,
status: str | None = None,
since: datetime | None = None,
@@ -536,10 +557,11 @@ def read_runs(
``since`` is inclusive and ``until`` exclusive, so a window of one minute
holds exactly the runs of the minute bucket the charts are drawn from.
``flow`` is a flow id.
"""
filters: list[ColumnElement[bool]] = []
if flow:
filters.append(col(FlowRun.flow) == flow)
filters.append(col(FlowRun.flow_id) == flow)
if status:
filters.append(col(FlowRun.status) == status)
if since:
@@ -563,12 +585,20 @@ def read_runs(
).one()
)
)
return {"data": rows, "count": count}
names = names_of({row.flow_id for row in rows}, controller.store)
data = [
RunRow.model_validate(
{**row.model_dump(), "flow": row.flow_id, "flow_name": names[row.flow_id]}
)
for row in rows
]
return {"data": data, "count": count}
@router.get("/events", response_model=list[EventRow])
def read_events(
session: SessionDep,
controller: FlowControllerDep,
kind: Literal["failure", "audit"] = "failure",
flow: str | None = None,
run: str | None = None,
@@ -582,7 +612,8 @@ def read_events(
takes, so a list can cover the span the charts beside it are drawn from.
``run`` narrows to one run — a batch run's id, or the journaled item a live
cascade came from. Rows recorded before the column existed carry none, so
an old failure answers no run at all rather than the wrong one.
an old failure answers no run at all rather than the wrong one. ``flow``
is a flow id.
"""
statement = select(EngineEvent).order_by(col(EngineEvent.ts).desc())
if kind == "audit":
@@ -590,14 +621,28 @@ def read_events(
else:
statement = statement.where(col(EngineEvent.type) != "audit")
if flow:
statement = statement.where(col(EngineEvent.flow) == flow)
statement = statement.where(col(EngineEvent.flow_id) == flow)
if run:
statement = statement.where(col(EngineEvent.run) == run)
if since:
statement = statement.where(col(EngineEvent.ts) >= _aware(since))
if until:
statement = statement.where(col(EngineEvent.ts) < _aware(until))
return list(session.exec(statement.limit(min(limit, 500))))
rows = list(session.exec(statement.limit(min(limit, 500))))
store = controller.store
names = names_of({row.flow_id for row in rows if row.flow_id}, store)
nodes = shown_nodes({row.node for row in rows if row.node}, store)
return [
EventRow.model_validate(
{
**row.model_dump(),
"flow": row.flow_id,
"flow_name": names.get(row.flow_id, ""),
"node": nodes.get(row.node, ""),
}
)
for row in rows
]
@router.get("/dead-letter", response_model=list[DeadLetter])
+164 -69
View File
@@ -22,13 +22,19 @@ from sqlalchemy import delete, func
from sqlalchemy import select as sa_select
from sqlmodel import Session, col, select
from fluksio.api.deps import CurrentUser, SessionDep, get_current_user
from fluksio.api.deps import (
CurrentUser,
FlowControllerDep,
SessionDep,
get_current_user,
)
from fluksio.core.config import settings
from fluksio.core.db import writing
from fluksio.flow.events import event_bus
from fluksio.flow.messages import requalify
from fluksio.flow.runs import OUTPUT_CAP, RunRejected, RunService, new_run_id
from fluksio.flow.store import FlowNotFound
from fluksio.flow.store import FlowNotFound, FlowStore
from fluksio.flow.tombstones import name_of, names_of, shown_nodes, stored_node
from fluksio.models import FlowRun, Message, Run, RunArtifact, RunMetric, RunNode
router = APIRouter(
@@ -105,6 +111,7 @@ class ImportArtifact(BaseModel):
class ImportEntry(BaseModel):
"""One run executed somewhere else, as it ended there."""
#: The flow's id.
flow: str
#: What it was called where it ran. Importing it again is a no-op.
external_id: str = Field(min_length=1, max_length=255)
@@ -133,6 +140,7 @@ class Imported(BaseModel):
class RunNodeRow(BaseModel):
#: ``<key>.<node>``, the way the engine names it.
node: str
status: str
attempt: int
@@ -153,6 +161,7 @@ class RunNodeRow(BaseModel):
class ArtifactRow(BaseModel):
name: str
#: ``<key>.<node>``.
node: str
digest: str
size: int
@@ -167,7 +176,11 @@ class RunRow(BaseModel):
"""A run without its result, which is the part that can be large."""
id: str
#: The flow's id, which the history is kept under.
flow: str
#: Its key: what it is called now, or what it was last called once it is
#: deleted. The id where nothing remembers either.
flow_name: str = ""
status: str
status_reason: str
cause: str
@@ -224,7 +237,9 @@ class RunDetail(RunRow):
class FlowRunsRow(BaseModel):
"""How much a flow has been run, for the screen's list of flows."""
#: The flow's id, and its key as `RunRow` has it.
flow: str
flow_name: str = ""
runs: int
running: int
queued: int
@@ -260,16 +275,36 @@ def _service(request: Request) -> RunService:
return service
@router.post("/flows/{name}", response_model=RunRow, status_code=202)
def _named[Row: RunRow](
model: type[Row], runs: list[Run], store: FlowStore
) -> list[Row]:
"""Runs as rows, ``flow`` their flow's id and ``flow_name`` its key.
One lookup for the lot, however many flows the page holds.
"""
names = names_of({run.flow_id for run in runs}, store)
return [
model.model_validate(
{**run.model_dump(), "flow": run.flow_id, "flow_name": names[run.flow_id]}
)
for run in runs
]
@router.post("/flows/{flow_id}", response_model=RunRow, status_code=202)
async def create_run(
name: str, body: RunCreate, request: Request, user: CurrentUser
flow_id: str,
body: RunCreate,
request: Request,
user: CurrentUser,
controller: FlowControllerDep,
) -> Any:
"""Queue one run of a flow."""
service = _service(request)
try:
return await run_in_threadpool(
run = await run_in_threadpool(
service.submit,
name,
flow_id,
params=body.params,
seed=body.seed,
cause=body.cause,
@@ -282,11 +317,16 @@ async def create_run(
raise HTTPException(status_code=404, detail=str(exc)) from exc
except RunRejected as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
return _named(RunRow, [run], controller.store)[0]
@router.post("/flows/{name}/sweep", response_model=list[RunRow], status_code=202)
@router.post("/flows/{flow_id}/sweep", response_model=list[RunRow], status_code=202)
async def create_sweep(
name: str, body: SweepCreate, request: Request, user: CurrentUser
flow_id: str,
body: SweepCreate,
request: Request,
user: CurrentUser,
controller: FlowControllerDep,
) -> Any:
"""Queue many runs of one flow under a shared group.
@@ -306,7 +346,7 @@ async def create_sweep(
def submit_all() -> list[Run]:
return [
service.submit(
name,
flow_id,
params=entry.params,
seed=entry.seed,
group_id=group,
@@ -320,11 +360,12 @@ async def create_sweep(
]
try:
return await run_in_threadpool(submit_all)
runs = await run_in_threadpool(submit_all)
except FlowNotFound as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except RunRejected as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
return _named(RunRow, runs, controller.store)
@router.post("/import", response_model=list[Imported], status_code=202)
@@ -369,6 +410,7 @@ async def import_runs(body: ImportCreate, request: Request, user: CurrentUser) -
@router.get("", response_model=list[RunRow])
def read_runs(
session: SessionDep,
controller: FlowControllerDep,
flow: str | None = None,
status: str | None = None,
group: str | None = None,
@@ -384,11 +426,11 @@ def read_runs(
first, so handing back the last row's ``created_at`` reads the next page
whatever landed meanwhile — which ``offset`` cannot, since a run submitted
between two pages shifts every row down one. ``since`` bounds the other
end and is inclusive.
end and is inclusive. ``flow`` is a flow id.
"""
statement = select(Run).order_by(col(Run.created_at).desc())
if flow:
statement = statement.where(col(Run.flow) == flow)
statement = statement.where(col(Run.flow_id) == flow)
if status:
statement = statement.where(col(Run.status) == status)
if group:
@@ -400,11 +442,11 @@ def read_runs(
if before:
statement = statement.where(col(Run.created_at) < _aware(before))
statement = statement.offset(max(0, offset)).limit(min(limit, 500))
return list(session.exec(statement))
return _named(RunRow, list(session.exec(statement)), controller.store)
@router.get("/overview", response_model=list[FlowRunsRow])
def read_overview(session: SessionDep) -> Any:
def read_overview(session: SessionDep, controller: FlowControllerDep) -> Any:
"""One row per flow that has ever run, busiest-recent first.
The list caps at 500 newest runs, so counting flows on the client goes
@@ -416,11 +458,11 @@ def read_overview(session: SessionDep) -> Any:
# rather than making it cheaper to find. A rollup is the answer if the
# scan ever measures.
statement = sa_select(
col(Run.flow),
col(Run.flow_id),
col(Run.status),
func.count(col(Run.id)),
func.max(col(Run.created_at)),
).group_by(col(Run.flow), col(Run.status))
).group_by(col(Run.flow_id), col(Run.status))
rows: dict[str, FlowRunsRow] = {}
for flow, status, count, latest in session.execute(statement):
@@ -432,6 +474,9 @@ def read_overview(session: SessionDep) -> Any:
row.queued += count
if row.last_created_at is None or latest > row.last_created_at:
row.last_created_at = latest
names = names_of(rows, controller.store)
for flow_id, row in rows.items():
row.flow_name = names[flow_id]
return sorted(rows.values(), key=lambda row: row.last_created_at, reverse=True)
@@ -463,7 +508,9 @@ EXPORT_CAP = 10_000
RUN_COLUMNS = (
"id",
#: The flow's id, and its key as `RunRow` has it.
"flow",
"flow_name",
"status",
"created_at",
"started_at",
@@ -489,14 +536,14 @@ def _selection(
since: datetime | None,
until: datetime | None,
) -> Any:
"""The query behind a selection of runs, newest first.
"""The query behind a selection of runs, newest first. ``flow`` is an id.
Separate from reading it, because the names route asks the same question
as a subquery rather than for the rows.
"""
statement = select(Run).order_by(col(Run.created_at).desc())
if flow:
statement = statement.where(col(Run.flow) == flow)
statement = statement.where(col(Run.flow_id) == flow)
if status:
statement = statement.where(col(Run.status) == status)
if group:
@@ -655,6 +702,7 @@ def _stream(
@router.get("/export/metrics")
def export_metrics(
session: SessionDep,
controller: FlowControllerDep,
flow: str | None = None,
status: str | None = None,
group: str | None = None,
@@ -679,7 +727,7 @@ def export_metrics(
for run in runs:
rows: list[dict[str, Any]] = []
for series, points in groupby(
_series(session, run.id), key=lambda row: row.name
_series(session, controller.store, run.id), key=lambda row: row.name
):
if wanted and series not in wanted:
continue
@@ -701,6 +749,7 @@ def export_metrics(
@router.get("/export/runs")
def export_runs(
session: SessionDep,
controller: FlowControllerDep,
flow: str | None = None,
status: str | None = None,
group: str | None = None,
@@ -734,9 +783,15 @@ def export_runs(
*(f"metric.{key}" for key in scores),
]
names = names_of({run.flow_id for run in runs}, controller.store)
def chunks() -> Iterator[list[dict[str, Any]]]:
for run in runs:
row = {column: _cell(getattr(run, column)) for column in RUN_COLUMNS}
own = {"flow": run.flow_id, "flow_name": names[run.flow_id]}
row = {
column: _cell(own[column] if column in own else getattr(run, column))
for column in RUN_COLUMNS
}
row.update((f"param.{k}", _cell(_dig(run.params, k))) for k in inputs)
row.update((f"metric.{k}", _cell(_dig(run.result, k))) for k in scores)
yield [row]
@@ -773,7 +828,7 @@ def read_metric_names(
@router.get("/{run_id}", response_model=RunDetail)
def read_run(run_id: str, session: SessionDep) -> Any:
def read_run(run_id: str, session: SessionDep, controller: FlowControllerDep) -> Any:
"""One run in full: what it was asked, what each node did, what it made."""
run = session.get(Run, run_id)
if run is None:
@@ -782,8 +837,16 @@ def read_run(run_id: str, session: SessionDep) -> Any:
artifacts = session.exec(
select(RunArtifact).where(col(RunArtifact.run_id) == run_id)
).all()
detail = RunDetail.model_validate(run, from_attributes=True)
detail.nodes = [RunNodeRow.model_validate(n, from_attributes=True) for n in nodes]
store = controller.store
detail = _named(RunDetail, [run], store)[0]
# Stored as `<flow id>.<node>`, shown as the engine names them.
shown = shown_nodes({n.node for n in nodes} | {a.node for a in artifacts}, store)
detail.nodes = [
RunNodeRow.model_validate(n, from_attributes=True).model_copy(
update={"node": shown[n.node]}
)
for n in nodes
]
# Worked out here, not stored: a source run is deleted long after the
# nodes restored from it were written.
sources = {n.cached_from for n in nodes if n.cached_from}
@@ -795,19 +858,29 @@ def read_run(run_id: str, session: SessionDep) -> Any:
for row in detail.nodes:
row.curve_lost = bool(row.cached_from) and row.cached_from not in kept
detail.artifacts = [
ArtifactRow.model_validate(a, from_attributes=True) for a in artifacts
ArtifactRow.model_validate(a, from_attributes=True).model_copy(
update={"node": shown[a.node]}
)
for a in artifacts
]
return detail
@router.get("/{run_id}/nodes/{node}/outputs", response_model=dict[str, Any] | None)
def read_node_outputs(run_id: str, node: str, session: SessionDep) -> Any:
def read_node_outputs(
run_id: str, node: str, session: SessionDep, controller: FlowControllerDep
) -> Any:
"""What one node returned, by message, as the stage cache stored it.
Kept only for a node that succeeded and may be reused, and only up to
256 kB of JSON. Null when the node published nothing.
The node is ``<key>.<node>``, as the run shows it. Kept only for a node
that succeeded and may be reused, and only up to 256 kB of JSON. Null when
the node published nothing.
"""
row = session.get(RunNode, {"run_id": run_id, "node": node})
run = session.get(Run, run_id)
if run is None:
raise HTTPException(status_code=404, detail="No such run")
stored = stored_node(node, controller.store, run.flow_id)
row = session.get(RunNode, {"run_id": run_id, "node": stored})
if row is None:
raise HTTPException(status_code=404, detail=f"No node '{node}' in this run")
if row.outputs is None:
@@ -823,7 +896,9 @@ def read_node_outputs(run_id: str, node: str, session: SessionDep) -> Any:
@router.post("/{run_id}/cancel", response_model=RunRow)
async def cancel_run(run_id: str, request: Request, session: SessionDep) -> Any:
async def cancel_run(
run_id: str, request: Request, session: SessionDep, controller: FlowControllerDep
) -> Any:
"""Stop a run. One already past its last node is left as it finished."""
run = session.get(Run, run_id)
if run is None:
@@ -831,21 +906,24 @@ async def cancel_run(run_id: str, request: Request, session: SessionDep) -> Any:
service = _service(request)
await run_in_threadpool(service.cancel, run_id)
session.refresh(run)
return run
return _named(RunRow, [run], controller.store)[0]
@router.post("/{run_id}/retry", response_model=RunRow, status_code=202)
async def retry_run(run_id: str, request: Request, user: CurrentUser) -> Any:
async def retry_run(
run_id: str, request: Request, user: CurrentUser, controller: FlowControllerDep
) -> Any:
"""Run the same thing again, as a new run pointing back at this one."""
service = _service(request)
try:
return await run_in_threadpool(service.retry, run_id, user.email)
run = await run_in_threadpool(service.retry, run_id, user.email)
except FlowNotFound as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except RunRejected as exc:
detail = str(exc)
status = 404 if detail.startswith("No run ") else 422
raise HTTPException(status_code=status, detail=detail) from exc
return _named(RunRow, [run], controller.store)[0]
def _drop_runs(session: Session, chosen: Any) -> None:
@@ -867,23 +945,38 @@ def _drop_runs(session: Session, chosen: Any) -> None:
session.execute(delete(Run).where(col(Run.id).in_(chosen)))
def _live_run(session: Session, flow: str) -> str | None:
def _live_run(session: Session, flow_id: str) -> str | None:
"""A run of this flow the driver is still writing to, if there is one."""
return session.exec(
select(Run.id).where(
col(Run.flow) == flow, col(Run.status).in_(("running", "queued"))
col(Run.flow_id) == flow_id, col(Run.status).in_(("running", "queued"))
)
).first()
def _audit(action: str, flow_id: str, user: CurrentUser, store: FlowStore) -> None:
"""Record who deleted runs. On the bus a flow goes by its key and id."""
event_bus.publish(
{
"type": "audit",
"action": action,
"flow": name_of(flow_id, store),
"flow_id": flow_id,
"user": user.email,
"ts": time.time(),
}
)
@router.delete("", response_model=Message)
def delete_runs(
session: SessionDep,
user: CurrentUser,
controller: FlowControllerDep,
flow: Annotated[str, Query(min_length=1)],
history: bool = False,
) -> Any:
"""Every run of one flow at once.
"""Every run of one flow at once, ``flow`` being its id.
The counterpart to the list's ``flow`` filter, and what a reseed needs:
deleting a flow leaves its runs standing on purpose, so this is the only
@@ -906,28 +999,22 @@ def delete_runs(
)
# Counted before the delete, and a subquery rather than a materialised
# list of ids on the way in: a demo can hold thousands.
chosen = select(col(Run.id)).where(col(Run.flow) == flow)
chosen = select(col(Run.id)).where(col(Run.flow_id) == flow)
gone = session.exec(
select(func.count()).select_from(Run).where(col(Run.flow) == flow)
select(func.count()).select_from(Run).where(col(Run.flow_id) == flow)
).one()
_drop_runs(session, chosen)
if history:
session.execute(delete(FlowRun).where(col(FlowRun.flow) == flow))
session.execute(delete(FlowRun).where(col(FlowRun.flow_id) == flow))
session.commit()
event_bus.publish(
{
"type": "audit",
"action": f"deleted {gone} run(s)",
"flow": flow,
"user": user.email,
"ts": time.time(),
}
)
_audit(f"deleted {gone} run(s)", flow, user, controller.store)
return Message(message=f"Deleted {gone} run(s) of '{flow}'")
@router.delete("/{run_id}", status_code=204)
def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response:
def delete_run(
run_id: str, session: SessionDep, user: CurrentUser, controller: FlowControllerDep
) -> Response:
"""Forget a run and everything hanging off it.
A live run is refused rather than raced: the driver writes its nodes back
@@ -949,29 +1036,24 @@ def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response:
"finish, before deleting it."
),
)
flow = run.flow
flow_id = run.flow_id
_drop_runs(session, [run_id])
session.commit()
event_bus.publish(
{
"type": "audit",
"action": f"deleted run {run_id}",
"flow": flow,
"user": user.email,
"ts": time.time(),
}
)
_audit(f"deleted run {run_id}", flow_id, user, controller.store)
return Response(status_code=204)
def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
def _series(
session: Session, store: FlowStore, run_id: str, name: str = ""
) -> list[RunMetric]:
"""A run's numbers, including the ones a cached node points at.
A cache hit replays no emissions, so a node restored from an earlier run has
no rows of its own — it carries that run's id instead, and its series is read
from there. Names are re-qualified on the way out, because the same node
reached through two flows publishes under two names and the caller asked for
this run's.
this run's. Nodes are found by flow id, which is how rows name them, and
names are moved between the two flows' keys, which is how they are named.
"""
statement = select(RunMetric).where(col(RunMetric.run_id) == run_id)
if name:
@@ -985,16 +1067,17 @@ def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
).all()
if restored:
run = session.get(Run, run_id)
flow = run.flow if run is not None else ""
flow = run.flow_id if run is not None else ""
# Two queries for the lot, rather than a `Run` lookup and a `RunMetric`
# query per restored node. A comparison of twenty runs calls this
# twenty times, so the per-node round trips multiplied.
sources = {
row.id: row.flow
row.id: row.flow_id
for row in session.exec(
select(Run).where(col(Run.id).in_({n.cached_from for n in restored}))
)
}
keys = names_of({flow, *sources.values()}, store)
wanted: dict[tuple[str, str], list[RunNode]] = {}
for node_row in restored:
source_flow = sources.get(node_row.cached_from)
@@ -1017,7 +1100,7 @@ def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
for row in cached:
for node_row in wanted.get((row.run_id, row.node), ()):
source_flow = sources[node_row.cached_from]
renamed = requalify(row.name, source_flow, flow)
renamed = requalify(row.name, keys[source_flow], keys[flow])
if name and renamed != name:
continue
rows.append(
@@ -1039,6 +1122,7 @@ def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
def read_metrics(
run_id: str,
session: SessionDep,
controller: FlowControllerDep,
name: str = "",
stride: Annotated[int, Query(ge=1, le=100_000)] = 1,
) -> Any:
@@ -1047,13 +1131,15 @@ def read_metrics(
``stride`` thins a long curve down: 3000 steps drawn on a 400-pixel chart
is 3000 points nobody can see.
"""
rows = _series(session, run_id, name)
rows = _series(session, controller.store, run_id, name)
if stride > 1:
rows = rows[:: max(1, stride)]
return rows
def _points(session: Session, run_id: str, metric: str, x: str) -> list[list[float]]:
def _points(
session: Session, store: FlowStore, run_id: str, metric: str, x: str
) -> list[list[float]]:
"""One run's readings of ``metric``, against whichever x was asked for.
The step is the default because it is what every run has. Time answers
@@ -1063,20 +1149,26 @@ def _points(session: Session, run_id: str, metric: str, x: str) -> list[list[flo
epoch, or samples seen — and is joined on the step the two share, which is
the only thing they have in common.
"""
rows = _series(session, run_id, metric)
rows = _series(session, store, run_id, metric)
if x == "time":
if not rows:
return []
start = min(row.ts for row in rows)
return [[row.ts - start, row.value] for row in rows]
if x and x != "step":
against = {row.step: row.value for row in _series(session, run_id, x)}
against = {row.step: row.value for row in _series(session, store, run_id, x)}
return [[against[row.step], row.value] for row in rows if row.step in against]
return [[float(row.step), row.value] for row in rows]
@router.get("/series/compare", response_model=SeriesAnswer)
def compare_metric(session: SessionDep, ids: str, metric: str, x: str = "") -> Any:
def compare_metric(
session: SessionDep,
controller: FlowControllerDep,
ids: str,
metric: str,
x: str = "",
) -> Any:
"""One metric across several runs, as the chart widget's series shape.
This is the comparison view: it answers in the same shape a flow answers a
@@ -1102,6 +1194,9 @@ def compare_metric(session: SessionDep, ids: str, metric: str, x: str = "") -> A
if run.seed is not None:
label = f"{run_id} (seed {run.seed})"
lines.append(
MetricSeries(label=label, points=_points(session, run_id, metric, x))
MetricSeries(
label=label,
points=_points(session, controller.store, run_id, metric, x),
)
)
return SeriesAnswer(metric=metric, x=x or "step", lines=lines)
+20 -8
View File
@@ -54,16 +54,19 @@ class SearchEntry(BaseModel):
"""
category: Category
#: The id the frontend routes on. For an input or output, the qualified
#: message, which is what the canvas names its label by. For a setting,
#: ``<node>.<setting>``: a node id has no dot, so the node is what comes
#: before the first one.
#: What it is called: a flow's or dashboard's key, or the id of anything
#: else. For an input or output, the qualified message, which is what the
#: canvas names its label by. For a setting, ``<node>.<setting>``: a node
#: id has no dot, so the node is what comes before the first one.
name: str
#: Human title, often empty — a flow is usually only its name.
title: str = ""
#: The flow a node, input or output sits in, or the dashboard a widget
#: sits on.
#: sits on, by key.
parent: str = ""
#: The id of the flow or dashboard opening it goes to: its own, or the one
#: it sits in. Empty for everything else, whose ``name`` is its id.
id: str = ""
#: Node type, input dtype, widget type, channel kind.
kind: str = ""
#: What a setting is set to, as text. Only ever what ``setting_text`` lets
@@ -159,13 +162,16 @@ def _build(
flow = controller.store.read_flow(name, draft=True)
except FlowNotFound:
continue
entries.append(SearchEntry(category="flow", name=flow.name, title=flow.title))
entries.append(
SearchEntry(category="flow", name=flow.name, title=flow.title, id=flow.id)
)
entries.extend(
SearchEntry(
category="node",
name=node.id,
title=node.title,
parent=flow.name,
id=flow.id,
kind=node.type,
)
for node in flow.nodes
@@ -184,6 +190,7 @@ def _build(
category="setting",
name=f"{node.id}.{key}",
parent=flow.name,
id=flow.id,
kind=node.type,
detail=text,
)
@@ -206,18 +213,22 @@ def _build(
# The label the canvas draws: bare when it is the flow's own.
title=message.removeprefix(f"{flow.name}."),
parent=flow.name,
id=flow.id,
kind=kind,
)
)
for summary in dashboards.list():
try:
dashboard = dashboards.read(summary.name, draft=True)
dashboard = dashboards.read(summary.id, draft=True)
except DashboardNotFound:
continue
entries.append(
SearchEntry(
category="dashboard", name=dashboard.name, title=dashboard.title
category="dashboard",
name=dashboard.name,
title=dashboard.title,
id=dashboard.id,
)
)
entries.extend(
@@ -226,6 +237,7 @@ def _build(
name=widget.id,
title=widget.title,
parent=dashboard.name,
id=dashboard.id,
kind=widget.type,
)
for widget in dashboard.widgets