History is kept by flow id: runs, cascades, rollups and events file the id in their flow column and nodes as <flow id>.<node>, the API answers flow plus flow_name and shows and takes nodes as <key>.<node>; events carry flow_id from the moment they are published; run queue items, the stage cache and cached curves keep ids and keys apart; endpoints, the brain graph, device bindings and search name the id to open
This commit is contained in:
@@ -110,19 +110,20 @@ async def delete_flavor(
|
||||
|
||||
|
||||
def _usages(controller: FlowControllerDep, name: str) -> list[str]:
|
||||
"""Which nodes name this flavor, published or in a draft."""
|
||||
"""Which nodes name this flavor, published or in a draft, as
|
||||
``<key>.<node>``."""
|
||||
used: list[str] = []
|
||||
for flow_name in controller.store.list_flows():
|
||||
for flow_id in controller.store.list_flows():
|
||||
seen = set()
|
||||
for draft in (False, True):
|
||||
if draft and not controller.store.has_draft(flow_name):
|
||||
if draft and not controller.store.has_draft(flow_id):
|
||||
continue
|
||||
try:
|
||||
flow = controller.store.read_flow(flow_name, draft=draft)
|
||||
flow = controller.store.read_flow(flow_id, draft=draft)
|
||||
except Exception:
|
||||
continue
|
||||
for node in flow.nodes:
|
||||
if node.resources is not None and node.resources.flavor == name:
|
||||
seen.add(f"{flow_name}.{node.id}")
|
||||
seen.add(f"{flow.name}.{node.id}")
|
||||
used.extend(sorted(seen))
|
||||
return used
|
||||
|
||||
@@ -119,6 +119,9 @@ class Endpoint(BaseModel):
|
||||
kind: str
|
||||
#: Stable within its kind, and used as the canvas node id.
|
||||
id: str
|
||||
#: The id of the dashboard or flow it sits in, which is what opening it
|
||||
#: goes by. Never read it off ``id``, whose flow part is a key.
|
||||
owner_id: str = ""
|
||||
label: str
|
||||
#: What sort of widget, or the node type in the other flow.
|
||||
detail: str = ""
|
||||
@@ -194,24 +197,31 @@ def _endpoints(controller: FlowController, definition: FlowDef) -> list[Endpoint
|
||||
found[f"dashboard:{binding['dashboard']}:{binding['widget']}"] = Endpoint(
|
||||
kind="dashboard",
|
||||
id=f"dashboard:{binding['dashboard']}:{binding['widget']}",
|
||||
owner_id=binding["dashboard"],
|
||||
label=binding["title"],
|
||||
detail=binding["type"],
|
||||
provides=[binding["provides"]] if binding["provides"] else [],
|
||||
requires=binding["requires"],
|
||||
)
|
||||
|
||||
def _flow_endpoint(key: str, detail: str) -> Endpoint:
|
||||
def _flow_endpoint(key: str, detail: str, owner_id: str = "") -> Endpoint:
|
||||
"""One label per node on the far side, however many names reach it."""
|
||||
return found.setdefault(
|
||||
f"flow:{key}",
|
||||
Endpoint(kind="flow", id=f"flow:{key}", label=key, detail=detail),
|
||||
Endpoint(
|
||||
kind="flow",
|
||||
id=f"flow:{key}",
|
||||
owner_id=owner_id,
|
||||
label=key,
|
||||
detail=detail,
|
||||
),
|
||||
)
|
||||
|
||||
# An outsider reaching into this flow.
|
||||
for other, node_id, provides, requires in controller.cross_flow_nodes(flow):
|
||||
endpoint = _flow_endpoint(f"{other}.{node_id}", "flow")
|
||||
endpoint.provides += provides
|
||||
endpoint.requires += requires
|
||||
for other in controller.cross_flow_nodes(flow):
|
||||
endpoint = _flow_endpoint(f"{other.flow}.{other.node}", "flow", other.flow_id)
|
||||
endpoint.provides += other.provides
|
||||
endpoint.requires += other.requires
|
||||
|
||||
# And this flow reaching out: its own ports bound to a message of another
|
||||
# flow. Read from the working document rather than from the store, so a
|
||||
@@ -235,6 +245,7 @@ def _endpoints(controller: FlowController, definition: FlowDef) -> list[Endpoint
|
||||
endpoint = _flow_endpoint(
|
||||
f"{far.flow}.{far.node}" if far else message,
|
||||
far.type if far else "flow",
|
||||
far.flow_id if far else "",
|
||||
)
|
||||
# An endpoint publishing into this flow is what this flow reads.
|
||||
side = endpoint.requires if ours_publishes else endpoint.provides
|
||||
@@ -289,14 +300,15 @@ def _read_flow(controller: FlowController, flow_id: str) -> FlowDef:
|
||||
raise _missing(flow_id)
|
||||
|
||||
|
||||
def _audit(action: str, flow: str, user: CurrentUser) -> None:
|
||||
"""Record who changed what, by the flow's key. The collector writes it
|
||||
down; the bus carries it."""
|
||||
def _audit(action: str, key: str, flow_id: str, user: CurrentUser) -> None:
|
||||
"""Record who changed what. The collector writes it down under the flow's
|
||||
id; the bus carries the key with it, as it does for every event."""
|
||||
event_bus.publish(
|
||||
{
|
||||
"type": "audit",
|
||||
"action": action,
|
||||
"flow": flow,
|
||||
"flow": key,
|
||||
"flow_id": flow_id,
|
||||
"user": user.email,
|
||||
"ts": time.time(),
|
||||
}
|
||||
@@ -415,13 +427,12 @@ async def delete_shared_node(
|
||||
first, so the source can be retired while flows still use it.
|
||||
"""
|
||||
if detach:
|
||||
running = await run_in_threadpool(controller.store.usages, lib_name, False)
|
||||
running = await run_in_threadpool(controller.store.lib_users, lib_name)
|
||||
try:
|
||||
await run_in_threadpool(controller.store.detach_lib, lib_name)
|
||||
except LibNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No shared node '{lib_name}'")
|
||||
# `usages` qualifies with the flow's id.
|
||||
for flow_id in sorted({usage.split(".", 1)[0] for usage in running}):
|
||||
for flow_id in running:
|
||||
await controller.reload_flow(flow_id)
|
||||
return Message(message=f"Deleted shared node '{lib_name}'")
|
||||
# The published flows too: a draft that unshared a node still runs it.
|
||||
@@ -643,7 +654,7 @@ async def publish_flow(
|
||||
status_code=409,
|
||||
detail={"message": str(exc), "current_version": exc.current},
|
||||
)
|
||||
_audit("published", definition.name, user)
|
||||
_audit("published", definition.name, flow_id, user)
|
||||
await controller.reload_flow(flow_id)
|
||||
return await run_in_threadpool(_detail, controller, published)
|
||||
|
||||
@@ -680,7 +691,7 @@ async def delete_flow(
|
||||
"""
|
||||
live = session.exec(
|
||||
select(Run.id).where(
|
||||
col(Run.flow) == flow_id, col(Run.status).in_(("running", "queued"))
|
||||
col(Run.flow_id) == flow_id, col(Run.status).in_(("running", "queued"))
|
||||
)
|
||||
).first()
|
||||
if live is not None:
|
||||
@@ -695,7 +706,7 @@ async def delete_flow(
|
||||
key = await run_in_threadpool(controller.store.delete_flow, flow_id)
|
||||
except FlowNotFound:
|
||||
raise _missing(flow_id)
|
||||
_audit("deleted", key, user)
|
||||
_audit("deleted", key, flow_id, user)
|
||||
# Its nodes first, or they would put values and work straight back.
|
||||
# Its runs stay: a run record is a record of what ran, and surviving the
|
||||
# flow it belonged to is the point of keeping one. `DELETE /runs?flow=`
|
||||
@@ -824,7 +835,7 @@ async def save_node_source(
|
||||
controller.store.write_lib_source, ref, source.code
|
||||
)
|
||||
if changed:
|
||||
_audit(f"edited shared node '{ref}'", name, user)
|
||||
_audit(f"edited shared node '{ref}'", name, flow_id, user)
|
||||
await controller.reload_lib_users(ref)
|
||||
else:
|
||||
await run_in_threadpool(
|
||||
@@ -905,7 +916,7 @@ async def start_flow(
|
||||
"""Let the engine run this flow again."""
|
||||
definition = _read_flow(controller, flow_id)
|
||||
await controller.set_enabled(flow_id, definition.name, True)
|
||||
_audit("started", definition.name, user)
|
||||
_audit("started", definition.name, flow_id, user)
|
||||
return await run_in_threadpool(_detail, controller, definition)
|
||||
|
||||
|
||||
@@ -916,7 +927,7 @@ async def stop_flow(
|
||||
"""Take this flow off the engine: no subscriptions, schedules or webhooks."""
|
||||
definition = _read_flow(controller, flow_id)
|
||||
await controller.set_enabled(flow_id, definition.name, False)
|
||||
_audit("stopped", definition.name, user)
|
||||
_audit("stopped", definition.name, flow_id, user)
|
||||
return await run_in_threadpool(_detail, controller, definition)
|
||||
|
||||
|
||||
@@ -1123,9 +1134,14 @@ async def train_node(
|
||||
if service is None:
|
||||
raise HTTPException(status_code=503, detail="Runs are not available")
|
||||
try:
|
||||
# Registered by `fluksio sync`, which gives it whatever id it gets: the
|
||||
# built-in flow is known by its key.
|
||||
train_id = controller.store.key_index().get(TRAIN_FLOW)
|
||||
if train_id is None:
|
||||
raise FlowNotFound(TRAIN_FLOW)
|
||||
run = await run_in_threadpool(
|
||||
service.submit,
|
||||
TRAIN_FLOW,
|
||||
train_id,
|
||||
params={"installation": orjson.dumps(wren.installation()).decode()},
|
||||
cause="train",
|
||||
actor=user.email,
|
||||
@@ -1153,11 +1169,15 @@ def read_training(
|
||||
controller: FlowControllerDep,
|
||||
session: SessionDep,
|
||||
) -> Any:
|
||||
"""A wren node's latest training run, or null when it has none."""
|
||||
"""A wren node's latest training run, or null when it has none.
|
||||
|
||||
Found by the node it trains alone: only a training run names one, and
|
||||
the training flow may have been registered again under another id since.
|
||||
"""
|
||||
_node_def(controller, flow_id, node_id)
|
||||
run = session.exec(
|
||||
select(Run)
|
||||
.where(col(Run.flow) == TRAIN_FLOW, col(Run.for_node) == f"{flow_id}.{node_id}")
|
||||
.where(col(Run.for_node) == f"{flow_id}.{node_id}")
|
||||
.order_by(col(Run.created_at).desc())
|
||||
).first()
|
||||
if run is None:
|
||||
|
||||
@@ -28,6 +28,7 @@ from fluksio.api.deps import (
|
||||
from fluksio.api.routes.runs import elapsed_ms
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.flow.controller import NodeStatus
|
||||
from fluksio.flow.tombstones import names_of, shown_nodes, stored_node
|
||||
from fluksio.models import EngineEvent, FlowRun, Message, MetricBucket
|
||||
|
||||
router = APIRouter(
|
||||
@@ -77,7 +78,9 @@ class SeriesPoint(BaseModel):
|
||||
|
||||
|
||||
class FlowRollup(BaseModel):
|
||||
#: The flow's id, and its key: what it is called now, or was last called.
|
||||
flow: str
|
||||
flow_name: str = ""
|
||||
executions: int
|
||||
errors: int
|
||||
messages: int
|
||||
@@ -100,7 +103,9 @@ class NodeTypeTiming(BaseModel):
|
||||
|
||||
class RunRow(BaseModel):
|
||||
id: str
|
||||
#: The flow's id, and its key as `FlowRollup` has it.
|
||||
flow: str
|
||||
flow_name: str = ""
|
||||
source: str
|
||||
#: The run this one continues: for a flush, the run whose value it released.
|
||||
parent: str = ""
|
||||
@@ -130,7 +135,11 @@ class EventRow(BaseModel):
|
||||
id: int
|
||||
ts: datetime
|
||||
type: str
|
||||
#: The flow's id, and its key as `FlowRollup` has it. Both empty for the
|
||||
#: engine as a whole.
|
||||
flow: str
|
||||
flow_name: str = ""
|
||||
#: ``<key>.<node>``, or empty.
|
||||
node: str
|
||||
detail: str
|
||||
actor: str
|
||||
@@ -138,6 +147,8 @@ class EventRow(BaseModel):
|
||||
|
||||
|
||||
class DeadLetter(BaseModel):
|
||||
"""A work item given up on, named as the engine runs it: by key."""
|
||||
|
||||
id: str
|
||||
ts: float
|
||||
flow: str
|
||||
@@ -317,6 +328,7 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
|
||||
@router.get("/timeseries", response_model=list[SeriesPoint])
|
||||
def read_timeseries(
|
||||
session: SessionDep,
|
||||
controller: FlowControllerDep,
|
||||
flow: str | None = None,
|
||||
node: str | None = None,
|
||||
hours: Annotated[int, Query(ge=0, le=8760)] = 24,
|
||||
@@ -331,6 +343,8 @@ def read_timeseries(
|
||||
the same pair ``/runs`` and ``/events`` take — so a chart dragged to a
|
||||
span can be re-fetched at that span's own resolution rather than
|
||||
magnifying the buckets it already has.
|
||||
|
||||
``flow`` is a flow id, ``node`` a ``<key>.<node>`` as the engine names it.
|
||||
"""
|
||||
hours = _window_hours(hours)
|
||||
# The database does the fold: a week of minute rows per node used to cross
|
||||
@@ -356,9 +370,10 @@ def read_timeseries(
|
||||
col(MetricBucket.bucket) < (_aware(until) if until else _closed_before(stride)),
|
||||
)
|
||||
if flow:
|
||||
statement = statement.where(col(MetricBucket.flow) == flow)
|
||||
statement = statement.where(col(MetricBucket.flow_id) == flow)
|
||||
if node:
|
||||
statement = statement.where(col(MetricBucket.node) == node)
|
||||
stored = stored_node(node, controller.store, flow or "")
|
||||
statement = statement.where(col(MetricBucket.node) == stored)
|
||||
|
||||
return [
|
||||
SeriesPoint(
|
||||
@@ -376,7 +391,9 @@ def read_timeseries(
|
||||
|
||||
@router.get("/flows", response_model=list[FlowRollup])
|
||||
def read_flow_rollups(
|
||||
session: SessionDep, hours: Annotated[int, Query(ge=0, le=8760)] = 24
|
||||
session: SessionDep,
|
||||
controller: FlowControllerDep,
|
||||
hours: Annotated[int, Query(ge=0, le=8760)] = 24,
|
||||
) -> Any:
|
||||
"""One row per flow, with a coarse trend of how much it ran."""
|
||||
hours = _window_hours(hours)
|
||||
@@ -399,7 +416,7 @@ def read_flow_rollups(
|
||||
).label("slot")
|
||||
statement = (
|
||||
sa_select(
|
||||
col(MetricBucket.flow),
|
||||
col(MetricBucket.flow_id),
|
||||
slot,
|
||||
func.sum(col(MetricBucket.executions)).label("executions"),
|
||||
func.sum(col(MetricBucket.errors)).label("errors"),
|
||||
@@ -412,13 +429,13 @@ def read_flow_rollups(
|
||||
col(MetricBucket.bucket) >= since,
|
||||
col(MetricBucket.bucket) < datetime.fromtimestamp(float(end), UTC),
|
||||
)
|
||||
.group_by(col(MetricBucket.flow), slot)
|
||||
.group_by(col(MetricBucket.flow_id), slot)
|
||||
)
|
||||
|
||||
rollups: dict[str, dict[str, Any]] = {}
|
||||
for row in session.execute(statement):
|
||||
entry = rollups.setdefault(
|
||||
row.flow,
|
||||
row.flow_id,
|
||||
{
|
||||
"executions": 0,
|
||||
"errors": 0,
|
||||
@@ -448,15 +465,17 @@ def read_flow_rollups(
|
||||
# mapping and subscript it.
|
||||
last_errors = dict(
|
||||
session.exec(
|
||||
select(col(EngineEvent.flow), func.max(col(EngineEvent.ts)))
|
||||
select(col(EngineEvent.flow_id), func.max(col(EngineEvent.ts)))
|
||||
.where(col(EngineEvent.type) == "node_error", col(EngineEvent.ts) >= since)
|
||||
.group_by(col(EngineEvent.flow))
|
||||
.group_by(col(EngineEvent.flow_id))
|
||||
).all()
|
||||
)
|
||||
|
||||
names = names_of(rollups, controller.store)
|
||||
return [
|
||||
FlowRollup(
|
||||
flow=flow,
|
||||
flow_name=names[flow],
|
||||
executions=entry["executions"],
|
||||
errors=entry["errors"],
|
||||
messages=entry["messages"],
|
||||
@@ -483,8 +502,9 @@ def read_node_type_timings(
|
||||
flows. A node deleted since it ran has no type left to count under.
|
||||
"""
|
||||
hours = _window_hours(hours)
|
||||
# By `<flow id>.<node>`, the way the rollups name a node.
|
||||
types = {
|
||||
f"{flow.name}.{node.id}": node.type
|
||||
f"{flow.id}.{node.id}": node.type
|
||||
for flow in controller.store.read_all()
|
||||
for node in flow.nodes
|
||||
}
|
||||
@@ -526,6 +546,7 @@ def read_node_type_timings(
|
||||
@router.get("/runs", response_model=RunPage)
|
||||
def read_runs(
|
||||
session: SessionDep,
|
||||
controller: FlowControllerDep,
|
||||
flow: str | None = None,
|
||||
status: str | None = None,
|
||||
since: datetime | None = None,
|
||||
@@ -536,10 +557,11 @@ def read_runs(
|
||||
|
||||
``since`` is inclusive and ``until`` exclusive, so a window of one minute
|
||||
holds exactly the runs of the minute bucket the charts are drawn from.
|
||||
``flow`` is a flow id.
|
||||
"""
|
||||
filters: list[ColumnElement[bool]] = []
|
||||
if flow:
|
||||
filters.append(col(FlowRun.flow) == flow)
|
||||
filters.append(col(FlowRun.flow_id) == flow)
|
||||
if status:
|
||||
filters.append(col(FlowRun.status) == status)
|
||||
if since:
|
||||
@@ -563,12 +585,20 @@ def read_runs(
|
||||
).one()
|
||||
)
|
||||
)
|
||||
return {"data": rows, "count": count}
|
||||
names = names_of({row.flow_id for row in rows}, controller.store)
|
||||
data = [
|
||||
RunRow.model_validate(
|
||||
{**row.model_dump(), "flow": row.flow_id, "flow_name": names[row.flow_id]}
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
return {"data": data, "count": count}
|
||||
|
||||
|
||||
@router.get("/events", response_model=list[EventRow])
|
||||
def read_events(
|
||||
session: SessionDep,
|
||||
controller: FlowControllerDep,
|
||||
kind: Literal["failure", "audit"] = "failure",
|
||||
flow: str | None = None,
|
||||
run: str | None = None,
|
||||
@@ -582,7 +612,8 @@ def read_events(
|
||||
takes, so a list can cover the span the charts beside it are drawn from.
|
||||
``run`` narrows to one run — a batch run's id, or the journaled item a live
|
||||
cascade came from. Rows recorded before the column existed carry none, so
|
||||
an old failure answers no run at all rather than the wrong one.
|
||||
an old failure answers no run at all rather than the wrong one. ``flow``
|
||||
is a flow id.
|
||||
"""
|
||||
statement = select(EngineEvent).order_by(col(EngineEvent.ts).desc())
|
||||
if kind == "audit":
|
||||
@@ -590,14 +621,28 @@ def read_events(
|
||||
else:
|
||||
statement = statement.where(col(EngineEvent.type) != "audit")
|
||||
if flow:
|
||||
statement = statement.where(col(EngineEvent.flow) == flow)
|
||||
statement = statement.where(col(EngineEvent.flow_id) == flow)
|
||||
if run:
|
||||
statement = statement.where(col(EngineEvent.run) == run)
|
||||
if since:
|
||||
statement = statement.where(col(EngineEvent.ts) >= _aware(since))
|
||||
if until:
|
||||
statement = statement.where(col(EngineEvent.ts) < _aware(until))
|
||||
return list(session.exec(statement.limit(min(limit, 500))))
|
||||
rows = list(session.exec(statement.limit(min(limit, 500))))
|
||||
store = controller.store
|
||||
names = names_of({row.flow_id for row in rows if row.flow_id}, store)
|
||||
nodes = shown_nodes({row.node for row in rows if row.node}, store)
|
||||
return [
|
||||
EventRow.model_validate(
|
||||
{
|
||||
**row.model_dump(),
|
||||
"flow": row.flow_id,
|
||||
"flow_name": names.get(row.flow_id, ""),
|
||||
"node": nodes.get(row.node, ""),
|
||||
}
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
@router.get("/dead-letter", response_model=list[DeadLetter])
|
||||
|
||||
@@ -22,13 +22,19 @@ from sqlalchemy import delete, func
|
||||
from sqlalchemy import select as sa_select
|
||||
from sqlmodel import Session, col, select
|
||||
|
||||
from fluksio.api.deps import CurrentUser, SessionDep, get_current_user
|
||||
from fluksio.api.deps import (
|
||||
CurrentUser,
|
||||
FlowControllerDep,
|
||||
SessionDep,
|
||||
get_current_user,
|
||||
)
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.core.db import writing
|
||||
from fluksio.flow.events import event_bus
|
||||
from fluksio.flow.messages import requalify
|
||||
from fluksio.flow.runs import OUTPUT_CAP, RunRejected, RunService, new_run_id
|
||||
from fluksio.flow.store import FlowNotFound
|
||||
from fluksio.flow.store import FlowNotFound, FlowStore
|
||||
from fluksio.flow.tombstones import name_of, names_of, shown_nodes, stored_node
|
||||
from fluksio.models import FlowRun, Message, Run, RunArtifact, RunMetric, RunNode
|
||||
|
||||
router = APIRouter(
|
||||
@@ -105,6 +111,7 @@ class ImportArtifact(BaseModel):
|
||||
class ImportEntry(BaseModel):
|
||||
"""One run executed somewhere else, as it ended there."""
|
||||
|
||||
#: The flow's id.
|
||||
flow: str
|
||||
#: What it was called where it ran. Importing it again is a no-op.
|
||||
external_id: str = Field(min_length=1, max_length=255)
|
||||
@@ -133,6 +140,7 @@ class Imported(BaseModel):
|
||||
|
||||
|
||||
class RunNodeRow(BaseModel):
|
||||
#: ``<key>.<node>``, the way the engine names it.
|
||||
node: str
|
||||
status: str
|
||||
attempt: int
|
||||
@@ -153,6 +161,7 @@ class RunNodeRow(BaseModel):
|
||||
|
||||
class ArtifactRow(BaseModel):
|
||||
name: str
|
||||
#: ``<key>.<node>``.
|
||||
node: str
|
||||
digest: str
|
||||
size: int
|
||||
@@ -167,7 +176,11 @@ class RunRow(BaseModel):
|
||||
"""A run without its result, which is the part that can be large."""
|
||||
|
||||
id: str
|
||||
#: The flow's id, which the history is kept under.
|
||||
flow: str
|
||||
#: Its key: what it is called now, or what it was last called once it is
|
||||
#: deleted. The id where nothing remembers either.
|
||||
flow_name: str = ""
|
||||
status: str
|
||||
status_reason: str
|
||||
cause: str
|
||||
@@ -224,7 +237,9 @@ class RunDetail(RunRow):
|
||||
class FlowRunsRow(BaseModel):
|
||||
"""How much a flow has been run, for the screen's list of flows."""
|
||||
|
||||
#: The flow's id, and its key as `RunRow` has it.
|
||||
flow: str
|
||||
flow_name: str = ""
|
||||
runs: int
|
||||
running: int
|
||||
queued: int
|
||||
@@ -260,16 +275,36 @@ def _service(request: Request) -> RunService:
|
||||
return service
|
||||
|
||||
|
||||
@router.post("/flows/{name}", response_model=RunRow, status_code=202)
|
||||
def _named[Row: RunRow](
|
||||
model: type[Row], runs: list[Run], store: FlowStore
|
||||
) -> list[Row]:
|
||||
"""Runs as rows, ``flow`` their flow's id and ``flow_name`` its key.
|
||||
|
||||
One lookup for the lot, however many flows the page holds.
|
||||
"""
|
||||
names = names_of({run.flow_id for run in runs}, store)
|
||||
return [
|
||||
model.model_validate(
|
||||
{**run.model_dump(), "flow": run.flow_id, "flow_name": names[run.flow_id]}
|
||||
)
|
||||
for run in runs
|
||||
]
|
||||
|
||||
|
||||
@router.post("/flows/{flow_id}", response_model=RunRow, status_code=202)
|
||||
async def create_run(
|
||||
name: str, body: RunCreate, request: Request, user: CurrentUser
|
||||
flow_id: str,
|
||||
body: RunCreate,
|
||||
request: Request,
|
||||
user: CurrentUser,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Queue one run of a flow."""
|
||||
service = _service(request)
|
||||
try:
|
||||
return await run_in_threadpool(
|
||||
run = await run_in_threadpool(
|
||||
service.submit,
|
||||
name,
|
||||
flow_id,
|
||||
params=body.params,
|
||||
seed=body.seed,
|
||||
cause=body.cause,
|
||||
@@ -282,11 +317,16 @@ async def create_run(
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except RunRejected as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
return _named(RunRow, [run], controller.store)[0]
|
||||
|
||||
|
||||
@router.post("/flows/{name}/sweep", response_model=list[RunRow], status_code=202)
|
||||
@router.post("/flows/{flow_id}/sweep", response_model=list[RunRow], status_code=202)
|
||||
async def create_sweep(
|
||||
name: str, body: SweepCreate, request: Request, user: CurrentUser
|
||||
flow_id: str,
|
||||
body: SweepCreate,
|
||||
request: Request,
|
||||
user: CurrentUser,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Queue many runs of one flow under a shared group.
|
||||
|
||||
@@ -306,7 +346,7 @@ async def create_sweep(
|
||||
def submit_all() -> list[Run]:
|
||||
return [
|
||||
service.submit(
|
||||
name,
|
||||
flow_id,
|
||||
params=entry.params,
|
||||
seed=entry.seed,
|
||||
group_id=group,
|
||||
@@ -320,11 +360,12 @@ async def create_sweep(
|
||||
]
|
||||
|
||||
try:
|
||||
return await run_in_threadpool(submit_all)
|
||||
runs = await run_in_threadpool(submit_all)
|
||||
except FlowNotFound as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except RunRejected as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
return _named(RunRow, runs, controller.store)
|
||||
|
||||
|
||||
@router.post("/import", response_model=list[Imported], status_code=202)
|
||||
@@ -369,6 +410,7 @@ async def import_runs(body: ImportCreate, request: Request, user: CurrentUser) -
|
||||
@router.get("", response_model=list[RunRow])
|
||||
def read_runs(
|
||||
session: SessionDep,
|
||||
controller: FlowControllerDep,
|
||||
flow: str | None = None,
|
||||
status: str | None = None,
|
||||
group: str | None = None,
|
||||
@@ -384,11 +426,11 @@ def read_runs(
|
||||
first, so handing back the last row's ``created_at`` reads the next page
|
||||
whatever landed meanwhile — which ``offset`` cannot, since a run submitted
|
||||
between two pages shifts every row down one. ``since`` bounds the other
|
||||
end and is inclusive.
|
||||
end and is inclusive. ``flow`` is a flow id.
|
||||
"""
|
||||
statement = select(Run).order_by(col(Run.created_at).desc())
|
||||
if flow:
|
||||
statement = statement.where(col(Run.flow) == flow)
|
||||
statement = statement.where(col(Run.flow_id) == flow)
|
||||
if status:
|
||||
statement = statement.where(col(Run.status) == status)
|
||||
if group:
|
||||
@@ -400,11 +442,11 @@ def read_runs(
|
||||
if before:
|
||||
statement = statement.where(col(Run.created_at) < _aware(before))
|
||||
statement = statement.offset(max(0, offset)).limit(min(limit, 500))
|
||||
return list(session.exec(statement))
|
||||
return _named(RunRow, list(session.exec(statement)), controller.store)
|
||||
|
||||
|
||||
@router.get("/overview", response_model=list[FlowRunsRow])
|
||||
def read_overview(session: SessionDep) -> Any:
|
||||
def read_overview(session: SessionDep, controller: FlowControllerDep) -> Any:
|
||||
"""One row per flow that has ever run, busiest-recent first.
|
||||
|
||||
The list caps at 500 newest runs, so counting flows on the client goes
|
||||
@@ -416,11 +458,11 @@ def read_overview(session: SessionDep) -> Any:
|
||||
# rather than making it cheaper to find. A rollup is the answer if the
|
||||
# scan ever measures.
|
||||
statement = sa_select(
|
||||
col(Run.flow),
|
||||
col(Run.flow_id),
|
||||
col(Run.status),
|
||||
func.count(col(Run.id)),
|
||||
func.max(col(Run.created_at)),
|
||||
).group_by(col(Run.flow), col(Run.status))
|
||||
).group_by(col(Run.flow_id), col(Run.status))
|
||||
|
||||
rows: dict[str, FlowRunsRow] = {}
|
||||
for flow, status, count, latest in session.execute(statement):
|
||||
@@ -432,6 +474,9 @@ def read_overview(session: SessionDep) -> Any:
|
||||
row.queued += count
|
||||
if row.last_created_at is None or latest > row.last_created_at:
|
||||
row.last_created_at = latest
|
||||
names = names_of(rows, controller.store)
|
||||
for flow_id, row in rows.items():
|
||||
row.flow_name = names[flow_id]
|
||||
return sorted(rows.values(), key=lambda row: row.last_created_at, reverse=True)
|
||||
|
||||
|
||||
@@ -463,7 +508,9 @@ EXPORT_CAP = 10_000
|
||||
|
||||
RUN_COLUMNS = (
|
||||
"id",
|
||||
#: The flow's id, and its key as `RunRow` has it.
|
||||
"flow",
|
||||
"flow_name",
|
||||
"status",
|
||||
"created_at",
|
||||
"started_at",
|
||||
@@ -489,14 +536,14 @@ def _selection(
|
||||
since: datetime | None,
|
||||
until: datetime | None,
|
||||
) -> Any:
|
||||
"""The query behind a selection of runs, newest first.
|
||||
"""The query behind a selection of runs, newest first. ``flow`` is an id.
|
||||
|
||||
Separate from reading it, because the names route asks the same question
|
||||
as a subquery rather than for the rows.
|
||||
"""
|
||||
statement = select(Run).order_by(col(Run.created_at).desc())
|
||||
if flow:
|
||||
statement = statement.where(col(Run.flow) == flow)
|
||||
statement = statement.where(col(Run.flow_id) == flow)
|
||||
if status:
|
||||
statement = statement.where(col(Run.status) == status)
|
||||
if group:
|
||||
@@ -655,6 +702,7 @@ def _stream(
|
||||
@router.get("/export/metrics")
|
||||
def export_metrics(
|
||||
session: SessionDep,
|
||||
controller: FlowControllerDep,
|
||||
flow: str | None = None,
|
||||
status: str | None = None,
|
||||
group: str | None = None,
|
||||
@@ -679,7 +727,7 @@ def export_metrics(
|
||||
for run in runs:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for series, points in groupby(
|
||||
_series(session, run.id), key=lambda row: row.name
|
||||
_series(session, controller.store, run.id), key=lambda row: row.name
|
||||
):
|
||||
if wanted and series not in wanted:
|
||||
continue
|
||||
@@ -701,6 +749,7 @@ def export_metrics(
|
||||
@router.get("/export/runs")
|
||||
def export_runs(
|
||||
session: SessionDep,
|
||||
controller: FlowControllerDep,
|
||||
flow: str | None = None,
|
||||
status: str | None = None,
|
||||
group: str | None = None,
|
||||
@@ -734,9 +783,15 @@ def export_runs(
|
||||
*(f"metric.{key}" for key in scores),
|
||||
]
|
||||
|
||||
names = names_of({run.flow_id for run in runs}, controller.store)
|
||||
|
||||
def chunks() -> Iterator[list[dict[str, Any]]]:
|
||||
for run in runs:
|
||||
row = {column: _cell(getattr(run, column)) for column in RUN_COLUMNS}
|
||||
own = {"flow": run.flow_id, "flow_name": names[run.flow_id]}
|
||||
row = {
|
||||
column: _cell(own[column] if column in own else getattr(run, column))
|
||||
for column in RUN_COLUMNS
|
||||
}
|
||||
row.update((f"param.{k}", _cell(_dig(run.params, k))) for k in inputs)
|
||||
row.update((f"metric.{k}", _cell(_dig(run.result, k))) for k in scores)
|
||||
yield [row]
|
||||
@@ -773,7 +828,7 @@ def read_metric_names(
|
||||
|
||||
|
||||
@router.get("/{run_id}", response_model=RunDetail)
|
||||
def read_run(run_id: str, session: SessionDep) -> Any:
|
||||
def read_run(run_id: str, session: SessionDep, controller: FlowControllerDep) -> Any:
|
||||
"""One run in full: what it was asked, what each node did, what it made."""
|
||||
run = session.get(Run, run_id)
|
||||
if run is None:
|
||||
@@ -782,8 +837,16 @@ def read_run(run_id: str, session: SessionDep) -> Any:
|
||||
artifacts = session.exec(
|
||||
select(RunArtifact).where(col(RunArtifact.run_id) == run_id)
|
||||
).all()
|
||||
detail = RunDetail.model_validate(run, from_attributes=True)
|
||||
detail.nodes = [RunNodeRow.model_validate(n, from_attributes=True) for n in nodes]
|
||||
store = controller.store
|
||||
detail = _named(RunDetail, [run], store)[0]
|
||||
# Stored as `<flow id>.<node>`, shown as the engine names them.
|
||||
shown = shown_nodes({n.node for n in nodes} | {a.node for a in artifacts}, store)
|
||||
detail.nodes = [
|
||||
RunNodeRow.model_validate(n, from_attributes=True).model_copy(
|
||||
update={"node": shown[n.node]}
|
||||
)
|
||||
for n in nodes
|
||||
]
|
||||
# Worked out here, not stored: a source run is deleted long after the
|
||||
# nodes restored from it were written.
|
||||
sources = {n.cached_from for n in nodes if n.cached_from}
|
||||
@@ -795,19 +858,29 @@ def read_run(run_id: str, session: SessionDep) -> Any:
|
||||
for row in detail.nodes:
|
||||
row.curve_lost = bool(row.cached_from) and row.cached_from not in kept
|
||||
detail.artifacts = [
|
||||
ArtifactRow.model_validate(a, from_attributes=True) for a in artifacts
|
||||
ArtifactRow.model_validate(a, from_attributes=True).model_copy(
|
||||
update={"node": shown[a.node]}
|
||||
)
|
||||
for a in artifacts
|
||||
]
|
||||
return detail
|
||||
|
||||
|
||||
@router.get("/{run_id}/nodes/{node}/outputs", response_model=dict[str, Any] | None)
|
||||
def read_node_outputs(run_id: str, node: str, session: SessionDep) -> Any:
|
||||
def read_node_outputs(
|
||||
run_id: str, node: str, session: SessionDep, controller: FlowControllerDep
|
||||
) -> Any:
|
||||
"""What one node returned, by message, as the stage cache stored it.
|
||||
|
||||
Kept only for a node that succeeded and may be reused, and only up to
|
||||
256 kB of JSON. Null when the node published nothing.
|
||||
The node is ``<key>.<node>``, as the run shows it. Kept only for a node
|
||||
that succeeded and may be reused, and only up to 256 kB of JSON. Null when
|
||||
the node published nothing.
|
||||
"""
|
||||
row = session.get(RunNode, {"run_id": run_id, "node": node})
|
||||
run = session.get(Run, run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail="No such run")
|
||||
stored = stored_node(node, controller.store, run.flow_id)
|
||||
row = session.get(RunNode, {"run_id": run_id, "node": stored})
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"No node '{node}' in this run")
|
||||
if row.outputs is None:
|
||||
@@ -823,7 +896,9 @@ def read_node_outputs(run_id: str, node: str, session: SessionDep) -> Any:
|
||||
|
||||
|
||||
@router.post("/{run_id}/cancel", response_model=RunRow)
|
||||
async def cancel_run(run_id: str, request: Request, session: SessionDep) -> Any:
|
||||
async def cancel_run(
|
||||
run_id: str, request: Request, session: SessionDep, controller: FlowControllerDep
|
||||
) -> Any:
|
||||
"""Stop a run. One already past its last node is left as it finished."""
|
||||
run = session.get(Run, run_id)
|
||||
if run is None:
|
||||
@@ -831,21 +906,24 @@ async def cancel_run(run_id: str, request: Request, session: SessionDep) -> Any:
|
||||
service = _service(request)
|
||||
await run_in_threadpool(service.cancel, run_id)
|
||||
session.refresh(run)
|
||||
return run
|
||||
return _named(RunRow, [run], controller.store)[0]
|
||||
|
||||
|
||||
@router.post("/{run_id}/retry", response_model=RunRow, status_code=202)
|
||||
async def retry_run(run_id: str, request: Request, user: CurrentUser) -> Any:
|
||||
async def retry_run(
|
||||
run_id: str, request: Request, user: CurrentUser, controller: FlowControllerDep
|
||||
) -> Any:
|
||||
"""Run the same thing again, as a new run pointing back at this one."""
|
||||
service = _service(request)
|
||||
try:
|
||||
return await run_in_threadpool(service.retry, run_id, user.email)
|
||||
run = await run_in_threadpool(service.retry, run_id, user.email)
|
||||
except FlowNotFound as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except RunRejected as exc:
|
||||
detail = str(exc)
|
||||
status = 404 if detail.startswith("No run ") else 422
|
||||
raise HTTPException(status_code=status, detail=detail) from exc
|
||||
return _named(RunRow, [run], controller.store)[0]
|
||||
|
||||
|
||||
def _drop_runs(session: Session, chosen: Any) -> None:
|
||||
@@ -867,23 +945,38 @@ def _drop_runs(session: Session, chosen: Any) -> None:
|
||||
session.execute(delete(Run).where(col(Run.id).in_(chosen)))
|
||||
|
||||
|
||||
def _live_run(session: Session, flow: str) -> str | None:
|
||||
def _live_run(session: Session, flow_id: str) -> str | None:
|
||||
"""A run of this flow the driver is still writing to, if there is one."""
|
||||
return session.exec(
|
||||
select(Run.id).where(
|
||||
col(Run.flow) == flow, col(Run.status).in_(("running", "queued"))
|
||||
col(Run.flow_id) == flow_id, col(Run.status).in_(("running", "queued"))
|
||||
)
|
||||
).first()
|
||||
|
||||
|
||||
def _audit(action: str, flow_id: str, user: CurrentUser, store: FlowStore) -> None:
|
||||
"""Record who deleted runs. On the bus a flow goes by its key and id."""
|
||||
event_bus.publish(
|
||||
{
|
||||
"type": "audit",
|
||||
"action": action,
|
||||
"flow": name_of(flow_id, store),
|
||||
"flow_id": flow_id,
|
||||
"user": user.email,
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.delete("", response_model=Message)
|
||||
def delete_runs(
|
||||
session: SessionDep,
|
||||
user: CurrentUser,
|
||||
controller: FlowControllerDep,
|
||||
flow: Annotated[str, Query(min_length=1)],
|
||||
history: bool = False,
|
||||
) -> Any:
|
||||
"""Every run of one flow at once.
|
||||
"""Every run of one flow at once, ``flow`` being its id.
|
||||
|
||||
The counterpart to the list's ``flow`` filter, and what a reseed needs:
|
||||
deleting a flow leaves its runs standing on purpose, so this is the only
|
||||
@@ -906,28 +999,22 @@ def delete_runs(
|
||||
)
|
||||
# Counted before the delete, and a subquery rather than a materialised
|
||||
# list of ids on the way in: a demo can hold thousands.
|
||||
chosen = select(col(Run.id)).where(col(Run.flow) == flow)
|
||||
chosen = select(col(Run.id)).where(col(Run.flow_id) == flow)
|
||||
gone = session.exec(
|
||||
select(func.count()).select_from(Run).where(col(Run.flow) == flow)
|
||||
select(func.count()).select_from(Run).where(col(Run.flow_id) == flow)
|
||||
).one()
|
||||
_drop_runs(session, chosen)
|
||||
if history:
|
||||
session.execute(delete(FlowRun).where(col(FlowRun.flow) == flow))
|
||||
session.execute(delete(FlowRun).where(col(FlowRun.flow_id) == flow))
|
||||
session.commit()
|
||||
event_bus.publish(
|
||||
{
|
||||
"type": "audit",
|
||||
"action": f"deleted {gone} run(s)",
|
||||
"flow": flow,
|
||||
"user": user.email,
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
_audit(f"deleted {gone} run(s)", flow, user, controller.store)
|
||||
return Message(message=f"Deleted {gone} run(s) of '{flow}'")
|
||||
|
||||
|
||||
@router.delete("/{run_id}", status_code=204)
|
||||
def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response:
|
||||
def delete_run(
|
||||
run_id: str, session: SessionDep, user: CurrentUser, controller: FlowControllerDep
|
||||
) -> Response:
|
||||
"""Forget a run and everything hanging off it.
|
||||
|
||||
A live run is refused rather than raced: the driver writes its nodes back
|
||||
@@ -949,29 +1036,24 @@ def delete_run(run_id: str, session: SessionDep, user: CurrentUser) -> Response:
|
||||
"finish, before deleting it."
|
||||
),
|
||||
)
|
||||
flow = run.flow
|
||||
flow_id = run.flow_id
|
||||
_drop_runs(session, [run_id])
|
||||
session.commit()
|
||||
event_bus.publish(
|
||||
{
|
||||
"type": "audit",
|
||||
"action": f"deleted run {run_id}",
|
||||
"flow": flow,
|
||||
"user": user.email,
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
_audit(f"deleted run {run_id}", flow_id, user, controller.store)
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
|
||||
def _series(
|
||||
session: Session, store: FlowStore, run_id: str, name: str = ""
|
||||
) -> list[RunMetric]:
|
||||
"""A run's numbers, including the ones a cached node points at.
|
||||
|
||||
A cache hit replays no emissions, so a node restored from an earlier run has
|
||||
no rows of its own — it carries that run's id instead, and its series is read
|
||||
from there. Names are re-qualified on the way out, because the same node
|
||||
reached through two flows publishes under two names and the caller asked for
|
||||
this run's.
|
||||
this run's. Nodes are found by flow id, which is how rows name them, and
|
||||
names are moved between the two flows' keys, which is how they are named.
|
||||
"""
|
||||
statement = select(RunMetric).where(col(RunMetric.run_id) == run_id)
|
||||
if name:
|
||||
@@ -985,16 +1067,17 @@ def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
|
||||
).all()
|
||||
if restored:
|
||||
run = session.get(Run, run_id)
|
||||
flow = run.flow if run is not None else ""
|
||||
flow = run.flow_id if run is not None else ""
|
||||
# Two queries for the lot, rather than a `Run` lookup and a `RunMetric`
|
||||
# query per restored node. A comparison of twenty runs calls this
|
||||
# twenty times, so the per-node round trips multiplied.
|
||||
sources = {
|
||||
row.id: row.flow
|
||||
row.id: row.flow_id
|
||||
for row in session.exec(
|
||||
select(Run).where(col(Run.id).in_({n.cached_from for n in restored}))
|
||||
)
|
||||
}
|
||||
keys = names_of({flow, *sources.values()}, store)
|
||||
wanted: dict[tuple[str, str], list[RunNode]] = {}
|
||||
for node_row in restored:
|
||||
source_flow = sources.get(node_row.cached_from)
|
||||
@@ -1017,7 +1100,7 @@ def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
|
||||
for row in cached:
|
||||
for node_row in wanted.get((row.run_id, row.node), ()):
|
||||
source_flow = sources[node_row.cached_from]
|
||||
renamed = requalify(row.name, source_flow, flow)
|
||||
renamed = requalify(row.name, keys[source_flow], keys[flow])
|
||||
if name and renamed != name:
|
||||
continue
|
||||
rows.append(
|
||||
@@ -1039,6 +1122,7 @@ def _series(session: Session, run_id: str, name: str = "") -> list[RunMetric]:
|
||||
def read_metrics(
|
||||
run_id: str,
|
||||
session: SessionDep,
|
||||
controller: FlowControllerDep,
|
||||
name: str = "",
|
||||
stride: Annotated[int, Query(ge=1, le=100_000)] = 1,
|
||||
) -> Any:
|
||||
@@ -1047,13 +1131,15 @@ def read_metrics(
|
||||
``stride`` thins a long curve down: 3000 steps drawn on a 400-pixel chart
|
||||
is 3000 points nobody can see.
|
||||
"""
|
||||
rows = _series(session, run_id, name)
|
||||
rows = _series(session, controller.store, run_id, name)
|
||||
if stride > 1:
|
||||
rows = rows[:: max(1, stride)]
|
||||
return rows
|
||||
|
||||
|
||||
def _points(session: Session, run_id: str, metric: str, x: str) -> list[list[float]]:
|
||||
def _points(
|
||||
session: Session, store: FlowStore, run_id: str, metric: str, x: str
|
||||
) -> list[list[float]]:
|
||||
"""One run's readings of ``metric``, against whichever x was asked for.
|
||||
|
||||
The step is the default because it is what every run has. Time answers
|
||||
@@ -1063,20 +1149,26 @@ def _points(session: Session, run_id: str, metric: str, x: str) -> list[list[flo
|
||||
epoch, or samples seen — and is joined on the step the two share, which is
|
||||
the only thing they have in common.
|
||||
"""
|
||||
rows = _series(session, run_id, metric)
|
||||
rows = _series(session, store, run_id, metric)
|
||||
if x == "time":
|
||||
if not rows:
|
||||
return []
|
||||
start = min(row.ts for row in rows)
|
||||
return [[row.ts - start, row.value] for row in rows]
|
||||
if x and x != "step":
|
||||
against = {row.step: row.value for row in _series(session, run_id, x)}
|
||||
against = {row.step: row.value for row in _series(session, store, run_id, x)}
|
||||
return [[against[row.step], row.value] for row in rows if row.step in against]
|
||||
return [[float(row.step), row.value] for row in rows]
|
||||
|
||||
|
||||
@router.get("/series/compare", response_model=SeriesAnswer)
|
||||
def compare_metric(session: SessionDep, ids: str, metric: str, x: str = "") -> Any:
|
||||
def compare_metric(
|
||||
session: SessionDep,
|
||||
controller: FlowControllerDep,
|
||||
ids: str,
|
||||
metric: str,
|
||||
x: str = "",
|
||||
) -> Any:
|
||||
"""One metric across several runs, as the chart widget's series shape.
|
||||
|
||||
This is the comparison view: it answers in the same shape a flow answers a
|
||||
@@ -1102,6 +1194,9 @@ def compare_metric(session: SessionDep, ids: str, metric: str, x: str = "") -> A
|
||||
if run.seed is not None:
|
||||
label = f"{run_id} (seed {run.seed})"
|
||||
lines.append(
|
||||
MetricSeries(label=label, points=_points(session, run_id, metric, x))
|
||||
MetricSeries(
|
||||
label=label,
|
||||
points=_points(session, controller.store, run_id, metric, x),
|
||||
)
|
||||
)
|
||||
return SeriesAnswer(metric=metric, x=x or "step", lines=lines)
|
||||
|
||||
@@ -54,16 +54,19 @@ class SearchEntry(BaseModel):
|
||||
"""
|
||||
|
||||
category: Category
|
||||
#: The id the frontend routes on. For an input or output, the qualified
|
||||
#: message, which is what the canvas names its label by. For a setting,
|
||||
#: ``<node>.<setting>``: a node id has no dot, so the node is what comes
|
||||
#: before the first one.
|
||||
#: What it is called: a flow's or dashboard's key, or the id of anything
|
||||
#: else. For an input or output, the qualified message, which is what the
|
||||
#: canvas names its label by. For a setting, ``<node>.<setting>``: a node
|
||||
#: id has no dot, so the node is what comes before the first one.
|
||||
name: str
|
||||
#: Human title, often empty — a flow is usually only its name.
|
||||
title: str = ""
|
||||
#: The flow a node, input or output sits in, or the dashboard a widget
|
||||
#: sits on.
|
||||
#: sits on, by key.
|
||||
parent: str = ""
|
||||
#: The id of the flow or dashboard opening it goes to: its own, or the one
|
||||
#: it sits in. Empty for everything else, whose ``name`` is its id.
|
||||
id: str = ""
|
||||
#: Node type, input dtype, widget type, channel kind.
|
||||
kind: str = ""
|
||||
#: What a setting is set to, as text. Only ever what ``setting_text`` lets
|
||||
@@ -159,13 +162,16 @@ def _build(
|
||||
flow = controller.store.read_flow(name, draft=True)
|
||||
except FlowNotFound:
|
||||
continue
|
||||
entries.append(SearchEntry(category="flow", name=flow.name, title=flow.title))
|
||||
entries.append(
|
||||
SearchEntry(category="flow", name=flow.name, title=flow.title, id=flow.id)
|
||||
)
|
||||
entries.extend(
|
||||
SearchEntry(
|
||||
category="node",
|
||||
name=node.id,
|
||||
title=node.title,
|
||||
parent=flow.name,
|
||||
id=flow.id,
|
||||
kind=node.type,
|
||||
)
|
||||
for node in flow.nodes
|
||||
@@ -184,6 +190,7 @@ def _build(
|
||||
category="setting",
|
||||
name=f"{node.id}.{key}",
|
||||
parent=flow.name,
|
||||
id=flow.id,
|
||||
kind=node.type,
|
||||
detail=text,
|
||||
)
|
||||
@@ -206,18 +213,22 @@ def _build(
|
||||
# The label the canvas draws: bare when it is the flow's own.
|
||||
title=message.removeprefix(f"{flow.name}."),
|
||||
parent=flow.name,
|
||||
id=flow.id,
|
||||
kind=kind,
|
||||
)
|
||||
)
|
||||
|
||||
for summary in dashboards.list():
|
||||
try:
|
||||
dashboard = dashboards.read(summary.name, draft=True)
|
||||
dashboard = dashboards.read(summary.id, draft=True)
|
||||
except DashboardNotFound:
|
||||
continue
|
||||
entries.append(
|
||||
SearchEntry(
|
||||
category="dashboard", name=dashboard.name, title=dashboard.title
|
||||
category="dashboard",
|
||||
name=dashboard.name,
|
||||
title=dashboard.title,
|
||||
id=dashboard.id,
|
||||
)
|
||||
)
|
||||
entries.extend(
|
||||
@@ -226,6 +237,7 @@ def _build(
|
||||
name=widget.id,
|
||||
title=widget.title,
|
||||
parent=dashboard.name,
|
||||
id=dashboard.id,
|
||||
kind=widget.type,
|
||||
)
|
||||
for widget in dashboard.widgets
|
||||
|
||||
@@ -25,7 +25,7 @@ import traceback
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum
|
||||
from typing import Any, TypeGuard, cast
|
||||
|
||||
@@ -228,11 +228,13 @@ class RunContext:
|
||||
class Wiring:
|
||||
"""One published node and the qualified messages it is bound to."""
|
||||
|
||||
#: The flow's key, which the messages are qualified with, and its id.
|
||||
flow: str
|
||||
node: str
|
||||
type: str
|
||||
provides: list[str]
|
||||
requires: list[str]
|
||||
flow_id: str = ""
|
||||
|
||||
|
||||
#: Nothing held, as distinct from a node that yielded ``None``.
|
||||
@@ -977,10 +979,9 @@ class FlowController:
|
||||
async def reload_lib_users(self, ref: str) -> list[str]:
|
||||
"""Rebuild the flows running a shared node whose source has changed.
|
||||
|
||||
Returns their ids, which is what `usages` qualifies with.
|
||||
Returns their ids.
|
||||
"""
|
||||
usages = self.store.usages(ref, draft=False)
|
||||
flows = sorted({usage.split(".", 1)[0] for usage in usages})
|
||||
flows = self.store.lib_users(ref)
|
||||
for flow in flows:
|
||||
await self.reload_flow(flow)
|
||||
return flows
|
||||
@@ -1718,19 +1719,20 @@ class FlowController:
|
||||
)
|
||||
if provides or requires:
|
||||
index.append(
|
||||
Wiring(other.name, node.id, node.type, provides, requires)
|
||||
Wiring(
|
||||
other.name, node.id, node.type, provides, requires, other.id
|
||||
)
|
||||
)
|
||||
self._wiring_cache = (revision, index)
|
||||
return index
|
||||
|
||||
def cross_flow_nodes(
|
||||
self, flow: str
|
||||
) -> list[tuple[str, str, list[str], list[str]]]:
|
||||
def cross_flow_nodes(self, flow: str) -> list[Wiring]:
|
||||
"""Nodes in other flows on the far side of a dotted message name.
|
||||
|
||||
Returns ``(flow, node id, messages of `flow` it provides, ones it
|
||||
requires)``. This is what link-in and link-out mean: the wiring is
|
||||
already real — the canvas simply had no way to draw the other end.
|
||||
Returns the far node's `Wiring`, narrowed to the messages of `flow` it
|
||||
provides and requires. This is what link-in and link-out mean: the
|
||||
wiring is already real — the canvas simply had no way to draw the
|
||||
other end. *flow* is a key.
|
||||
"""
|
||||
prefix = f"{flow}."
|
||||
found = []
|
||||
@@ -1740,7 +1742,7 @@ class FlowController:
|
||||
provides = [name for name in entry.provides if name.startswith(prefix)]
|
||||
requires = [name for name in entry.requires if name.startswith(prefix)]
|
||||
if provides or requires:
|
||||
found.append((entry.flow, entry.node, provides, requires))
|
||||
found.append(replace(entry, provides=provides, requires=requires))
|
||||
return found
|
||||
|
||||
def message_node(
|
||||
@@ -1818,6 +1820,7 @@ class FlowController:
|
||||
group.members.append(member)
|
||||
if flow.name not in group.flows:
|
||||
group.flows.append(flow.name)
|
||||
group.flow_ids.append(flow.id)
|
||||
# A merged neuron stands for several nodes, so one of them being
|
||||
# unable to run is enough to mark it; the first reason wins.
|
||||
if group.issue is None:
|
||||
|
||||
@@ -979,7 +979,7 @@ class DashboardStore:
|
||||
return copy
|
||||
|
||||
def bindings_for(self, flow: str) -> Bindings:
|
||||
"""Every widget bound to a message of ``flow``.
|
||||
"""Every widget bound to a message of the flow with key ``flow``.
|
||||
|
||||
What the canvas draws as an endpoint: a control that sets one of this
|
||||
flow's messages, or a tile that shows one. Without this a dashboard is
|
||||
@@ -993,6 +993,8 @@ class DashboardStore:
|
||||
defn = DashboardDef.model_validate_json(path.read_text())
|
||||
except Exception:
|
||||
continue
|
||||
# By id, which is what opening it takes; the title is the label.
|
||||
dash_id = defn.id or path.parent.name
|
||||
# A bound setting is a consumer too — the dashboard itself reading
|
||||
# a message rather than any tile on it — so the canvas accounts for
|
||||
# it the same way. ``widget`` is what the endpoint id is built
|
||||
@@ -1003,7 +1005,7 @@ class DashboardStore:
|
||||
continue
|
||||
found.append(
|
||||
{
|
||||
"dashboard": defn.name,
|
||||
"dashboard": dash_id,
|
||||
"dashboard_title": defn.title or defn.name,
|
||||
"widget": f"settings.{name}",
|
||||
"title": f"{defn.title or defn.name} {name}",
|
||||
@@ -1020,7 +1022,7 @@ class DashboardStore:
|
||||
continue
|
||||
found.append(
|
||||
{
|
||||
"dashboard": defn.name,
|
||||
"dashboard": dash_id,
|
||||
"dashboard_title": defn.title or defn.name,
|
||||
"widget": widget.id,
|
||||
"title": widget.title or widget.id,
|
||||
|
||||
@@ -77,6 +77,7 @@ class Binding:
|
||||
"""One port of a device, wired to a message by a device node."""
|
||||
|
||||
device: str
|
||||
#: The flow's key, which ``message`` is qualified with.
|
||||
flow: str
|
||||
node: str
|
||||
port: str
|
||||
@@ -88,6 +89,8 @@ class Binding:
|
||||
dtype: str
|
||||
#: Passed through to the device unread.
|
||||
config: dict[str, Any] = field(default_factory=dict)
|
||||
#: The flow's id, which is where wiring the port is edited.
|
||||
flow_id: str = ""
|
||||
|
||||
|
||||
class DeviceIo(BaseModel):
|
||||
@@ -102,7 +105,9 @@ class DeviceIo(BaseModel):
|
||||
class DeviceBinding(BaseModel):
|
||||
"""Where a port is wired: the device node holding it."""
|
||||
|
||||
#: The flow's key, and its id to open it by.
|
||||
flow: str
|
||||
flow_id: str = ""
|
||||
node: str
|
||||
port: str
|
||||
message: str
|
||||
@@ -129,6 +134,7 @@ def bindings_of(flow: FlowDef) -> list[Binding]:
|
||||
Binding(
|
||||
device=device,
|
||||
flow=flow.name,
|
||||
flow_id=flow.id,
|
||||
node=node.id,
|
||||
port=spec.port,
|
||||
message=qualify(flow.name, spec.name),
|
||||
@@ -145,9 +151,10 @@ class DeviceDef(BaseModel):
|
||||
|
||||
id: str
|
||||
title: str = ""
|
||||
#: Ordered. The first one is what the device opens after pairing, and the
|
||||
#: rail follows this order. A name that no longer resolves is a dashboard
|
||||
#: someone deleted, and the device skips it.
|
||||
#: Dashboard ids, ordered. The first one is what the device opens after
|
||||
#: pairing, and the rail follows this order. An id that no longer resolves
|
||||
#: is a dashboard someone deleted, and the device skips it. A dashboard from
|
||||
#: before ids has its key as its id, so a list from then still holds.
|
||||
dashboards: list[str] = Field(default_factory=list)
|
||||
#: Bigger controls, a bigger rail and no hover states, for a screen that is
|
||||
#: touched rather than pointed at. A fact about the screen, not about any
|
||||
@@ -513,6 +520,7 @@ class DeviceStore:
|
||||
(
|
||||
DeviceBinding(
|
||||
flow=b.flow,
|
||||
flow_id=b.flow_id,
|
||||
node=b.node,
|
||||
port=b.port,
|
||||
message=b.message,
|
||||
@@ -1013,7 +1021,7 @@ def dashboards_for(
|
||||
) -> list[DashboardDef]:
|
||||
"""The published documents this device shows, in rail order.
|
||||
|
||||
Published, since that is what a device draws. A name that no longer
|
||||
Published, since that is what a device draws. An id that no longer
|
||||
resolves is skipped, and a device that is gone shows nothing.
|
||||
|
||||
Handed back whole rather than walked here, because what a device may do
|
||||
@@ -1022,7 +1030,7 @@ def dashboards_for(
|
||||
"""
|
||||
if device is None:
|
||||
return []
|
||||
return [defn for name in device.dashboards if (defn := store.published(name))]
|
||||
return [defn for dash_id in device.dashboards if (defn := store.published(dash_id))]
|
||||
|
||||
|
||||
def messages_of(defn: DashboardDef) -> set[str]:
|
||||
|
||||
@@ -13,7 +13,7 @@ import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from collections import deque
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Callable
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
|
||||
@@ -118,6 +118,11 @@ class EventBus:
|
||||
#: Subscribers that lost an event since they last asked. Touched on
|
||||
#: the event loop only, by the flush and the subscribers.
|
||||
self._dropped: set[asyncio.Queue[dict[str, Any]]] = set()
|
||||
#: The id of the flow holding a key, once the store exists. An event
|
||||
#: naming a flow by its key gets its ``flow_id`` here, on the way in,
|
||||
#: so what holds the event later is not left to look up a key that
|
||||
#: may have moved on by then.
|
||||
self.flow_id_of: Callable[[str], str] | None = None
|
||||
|
||||
def bind(self, loop: asyncio.AbstractEventLoop) -> None:
|
||||
"""Attach the bus to the running event loop (called once at startup)."""
|
||||
@@ -125,6 +130,11 @@ class EventBus:
|
||||
|
||||
def publish(self, event: dict[str, Any]) -> None:
|
||||
"""Publish an event from any thread."""
|
||||
resolve = self.flow_id_of
|
||||
if resolve is not None and "flow" in event and "flow_id" not in event:
|
||||
# An event about the engine as a whole names no flow, and no id.
|
||||
flow = str(event["flow"] or "")
|
||||
event["flow_id"] = resolve(flow) if flow else ""
|
||||
kind = event.get("type")
|
||||
if kind == "node_log":
|
||||
# deque.append is atomic, so worker threads need no lock here.
|
||||
|
||||
@@ -26,6 +26,7 @@ from fluksio.core.config import settings
|
||||
from fluksio.core.db import engine
|
||||
from fluksio.flow import tombstones
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.messages import requalify
|
||||
from fluksio.models import EngineEvent, FlowRun, MetricBucket
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -84,6 +85,19 @@ def _minute(ts: float) -> int:
|
||||
return int(ts // 60) * 60
|
||||
|
||||
|
||||
def _filed(event: dict[str, Any], node: str = "") -> tuple[str, str]:
|
||||
"""The flow id and the ``<flow id>.<node>`` an event is kept under.
|
||||
|
||||
The event names both by the flow's key and brings the id along from the
|
||||
bus. One from a bus that knows no ids is kept under the key, which is
|
||||
what the id of a flow from before ids is.
|
||||
"""
|
||||
flow = str(event.get("flow") or "")
|
||||
flow_id = str(event.get("flow_id") or flow)
|
||||
node = node or str(event.get("node") or "")
|
||||
return flow_id[:NAME_MAX], requalify(node, flow, flow_id)[:NAME_MAX]
|
||||
|
||||
|
||||
def _detail(event: dict[str, Any]) -> str:
|
||||
text = str(event.get("error") or event.get("reason") or event.get("detail") or "")
|
||||
if event.get("type") == "cascade_dropped":
|
||||
@@ -154,11 +168,7 @@ class MetricsCollector:
|
||||
def _bucket(self, event: dict[str, Any]) -> dict[str, float]:
|
||||
# ponytail: one collector, one row per node per minute; coarsen the
|
||||
# bucket if the node count ever reaches thousands.
|
||||
key = (
|
||||
str(event.get("flow") or "")[:NAME_MAX],
|
||||
str(event.get("node") or "")[:NAME_MAX],
|
||||
_minute(float(event.get("ts") or time.time())),
|
||||
)
|
||||
key = (*_filed(event), _minute(float(event.get("ts") or time.time())))
|
||||
return self._buckets.setdefault(
|
||||
key,
|
||||
{
|
||||
@@ -216,12 +226,13 @@ class MetricsCollector:
|
||||
key = (str(event.get("flow") or ""), str(event.get("node") or ""))
|
||||
traceback = self._tracebacks.pop(key, (0.0, ""))[1]
|
||||
error = str(event.get("error") or "")
|
||||
flow_id, node = _filed(event)
|
||||
self._pending.append(
|
||||
EngineEvent(
|
||||
ts=datetime.fromtimestamp(ts, UTC),
|
||||
type="node_error",
|
||||
flow=str(event.get("flow") or ""),
|
||||
node=str(event.get("node") or ""),
|
||||
flow_id=flow_id,
|
||||
node=node,
|
||||
detail=(f"{error}\n{traceback}" if traceback else error)[
|
||||
:DETAIL_CAP
|
||||
],
|
||||
@@ -252,12 +263,13 @@ class MetricsCollector:
|
||||
|
||||
if kind == "node_health":
|
||||
if event.get("health") == "down":
|
||||
flow_id, node = _filed(event)
|
||||
self._pending.append(
|
||||
EngineEvent(
|
||||
ts=datetime.fromtimestamp(ts, UTC),
|
||||
type="node_health",
|
||||
flow=str(event.get("flow") or ""),
|
||||
node=str(event.get("node") or ""),
|
||||
flow_id=flow_id,
|
||||
node=node,
|
||||
detail=_detail(event) or "Reported itself down.",
|
||||
run=run_id,
|
||||
)
|
||||
@@ -269,7 +281,7 @@ class MetricsCollector:
|
||||
EngineEvent(
|
||||
ts=datetime.fromtimestamp(ts, UTC),
|
||||
type="audit",
|
||||
flow=str(event.get("flow") or ""),
|
||||
flow_id=_filed(event)[0],
|
||||
detail=str(event.get("action") or ""),
|
||||
actor=str(event.get("user") or ""),
|
||||
)
|
||||
@@ -277,12 +289,13 @@ class MetricsCollector:
|
||||
return
|
||||
|
||||
if kind in RECORDED:
|
||||
flow_id, node = _filed(event, str(event.get("task") or ""))
|
||||
self._pending.append(
|
||||
EngineEvent(
|
||||
ts=datetime.fromtimestamp(ts, UTC),
|
||||
type=kind,
|
||||
flow=str(event.get("flow") or ""),
|
||||
node=str(event.get("node") or event.get("task") or ""),
|
||||
flow_id=flow_id,
|
||||
node=node,
|
||||
detail=_detail(event),
|
||||
run=run_id,
|
||||
)
|
||||
@@ -307,7 +320,7 @@ class MetricsCollector:
|
||||
return
|
||||
self._runs[run_id] = {
|
||||
"id": run_id,
|
||||
"flow": str(event.get("flow") or "")[:NAME_MAX],
|
||||
"flow_id": _filed(event)[0],
|
||||
"source": str(event.get("cause") or ""),
|
||||
"parent": str(event.get("parent") or "")[:64],
|
||||
"started_ts": ts,
|
||||
@@ -417,12 +430,12 @@ class MetricsCollector:
|
||||
),
|
||||
[
|
||||
{
|
||||
"flow": flow,
|
||||
"flow_id": flow_id,
|
||||
"node": node,
|
||||
"bucket": datetime.fromtimestamp(minute, UTC),
|
||||
**agg,
|
||||
}
|
||||
for (flow, node, minute), agg in buckets.items()
|
||||
for (flow_id, node, minute), agg in buckets.items()
|
||||
],
|
||||
)
|
||||
|
||||
@@ -434,10 +447,11 @@ class MetricsCollector:
|
||||
session.execute(
|
||||
statement.on_conflict_do_update(
|
||||
index_elements=["id"],
|
||||
# Not the flow: a run is of the one it started in.
|
||||
set_={
|
||||
key: statement.excluded[key]
|
||||
for key in values[0]
|
||||
if key != "id"
|
||||
if key not in ("id", "flow_id")
|
||||
},
|
||||
),
|
||||
values,
|
||||
|
||||
@@ -49,9 +49,10 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlmodel import Session, col, select
|
||||
|
||||
from fluksio.core.db import engine as db_engine
|
||||
from fluksio.flow import tombstones
|
||||
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
|
||||
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
|
||||
from fluksio.flow.messages import qualify
|
||||
from fluksio.flow.messages import qualify, requalify
|
||||
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
|
||||
from fluksio.flow.queue import WorkItem, WorkQueue
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||
@@ -531,8 +532,14 @@ class MetricSink:
|
||||
batch: int = METRIC_BATCH,
|
||||
interval: float = METRIC_FLUSH_S,
|
||||
on_flush: Callable[[list[str]], None] | None = None,
|
||||
key: str = "",
|
||||
flow_id: str = "",
|
||||
) -> None:
|
||||
self.run_id = run_id
|
||||
#: The run's flow, by the key its nodes are named with and by the id
|
||||
#: a row names them with: ``<flow id>.<node>``.
|
||||
self._key = key
|
||||
self._flow_id = flow_id
|
||||
self._batch = batch
|
||||
self._interval = interval
|
||||
self._on_flush = on_flush
|
||||
@@ -562,7 +569,7 @@ class MetricSink:
|
||||
run_id=self.run_id,
|
||||
name=name[:128],
|
||||
step=step,
|
||||
node=node_id[:255],
|
||||
node=requalify(node_id, self._key, self._flow_id)[:255],
|
||||
ts=now,
|
||||
value=float(value),
|
||||
)
|
||||
@@ -802,8 +809,17 @@ class RunCache:
|
||||
not a hit: the reference would name bytes nobody can open.
|
||||
"""
|
||||
|
||||
def __init__(self, artifacts: ArtifactStore | None = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
artifacts: ArtifactStore | None = None,
|
||||
key_of: Callable[[str], str] | None = None,
|
||||
) -> None:
|
||||
self.artifacts = artifacts
|
||||
#: The key a source run's outputs are named with, from its flow's id:
|
||||
#: the flow's key now, or the last one it had if it is gone. Without
|
||||
#: one the id is taken for the key, which it is for a flow from before
|
||||
#: ids.
|
||||
self.key_of = key_of or (lambda flow_id: flow_id)
|
||||
|
||||
def lookup(self, key: str) -> CacheHit | None:
|
||||
if not key:
|
||||
@@ -829,7 +845,7 @@ class RunCache:
|
||||
continue
|
||||
source = session.get(Run, row.run_id)
|
||||
return CacheHit(
|
||||
flow=source.flow if source is not None else "",
|
||||
flow=self.key_of(source.flow_id) if source else "",
|
||||
outputs=outputs,
|
||||
# A row that was itself restored holds no series, so
|
||||
# follow its pointer rather than adding a hop to it.
|
||||
@@ -862,7 +878,9 @@ class RunService:
|
||||
self.controller = controller
|
||||
self.queue = queue
|
||||
self._artifacts = artifacts
|
||||
self._cache = RunCache(artifacts)
|
||||
self._cache = RunCache(
|
||||
artifacts, lambda flow_id: tombstones.name_of(flow_id, controller.store)
|
||||
)
|
||||
# Without one, a run gets a private in-memory state — which is exactly
|
||||
# the isolation it wants, minus surviving the process.
|
||||
self._state_factory = state_factory or (lambda _ns: MemoryState())
|
||||
@@ -912,10 +930,11 @@ class RunService:
|
||||
"""
|
||||
with Session(db_engine) as session:
|
||||
waiting = session.exec(
|
||||
select(Run.id, Run.flow).where(col(Run.status) == "queued")
|
||||
select(Run.id, Run.flow_id).where(col(Run.status) == "queued")
|
||||
).all()
|
||||
for run_id, flow in waiting:
|
||||
self.queue.add(WorkItem(kind="run", node="", flow=flow, run_id=run_id))
|
||||
# A run item names its flow by id, as `submit` queues it.
|
||||
for run_id, flow_id in waiting:
|
||||
self.queue.add(WorkItem(kind="run", node="", flow=flow_id, run_id=run_id))
|
||||
if waiting:
|
||||
logger.info(
|
||||
"Requeued %d run(s) left queued by an earlier engine", len(waiting)
|
||||
@@ -940,7 +959,7 @@ class RunService:
|
||||
|
||||
def submit(
|
||||
self,
|
||||
flow_name: str,
|
||||
flow_id: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
seed: int | None = None,
|
||||
group_id: str | None = None,
|
||||
@@ -953,10 +972,12 @@ class RunService:
|
||||
passed: list[str] | None = None,
|
||||
for_node: str | None = None,
|
||||
) -> Run:
|
||||
"""Journal a run and wake an engine up for it. Never blocks on it.
|
||||
"""Journal a run of the flow *flow_id* and wake an engine up for it.
|
||||
Never blocks on it.
|
||||
|
||||
``passed`` overrides which of ``params`` count as the caller's own, for
|
||||
a retry handing on its parent's whole params.
|
||||
a retry handing on its parent's whole params. ``for_node`` is a
|
||||
``<flow id>.<node>``.
|
||||
"""
|
||||
# Before anything else, including reading the flow: a caller retrying a
|
||||
# submit it never got an answer for is owed the run that answer was
|
||||
@@ -965,7 +986,7 @@ class RunService:
|
||||
existing = self._by_key(idempotency_key)
|
||||
if existing is not None:
|
||||
return existing
|
||||
flow = self.controller.store.read_flow(flow_name, draft=draft)
|
||||
flow = self.controller.store.read_flow(flow_id, draft=draft)
|
||||
issues = batch_issues(flow, self.controller.store, draft=draft)
|
||||
if issues:
|
||||
raise RunRejected(" ".join(issues))
|
||||
@@ -973,7 +994,7 @@ class RunService:
|
||||
|
||||
run = Run(
|
||||
id=new_run_id(),
|
||||
flow=flow.name,
|
||||
flow_id=flow.id,
|
||||
flow_version=flow.version,
|
||||
commit=self.controller.store.head(),
|
||||
origin_commit=origin_commit(flow),
|
||||
@@ -1011,7 +1032,8 @@ class RunService:
|
||||
return existing
|
||||
session.refresh(run)
|
||||
|
||||
self.queue.add(WorkItem(kind="run", node="", flow=flow.name, run_id=run.id))
|
||||
# By id, like the run: what the item names outlives a rename of the key.
|
||||
self.queue.add(WorkItem(kind="run", node="", flow=flow.id, run_id=run.id))
|
||||
return run
|
||||
|
||||
def _complete(
|
||||
@@ -1087,10 +1109,10 @@ class RunService:
|
||||
{"external_id": external_id, "id": existing, "created": False}
|
||||
)
|
||||
continue
|
||||
name = str(entry["flow"])
|
||||
if name not in flows:
|
||||
flows[name] = self.controller.store.read_flow(name)
|
||||
flow = flows[name]
|
||||
flow_id = str(entry["flow"])
|
||||
if flow_id not in flows:
|
||||
flows[flow_id] = self.controller.store.read_flow(flow_id)
|
||||
flow = flows[flow_id]
|
||||
try:
|
||||
params, seed, passed = self._complete(
|
||||
flow, entry["params"], entry["seed"]
|
||||
@@ -1100,7 +1122,7 @@ class RunService:
|
||||
started, finished = entry["started_at"], entry["finished_at"]
|
||||
run = Run(
|
||||
id=new_run_id(),
|
||||
flow=flow.name,
|
||||
flow_id=flow.id,
|
||||
flow_version=flow.version,
|
||||
params=params,
|
||||
passed=passed,
|
||||
@@ -1207,7 +1229,7 @@ class RunService:
|
||||
# The gate first, so nothing new is submitted while the running nodes
|
||||
# are being killed; a gated node is never handed to the executor, so
|
||||
# the graph drains instead of going further.
|
||||
pipeline.pause(self._flow_of(run_id) or "")
|
||||
pipeline.pause(self._key_of(run_id))
|
||||
workers = self.controller.workers
|
||||
if workers is not None:
|
||||
# Keyed by run, so a sweep cancelling one config leaves the others
|
||||
@@ -1239,7 +1261,7 @@ class RunService:
|
||||
raise RunRejected("That run has not finished; cancel it first")
|
||||
source = run.model_copy()
|
||||
return self.submit(
|
||||
source.flow,
|
||||
source.flow_id,
|
||||
params=dict(source.params),
|
||||
seed=source.seed,
|
||||
group_id=source.group_id,
|
||||
@@ -1254,10 +1276,11 @@ class RunService:
|
||||
for_node=source.for_node,
|
||||
)
|
||||
|
||||
def _flow_of(self, run_id: str) -> str | None:
|
||||
def _key_of(self, run_id: str) -> str:
|
||||
"""The key a run's pipeline runs its flow under, which it pauses by."""
|
||||
with Session(db_engine) as session:
|
||||
run = session.get(Run, run_id)
|
||||
return run.flow if run else None
|
||||
return tombstones.name_of(run.flow_id, self.controller.store) if run else ""
|
||||
|
||||
def _finish_unstarted(self, run_id: str, status: str) -> bool:
|
||||
"""Cancel a run nothing of has executed, if it is still in ``status``."""
|
||||
@@ -1464,23 +1487,31 @@ class RunService:
|
||||
state: StateBackend | None = None
|
||||
errors = 0
|
||||
|
||||
# The key its nodes and messages are named with, the id its rows are
|
||||
# filed under. Read from the store and its tombstones rather than the
|
||||
# flow, which may be gone before the run gets as far as reading it.
|
||||
key = tombstones.name_of(run.flow_id, self.controller.store)
|
||||
|
||||
def observe(outcome: NodeOutcome) -> None:
|
||||
nonlocal errors
|
||||
if not outcome.ok:
|
||||
errors += 1
|
||||
self._record_node(run_id, outcome)
|
||||
self._record_node(run, key, outcome)
|
||||
|
||||
sink = MetricSink(
|
||||
run_id,
|
||||
on_flush=lambda names: self._publish_event(
|
||||
{
|
||||
"type": "run_metric",
|
||||
"flow": run.flow,
|
||||
"flow": key,
|
||||
"flow_id": run.flow_id,
|
||||
"run": run_id,
|
||||
"names": names,
|
||||
"ts": time.time(),
|
||||
}
|
||||
),
|
||||
key=key,
|
||||
flow_id=run.flow_id,
|
||||
)
|
||||
try:
|
||||
with self._lock:
|
||||
@@ -1488,7 +1519,7 @@ class RunService:
|
||||
# building one loads every python node in a worker, and waits
|
||||
# for one while all of them are busy — for longer than a lease.
|
||||
self._active[run_id] = None
|
||||
flow = self.controller.store.read_flow(run.flow, draft=run.draft)
|
||||
flow = self.controller.store.read_flow(run.flow_id, draft=run.draft)
|
||||
# Read again, now that it is this run's turn: a sweep queues every
|
||||
# run at once, and the code on disk is free to move in the hours
|
||||
# before the last of them starts. What the record must name is the
|
||||
@@ -1518,7 +1549,7 @@ class RunService:
|
||||
status, reason = "cancelled", "Cancelled before it started"
|
||||
return
|
||||
self._active[run_id] = pipeline
|
||||
self._publish(run, "run_started")
|
||||
self._publish(run, key, "run_started")
|
||||
pipeline.run(seed_values(flow, run.params, run.seed))
|
||||
result = collect_result(flow, state)
|
||||
with self._lock:
|
||||
@@ -1542,7 +1573,7 @@ class RunService:
|
||||
duration = round((time.perf_counter() - started) * 1000, 2)
|
||||
self._finish(run_id, status, reason, result, duration)
|
||||
run.status = status
|
||||
self._publish(run, "run_finished")
|
||||
self._publish(run, key, "run_finished")
|
||||
self._release_cards(run)
|
||||
# Its values were only ever this run's; nothing reads them once it
|
||||
# has a result. On Redis the namespace would expire anyway.
|
||||
@@ -1582,11 +1613,15 @@ class RunService:
|
||||
except Exception:
|
||||
logger.warning("Could not retire the GPU workers of run %s", run.id)
|
||||
|
||||
def _record_node(self, run_id: str, outcome: NodeOutcome) -> None:
|
||||
def _record_node(self, run: Run, key: str, outcome: NodeOutcome) -> None:
|
||||
"""Write what one node did, under ``<flow id>.<node>``. The outputs
|
||||
and artifacts stay named by message, which is by the flow's *key*."""
|
||||
run_id = run.id
|
||||
node = requalify(outcome.node, key, run.flow_id)[:255]
|
||||
outputs = _cacheable(outcome)
|
||||
row = RunNode(
|
||||
run_id=run_id,
|
||||
node=outcome.node[:255],
|
||||
node=node,
|
||||
status="cached" if outcome.cached else ("ok" if outcome.ok else "error"),
|
||||
started_at=datetime.now(UTC),
|
||||
duration_ms=outcome.duration_ms,
|
||||
@@ -1614,7 +1649,7 @@ class RunService:
|
||||
run_id=run_id,
|
||||
name=message[:255],
|
||||
filename=str(ref.get("name") or "")[:255] or None,
|
||||
node=outcome.node[:255],
|
||||
node=node,
|
||||
digest=str(ref.get("digest") or "")[:71],
|
||||
size=int(ref.get("size") or 0),
|
||||
media_type=str(
|
||||
@@ -1667,11 +1702,12 @@ class RunService:
|
||||
except Exception:
|
||||
logger.exception("Could not close run %s", run_id)
|
||||
|
||||
def _publish(self, run: Run, kind: str) -> None:
|
||||
def _publish(self, run: Run, key: str, kind: str) -> None:
|
||||
self._publish_event(
|
||||
{
|
||||
"type": kind,
|
||||
"flow": run.flow,
|
||||
"flow": key,
|
||||
"flow_id": run.flow_id,
|
||||
"run": run.id,
|
||||
"status": run.status,
|
||||
"group": run.group_id or "",
|
||||
|
||||
@@ -529,7 +529,10 @@ class BrainNode(BaseModel):
|
||||
label: str
|
||||
kind: str
|
||||
members: list[str] = Field(default_factory=list)
|
||||
#: The keys of the flows its members sit in, and their ids in the same
|
||||
#: order, which is what opening one goes by.
|
||||
flows: list[str] = Field(default_factory=list)
|
||||
flow_ids: list[str] = Field(default_factory=list)
|
||||
issue: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
|
||||
@@ -29,12 +29,12 @@ import shutil
|
||||
import string
|
||||
import subprocess
|
||||
import threading
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
from fluksio.flow.schemas import FlowDef, _validate_name
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef, _validate_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fluksio.flow.tombstones import Tombstones
|
||||
@@ -603,23 +603,30 @@ class FlowStore:
|
||||
self._commit(f"Detach and delete shared node '{name}'")
|
||||
|
||||
def usages(self, lib_name: str, draft: bool = True) -> list[str]:
|
||||
"""Every ``flow.node`` pointing at this shared source.
|
||||
"""Every ``<key>.<node>`` pointing at this shared source.
|
||||
|
||||
Read from the drafts the editor shows, or with ``draft`` off from the
|
||||
published flows the engine runs.
|
||||
"""
|
||||
used = []
|
||||
for name in self.list_flows():
|
||||
return sorted(
|
||||
f"{flow.name}.{node.id}" for flow, node in self._lib_nodes(lib_name, draft)
|
||||
)
|
||||
|
||||
def lib_users(self, lib_name: str) -> list[str]:
|
||||
"""The ids of the published flows running this shared source."""
|
||||
return sorted({flow.id for flow, _ in self._lib_nodes(lib_name, False)})
|
||||
|
||||
def _lib_nodes(
|
||||
self, lib_name: str, draft: bool
|
||||
) -> Iterator[tuple[FlowDef, NodeDef]]:
|
||||
for flow_id in self.list_flows():
|
||||
try:
|
||||
definition = self.read_flow(name, draft=draft)
|
||||
definition = self.read_flow(flow_id, draft=draft)
|
||||
except FlowNotFound:
|
||||
continue
|
||||
used += [
|
||||
f"{name}.{node.id}"
|
||||
for node in definition.nodes
|
||||
if node.source_ref == lib_name
|
||||
]
|
||||
return sorted(used)
|
||||
for node in definition.nodes:
|
||||
if node.source_ref == lib_name:
|
||||
yield definition, node
|
||||
|
||||
def _set_source_ref(self, flow: str, node_id: str, ref: str | None) -> None:
|
||||
"""Point a node at a shared source, in the draft the editor is on."""
|
||||
|
||||
@@ -5,6 +5,10 @@ flow's id is opaque, so a tombstone keeps the key it last had, and `names_of`
|
||||
reads that back for anything showing the history. A flow deleted before
|
||||
tombstones existed has its old key as its id, which already reads right.
|
||||
|
||||
History names a node ``<flow id>.<node>``. The API shows and takes it as
|
||||
``<key>.<node>``, the way the engine names it, and `shown_nodes` and
|
||||
`stored_node` translate between the two.
|
||||
|
||||
The flow store is the one writer: it buries a flow before removing its files,
|
||||
asks before issuing an id, and lifts the tombstone of a flow restored under a
|
||||
pinned id once the restored document is on disk.
|
||||
@@ -35,20 +39,20 @@ def referenced(session: Session, flow_id: str) -> bool:
|
||||
queries = [
|
||||
select(Run.id).where(
|
||||
or_(
|
||||
col(Run.flow) == flow_id,
|
||||
col(Run.flow_id) == flow_id,
|
||||
col(Run.for_node).startswith(nodes, autoescape=True),
|
||||
)
|
||||
),
|
||||
select(FlowRun.id).where(col(FlowRun.flow) == flow_id),
|
||||
select(MetricBucket.flow).where(
|
||||
select(FlowRun.id).where(col(FlowRun.flow_id) == flow_id),
|
||||
select(MetricBucket.flow_id).where(
|
||||
or_(
|
||||
col(MetricBucket.flow) == flow_id,
|
||||
col(MetricBucket.flow_id) == flow_id,
|
||||
col(MetricBucket.node).startswith(nodes, autoescape=True),
|
||||
)
|
||||
),
|
||||
select(EngineEvent.id).where(
|
||||
or_(
|
||||
col(EngineEvent.flow) == flow_id,
|
||||
col(EngineEvent.flow_id) == flow_id,
|
||||
col(EngineEvent.node).startswith(nodes, autoescape=True),
|
||||
)
|
||||
),
|
||||
@@ -118,6 +122,44 @@ def name_of(flow_id: str, store: FlowStore) -> str:
|
||||
return names_of([flow_id], store)[flow_id]
|
||||
|
||||
|
||||
def id_of(key: str, store: FlowStore) -> str:
|
||||
"""The flow id behind *key*: the live flow's, else that of the one deleted
|
||||
flow that last held it, else *key* itself, as for history from before ids.
|
||||
"""
|
||||
live = store.key_index().get(key)
|
||||
if live is not None:
|
||||
return live
|
||||
with Session(engine) as session:
|
||||
graves = session.exec(
|
||||
select(FlowTombstone.id).where(col(FlowTombstone.name) == key).limit(2)
|
||||
).all()
|
||||
return graves[0] if len(graves) == 1 else key
|
||||
|
||||
|
||||
def shown_nodes(nodes: Iterable[str], store: FlowStore) -> dict[str, str]:
|
||||
"""Each stored ``<flow id>.<node>`` as ``<key>.<node>``, with one lookup."""
|
||||
split = {node: node.partition(".") for node in nodes}
|
||||
names = names_of({flow for flow, dot, _ in split.values() if dot}, store)
|
||||
return {
|
||||
node: f"{names[flow]}.{rest}" if dot else node
|
||||
for node, (flow, dot, rest) in split.items()
|
||||
}
|
||||
|
||||
|
||||
def stored_node(node: str, store: FlowStore, flow_id: str = "") -> str:
|
||||
"""A ``<key>.<node>`` as history stores it, ``<flow id>.<node>``.
|
||||
|
||||
With *flow_id*, a key that is that flow's own resolves to it, which is how
|
||||
a node of a deleted or renamed flow is still found by the name it shows.
|
||||
"""
|
||||
key, dot, rest = node.partition(".")
|
||||
if not dot:
|
||||
return node
|
||||
if flow_id and name_of(flow_id, store) == key:
|
||||
return f"{flow_id}.{rest}"
|
||||
return f"{id_of(key, store)}.{rest}"
|
||||
|
||||
|
||||
def prune(session: Session) -> int:
|
||||
"""Drop the tombstones no history row names any more. Returns how many."""
|
||||
stale = [
|
||||
|
||||
@@ -308,6 +308,10 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
# after them write documents back with whatever id they read.
|
||||
await run_in_threadpool(assign_ids, settings.FLOWS_DIR)
|
||||
store = FlowStore(settings.FLOWS_DIR, tombstones=Tombstones())
|
||||
# Events name a flow by its key, and carry its id from here on. A key
|
||||
# no flow holds any more is passed on as it is, which is the id of a
|
||||
# flow from before ids.
|
||||
event_bus.flow_id_of = lambda key: store.key_index().get(key, key)
|
||||
# The packages node code imports. In the background, because a first
|
||||
# install of anything substantial takes minutes and `uv` is given five
|
||||
# of them twice over — held here, the container never answers its
|
||||
|
||||
@@ -3,7 +3,7 @@ from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from pydantic import EmailStr
|
||||
from sqlalchemy import JSON, Column, Index
|
||||
from sqlalchemy import JSON, Column, Index, String
|
||||
from sqlmodel import Field, SQLModel
|
||||
|
||||
from fluksio.core.types import UTCDateTime
|
||||
@@ -237,7 +237,8 @@ class MetricBucket(SQLModel, table=True):
|
||||
|
||||
__tablename__ = "metric_minute"
|
||||
|
||||
flow: str = Field(primary_key=True, max_length=255)
|
||||
flow_id: str = Field(sa_column=Column("flow", String(255), primary_key=True))
|
||||
#: ``<flow id>.<node>``.
|
||||
node: str = Field(primary_key=True, max_length=255)
|
||||
bucket: datetime = Field(
|
||||
primary_key=True,
|
||||
@@ -273,7 +274,9 @@ class EngineEvent(SQLModel, table=True):
|
||||
)
|
||||
#: node_error, flow_quarantined, engine_degraded, …, or audit.
|
||||
type: str = Field(max_length=32, index=True)
|
||||
flow: str = ""
|
||||
#: The flow's id, empty for the engine as a whole.
|
||||
flow_id: str = Field(default="", sa_column=Column("flow", String, nullable=False))
|
||||
#: ``<flow id>.<node>``, or empty.
|
||||
node: str = ""
|
||||
#: The error and its traceback, or what an audit entry says was done.
|
||||
detail: str = ""
|
||||
@@ -298,7 +301,9 @@ class FlowRun(SQLModel, table=True):
|
||||
#: The queue entry id, a `manual-` one for a run that never queued, or the
|
||||
#: `Run.id` of a batch run.
|
||||
id: str = Field(primary_key=True, max_length=64)
|
||||
flow: str = Field(index=True, max_length=255)
|
||||
flow_id: str = Field(
|
||||
sa_column=Column("flow", String(255), index=True, nullable=False)
|
||||
)
|
||||
#: What caused it: the work item's cause, "manual", or "run" for a batch.
|
||||
source: str = ""
|
||||
#: The run this one continues: on a rate-limit flush, the run whose value
|
||||
@@ -345,7 +350,9 @@ class Run(SQLModel, table=True):
|
||||
)
|
||||
|
||||
id: str = Field(primary_key=True, max_length=64)
|
||||
flow: str = Field(index=True, max_length=255)
|
||||
flow_id: str = Field(
|
||||
sa_column=Column("flow", String(255), index=True, nullable=False)
|
||||
)
|
||||
#: The flow document this ran, and the commit it was read at, so a result
|
||||
#: can be traced back to the code that produced it.
|
||||
flow_version: int = 1
|
||||
@@ -390,8 +397,9 @@ class Run(SQLModel, table=True):
|
||||
)
|
||||
#: Where it was asked for: api, cli, sdk, hook, sweep, import or train.
|
||||
cause: str = Field(default="api", max_length=32)
|
||||
#: The node it was asked for from, as `flow.node`: the wren node a
|
||||
#: training run is for. Null for every other run.
|
||||
#: The node it was asked for from, as ``<flow id>.<node>``: the wren node a
|
||||
#: training run is for, in another flow than the run's own. Null for every
|
||||
#: other run.
|
||||
for_node: str | None = Field(default=None, max_length=255)
|
||||
#: Re-execute every node, whatever the stage cache holds for it.
|
||||
no_cache: bool = False
|
||||
|
||||
Reference in New Issue
Block a user