Flows and dashboards are created by POST under an issued or superuser-pinned id; PUT only updates; deleted flows leave a tombstone with their key; the controller stores by id and runs by key, hooks live under /hooks/<flow id>; a key with state or queued work behind it is refused until cleared

This commit is contained in:
2026-11-02 17:41:23 +01:00
parent fae85f5ecb
commit 08a6376836
63 changed files with 2832 additions and 936 deletions
+31 -1
View File
@@ -4,7 +4,7 @@ from typing import Annotated, Any
from urllib.parse import unquote
import jwt
from fastapi import Depends, HTTPException, Request, status
from fastapi import APIRouter, Depends, HTTPException, Request, status
from fastapi.security import OAuth2PasswordBearer
from jwt.exceptions import InvalidTokenError
from pydantic import ValidationError
@@ -615,3 +615,33 @@ def get_code_author(current_user: CurrentUser) -> User:
if not current_user.is_superuser:
raise HTTPException(status_code=403, detail=CODE_IS_SUPERUSERS)
return current_user
def check_pin(pin: str, user: User, *routers: APIRouter) -> None:
"""Refuse a pinned id from anyone but a superuser, or one a route shadows.
Creating a flow or a dashboard gives it an opaque id. A pinned one is how
the Demo, a test or a restore keeps the id it already knows. *routers* are
the ones serving that kind's paths: a pin equal to one of their literal
segments, such as ``graph``, would be unreachable behind that route.
"""
if not pin:
return
if not user.is_superuser:
raise HTTPException(status_code=403, detail="Only a superuser can choose an id")
shadowed = {
segment
for router in routers
for route in router.routes
if "{"
not in (
segment := getattr(route, "path", "")[len(router.prefix) :]
.strip("/")
.split("/")[0]
)
}
if pin in shadowed:
raise HTTPException(
status_code=422,
detail=f"'{pin}' is a path of the API, so it cannot be an id",
)
+114 -83
View File
@@ -1,4 +1,8 @@
"""Dashboards: documents of widgets bound to message names."""
"""Dashboards: documents of widgets bound to message names.
A dashboard is addressed by its id in every path. Its key, ``name``, is a
label unique among dashboards and what ``GET /dashboards?name=`` looks up.
"""
import time
from typing import Any
@@ -7,18 +11,24 @@ from fastapi import APIRouter, Depends, HTTPException
from fastapi.concurrency import run_in_threadpool
from pydantic import BaseModel
from fluksio.api.deps import DashboardStoreDep, FlowControllerDep, get_current_user
from fluksio.api.deps import (
CurrentUser,
DashboardStoreDep,
FlowControllerDep,
check_pin,
get_current_user,
)
from fluksio.flow.dashboards import (
DashboardCreate,
DashboardDef,
DashboardExists,
DashboardNotFound,
DashboardsPublic,
default_dashboard,
results_dashboard,
results_name,
)
from fluksio.flow.events import event_bus
from fluksio.flow.store import FlowNotFound, StaleVersion
from fluksio.flow.store import FlowNotFound, IdTaken, StaleVersion
from fluksio.models import Message
router = APIRouter(
@@ -35,97 +45,118 @@ def _apply_history_limits(store: Any, controller: Any) -> None:
controller.set_history_limits(store.history_requirements())
def _missing(dashboard_id: str) -> HTTPException:
return HTTPException(
status_code=404, detail=f"No dashboard with the id '{dashboard_id}'"
)
@router.get("/", response_model=DashboardsPublic)
async def read_dashboards(store: DashboardStoreDep) -> Any:
"""Every dashboard, without its contents."""
async def read_dashboards(store: DashboardStoreDep, name: str | None = None) -> Any:
"""Every dashboard, without its contents, or with ``name`` the one with
that key."""
summaries = await run_in_threadpool(store.list)
if name is not None:
summaries = [summary for summary in summaries if summary.name == name]
return DashboardsPublic(data=summaries, count=len(summaries))
@router.get("/{name}", response_model=DashboardDef)
@router.post("/", response_model=DashboardDef, status_code=201)
async def create_dashboard(
body: DashboardCreate, store: DashboardStoreDep, user: CurrentUser
) -> Any:
"""Start a dashboard: one empty grid under a key of its own.
The server gives it an opaque id; a superuser may pin one with ``id``. A
draft, like every edit that follows it — a dashboard reaches a panel only
once someone publishes it, so an empty one never does.
"""
check_pin(body.id, user, router)
try:
# No history limits to apply: they are read from the published
# documents, and this one is not one of them yet.
return await run_in_threadpool(
store.create, default_dashboard(body.name, body.title), body.id
)
except (DashboardExists, IdTaken) as exc:
raise HTTPException(status_code=409, detail=str(exc))
@router.get("/{dashboard_id}", response_model=DashboardDef)
async def read_dashboard(
name: str, store: DashboardStoreDep, draft: bool = False
dashboard_id: str, store: DashboardStoreDep, draft: bool = False
) -> Any:
"""What a panel shows, or with ``draft`` the copy the editor is on."""
try:
return await run_in_threadpool(store.read, name, draft)
return await run_in_threadpool(store.read, dashboard_id, draft)
except DashboardNotFound:
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
raise _missing(dashboard_id)
@router.post("/{name}", response_model=DashboardDef)
async def create_dashboard(name: str, store: DashboardStoreDep) -> Any:
"""Start a dashboard: one page, one section, nothing on it yet.
A draft, like every edit that follows it — a dashboard reaches a panel
only once someone publishes it, so an empty one never does.
"""
if await run_in_threadpool(store.exists, name):
raise HTTPException(status_code=409, detail=f"'{name}' already exists")
try:
defn = default_dashboard(name)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc))
# No history limits to apply: they are read from the published documents,
# and this one is not one of them yet.
return await run_in_threadpool(store.write_draft, defn, 0)
@router.post("/from-flow/{flow}", response_model=DashboardDef)
@router.post("/from-flow/{flow_id}", response_model=DashboardDef, status_code=201)
async def generate_results_dashboard(
flow: str,
flow_id: str,
store: DashboardStoreDep,
controller: FlowControllerDep,
) -> Any:
"""Draw a batch flow's results as a dashboard, from the ports it declares.
Published straight away rather than left as a draft: there is nothing to
review that the flow did not already say, and what makes it useful is
being able to open it against a run immediately. It is an ordinary
dashboard afterwards — editing it is how it stops being generic.
Keyed after the flow's key, ``<key>_results``, which a second one refuses
(409) rather than numbering. Published straight away rather than left as a
draft: there is nothing to review that the flow did not already say, and
what makes it useful is being able to open it against a run immediately.
It is an ordinary dashboard afterwards — editing it is how it stops being
generic.
"""
try:
definition = await run_in_threadpool(controller.store.read_flow, flow)
definition = await run_in_threadpool(controller.store.read_flow, flow_id)
except FlowNotFound:
raise HTTPException(status_code=404, detail=f"No flow named '{flow}'")
raise HTTPException(status_code=404, detail=f"No flow with the id '{flow_id}'")
if definition.mode != "batch":
raise HTTPException(
status_code=422,
detail=f"'{flow}' is a live flow; a results dashboard is a run's view",
detail=(
f"'{definition.name}' is a live flow; a results dashboard is a "
"run's view"
),
)
name = results_name(flow)
if await run_in_threadpool(store.exists, name):
raise HTTPException(status_code=409, detail=f"'{name}' already exists")
try:
written = await run_in_threadpool(store.write, results_dashboard(definition))
written = await run_in_threadpool(
store.create, results_dashboard(definition), "", True
)
except DashboardExists as exc:
raise HTTPException(status_code=409, detail=str(exc))
await run_in_threadpool(_apply_history_limits, store, controller)
event_bus.publish(
{"type": "dashboard_changed", "dashboard": name, "ts": time.time()}
{"type": "dashboard_changed", "dashboard": written.id, "ts": time.time()}
)
return written
@router.put("/{name}", response_model=DashboardDef)
@router.put("/{dashboard_id}", response_model=DashboardDef)
async def save_dashboard(
name: str,
dashboard_id: str,
body: DashboardDef,
store: DashboardStoreDep,
) -> Any:
"""Save unpublished changes, refusing a save someone else has moved past.
This writes a draft: panels keep showing the published document until
someone publishes, so nothing here can change what a wall is displaying —
which is also why the engine's history limits are left alone.
Only to a dashboard that exists: creating one is ``POST /dashboards``, and
neither the id nor the key changes here. This writes a draft: panels keep
showing the published document until someone publishes, so nothing here
can change what a wall is displaying — which is also why the engine's
history limits are left alone.
"""
if body.name != name:
raise HTTPException(status_code=422, detail="The name in the body must match")
if body.id and body.id != name:
if body.id and body.id != dashboard_id:
raise HTTPException(status_code=422, detail="The id in the body must match")
try:
saved = await run_in_threadpool(store.write_draft, body, body.version)
return await run_in_threadpool(
store.write_draft,
body.model_copy(update={"id": dashboard_id}),
body.version,
)
except DashboardNotFound:
raise _missing(dashboard_id)
except StaleVersion as exc:
raise HTTPException(
status_code=409,
@@ -134,27 +165,25 @@ async def save_dashboard(
"current_version": exc.current,
},
)
except DashboardExists as exc:
raise HTTPException(status_code=409, detail=str(exc))
return saved
@router.post("/{name}/publish", response_model=DashboardDef)
@router.post("/{dashboard_id}/publish", response_model=DashboardDef)
async def publish_dashboard(
name: str,
dashboard_id: str,
body: PublishRequest,
store: DashboardStoreDep,
controller: FlowControllerDep,
) -> Any:
"""Put the unpublished changes on the panels."""
if not await run_in_threadpool(store.exists, name):
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
if not await run_in_threadpool(store.has_draft, name):
if not await run_in_threadpool(store.exists, dashboard_id):
raise _missing(dashboard_id)
if not await run_in_threadpool(store.has_draft, dashboard_id):
raise HTTPException(
status_code=400, detail=f"Dashboard '{name}' has no unpublished changes"
status_code=400,
detail=f"Dashboard '{dashboard_id}' has no unpublished changes",
)
try:
published = await run_in_threadpool(store.publish, name, body.version)
published = await run_in_threadpool(store.publish, dashboard_id, body.version)
except StaleVersion as exc:
raise HTTPException(
status_code=409,
@@ -167,50 +196,52 @@ async def publish_dashboard(
# What a panel is showing has changed. Panels watch the flow socket, and
# the tile values alone cannot tell them the document itself moved.
event_bus.publish(
{"type": "dashboard_changed", "dashboard": name, "ts": time.time()}
{"type": "dashboard_changed", "dashboard": dashboard_id, "ts": time.time()}
)
return published
@router.post("/{name}/discard", response_model=DashboardDef)
async def discard_dashboard_draft(name: str, store: DashboardStoreDep) -> Any:
@router.post("/{dashboard_id}/discard", response_model=DashboardDef)
async def discard_dashboard_draft(dashboard_id: str, store: DashboardStoreDep) -> Any:
"""Throw the unpublished changes away and go back to what is shown."""
if not await run_in_threadpool(store.exists, name):
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
if not await run_in_threadpool(store.has_draft, name):
if not await run_in_threadpool(store.exists, dashboard_id):
raise _missing(dashboard_id)
if not await run_in_threadpool(store.has_draft, dashboard_id):
raise HTTPException(
status_code=400, detail=f"Dashboard '{name}' has no unpublished changes"
status_code=400,
detail=f"Dashboard '{dashboard_id}' has no unpublished changes",
)
if not await run_in_threadpool(store.is_published, name):
if not await run_in_threadpool(store.is_published, dashboard_id):
raise HTTPException(
status_code=400,
detail=(
f"Dashboard '{name}' has never been published. Delete it instead "
"of discarding it"
f"Dashboard '{dashboard_id}' has never been published. Delete it "
"instead of discarding it"
),
)
return await run_in_threadpool(store.discard_draft, name)
return await run_in_threadpool(store.discard_draft, dashboard_id)
@router.delete("/{name}", response_model=Message)
@router.delete("/{dashboard_id}", response_model=Message)
async def delete_dashboard(
name: str, store: DashboardStoreDep, controller: FlowControllerDep
dashboard_id: str, store: DashboardStoreDep, controller: FlowControllerDep
) -> Any:
try:
await run_in_threadpool(store.delete, name)
await run_in_threadpool(store.delete, dashboard_id)
except DashboardNotFound:
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
raise _missing(dashboard_id)
await run_in_threadpool(_apply_history_limits, store, controller)
return Message(message=f"Deleted dashboard '{name}'")
return Message(message=f"Deleted dashboard '{dashboard_id}'")
@router.post("/{name}/duplicate", response_model=DashboardDef)
async def duplicate_dashboard(name: str, store: DashboardStoreDep) -> Any:
@router.post("/{dashboard_id}/duplicate", response_model=DashboardDef)
async def duplicate_dashboard(dashboard_id: str, store: DashboardStoreDep) -> Any:
"""Copy a dashboard into a new one that is only a draft, named after it.
No history limits to apply, as for a new dashboard: nothing published moved.
The copy gets an id of its own. No history limits to apply, as for a new
dashboard: nothing published moved.
"""
try:
return await run_in_threadpool(store.duplicate, name)
return await run_in_threadpool(store.duplicate, dashboard_id)
except DashboardNotFound:
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
raise _missing(dashboard_id)
+250 -174
View File
@@ -1,4 +1,10 @@
"""The flow API. Everything the editor can do is available here first."""
"""The flow API. Everything the editor can do is available here first.
A flow is addressed by its id in every path, and never by its key: the key is
the namespace of its messages, which is what the engine goes by, so each route
reads the flow and runs, pauses or reads values under the key it holds. Node
ids in paths are the flow's own, unqualified.
"""
import asyncio
import logging
@@ -30,6 +36,7 @@ from fluksio.api.deps import (
HiddenDep,
SessionDep,
TokenDep,
check_pin,
decode_token,
get_code_author,
get_current_user,
@@ -63,6 +70,7 @@ from fluksio.flow.runs import RunRejected
from fluksio.flow.schemas import (
NAME_PATTERN,
BrainGraph,
FlowCreate,
FlowDef,
FlowsPublic,
FlowStatePublic,
@@ -81,6 +89,7 @@ from fluksio.flow.state import as_number
from fluksio.flow.store import (
FlowExists,
FlowNotFound,
IdTaken,
LibExists,
LibNotFound,
StaleVersion,
@@ -247,10 +256,10 @@ def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail:
"paused": controller.is_paused(name),
}
endpoints = _endpoints(controller, definition)
if controller.store.has_draft(name):
if controller.store.has_draft(definition.id):
# Report the draft the editor is showing, not the version running
# underneath it — otherwise a node the author just broke looks fine.
preview = controller.preview(name)
preview = controller.preview(definition.id)
return FlowDetail(
definition=definition,
nodes=preview.nodes,
@@ -268,16 +277,21 @@ def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail:
)
def _read_flow(controller: FlowController, name: str) -> FlowDef:
def _missing(flow_id: str) -> HTTPException:
return HTTPException(status_code=404, detail=f"No flow with the id '{flow_id}'")
def _read_flow(controller: FlowController, flow_id: str) -> FlowDef:
"""The working copy: the draft when there is one, else what is published."""
try:
return controller.store.read_flow(name, draft=True)
return controller.store.read_flow(flow_id, draft=True)
except FlowNotFound:
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
raise _missing(flow_id)
def _audit(action: str, flow: str, user: CurrentUser) -> None:
"""Record who changed what. The collector writes it down; the bus carries it."""
"""Record who changed what, by the flow's key. The collector writes it
down; the bus carries it."""
event_bus.publish(
{
"type": "audit",
@@ -296,6 +310,7 @@ def _source_ref(definition: FlowDef, node_id: str) -> str | None:
def _require_enabled(controller: FlowController, name: str) -> None:
"""Refuse running a stopped flow, *name* being its key."""
if not controller.is_enabled(name):
raise HTTPException(
status_code=409,
@@ -306,7 +321,8 @@ def _require_enabled(controller: FlowController, name: str) -> None:
def _flow_state(
controller: FlowController, name: str, hidden: frozenset[str]
) -> FlowStatePublic:
"""The flow's values, but those of a camera the caller may not see."""
"""The values of the flow with key *name*, but a camera's the caller may
not see."""
return FlowStatePublic(
values={
key: MessageValue(**value)
@@ -323,29 +339,34 @@ def _flow_state(
@router.get("/", response_model=FlowsPublic)
def read_flows(controller: FlowControllerDep) -> Any:
"""List every flow."""
def read_flows(controller: FlowControllerDep, name: str | None = None) -> Any:
"""List every flow, or with ``name`` the one holding that key."""
ids = controller.store.list_flows()
if name is not None:
found = controller.store.key_index().get(name)
ids = [found] if found else []
summaries = []
for name in controller.store.list_flows():
for flow_id in ids:
try:
definition = controller.store.read_flow(name, draft=True)
definition = controller.store.read_flow(flow_id, draft=True)
except FlowNotFound:
continue
statuses = controller.node_statuses(name)
key = definition.name
statuses = controller.node_statuses(key)
summaries.append(
FlowSummary(
id=definition.id,
name=definition.name,
name=key,
title=definition.title,
mode=definition.mode,
node_count=len(definition.nodes),
error_count=sum(1 for s in statuses if s.status == "error"),
has_draft=controller.store.has_draft(name),
enabled=controller.is_enabled(name),
paused=controller.is_paused(name),
quarantined=controller.is_quarantined(name),
has_draft=controller.store.has_draft(flow_id),
enabled=controller.is_enabled(key),
paused=controller.is_paused(key),
quarantined=controller.is_quarantined(key),
version=definition.version,
updated_at=controller.store.updated_at(name),
updated_at=controller.store.updated_at(flow_id),
)
)
return FlowsPublic(data=summaries, count=len(summaries))
@@ -359,7 +380,7 @@ def read_node_types() -> Any:
return node_type_info()
# Above "/{name}" for the same reason "library" is: a flow called "graph" would
# Above "/{flow_id}" for the same reason "library" is: a flow called "graph" would
# otherwise be unreachable.
@router.get("/graph", response_model=BrainGraph)
def read_graph(controller: FlowControllerDep) -> Any:
@@ -370,7 +391,7 @@ def read_graph(controller: FlowControllerDep) -> Any:
# -----------------------------------------------------------------------------
# Shared nodes
#
# Declared above the "/{name}" routes: "library" would otherwise be read as a
# Declared above the "/{flow_id}" routes: "library" would otherwise be read as a
# flow name.
# -----------------------------------------------------------------------------
@@ -399,8 +420,9 @@ async def delete_shared_node(
await run_in_threadpool(controller.store.detach_lib, lib_name)
except LibNotFound:
raise HTTPException(status_code=404, detail=f"No shared node '{lib_name}'")
for flow in sorted({usage.split(".", 1)[0] for usage in running}):
await controller.reload_flow(flow)
# `usages` qualifies with the flow's id.
for flow_id in sorted({usage.split(".", 1)[0] for usage in running}):
await controller.reload_flow(flow_id)
return Message(message=f"Deleted shared node '{lib_name}'")
# The published flows too: a draft that unshared a node still runs it.
used_by = sorted(
@@ -418,10 +440,10 @@ async def delete_shared_node(
return Message(message=f"Deleted shared node '{lib_name}'")
@router.get("/{name}", response_model=FlowDetail)
def read_flow(name: str, controller: FlowControllerDep) -> Any:
@router.get("/{flow_id}", response_model=FlowDetail)
def read_flow(flow_id: str, controller: FlowControllerDep) -> Any:
"""Read one flow, with the state of its nodes."""
return _detail(controller, _read_flow(controller, name))
return _detail(controller, _read_flow(controller, flow_id))
def _check_flavors(definition: FlowDef) -> None:
@@ -518,147 +540,192 @@ def _refuse_code_changes(definition: FlowDef, before: FlowDef | None) -> None:
)
@router.put("/{name}", response_model=FlowDetail)
def _check_document(definition: FlowDef) -> None:
"""What a create and a save both refuse in a document."""
if len({n.id for n in definition.nodes}) != len(definition.nodes):
raise HTTPException(status_code=400, detail="Node names must be unique")
_check_flavors(definition)
_check_origin(definition)
@router.post("/", response_model=FlowDetail, status_code=201)
async def create_flow(
body: FlowCreate, controller: FlowControllerDep, user: CurrentUser
) -> Any:
"""Create a flow. It is only a draft until someone publishes it.
The server gives it an opaque id. A superuser may pin one with ``id``: an
id no flow holds, or one whose history this flow takes up again, under the
key that history was kept under. A key with state or queued work still
behind it is refused until that is gone, so a new flow never inherits what
another left.
"""
check_pin(body.id, user, router, ws_router)
definition = body.document()
_check_document(definition)
if not user.is_superuser:
_refuse_code_changes(definition, None)
residue = await run_in_threadpool(controller.key_residue, definition.name)
if residue:
raise HTTPException(
status_code=409,
detail=(
f"The key '{definition.name}' still has values or queued work "
f"behind it ({', '.join(residue[:3])}). Wait for the work to "
"finish, or stop what still writes to it."
),
)
try:
stored = await run_in_threadpool(controller.store.create, definition, body.id)
except (FlowExists, IdTaken) as exc:
raise HTTPException(status_code=409, detail=str(exc))
return await run_in_threadpool(_detail, controller, stored)
@router.put("/{flow_id}", response_model=FlowDetail)
async def save_flow(
name: str,
flow_id: str,
definition: FlowDef,
controller: FlowControllerDep,
user: CurrentUser,
) -> Any:
"""Save unpublished changes to a flow.
This writes a draft: the running pipeline keeps the published version until
someone publishes. ``version`` is the one the editor last saw — a mismatch
means another client saved in between and answers 409 rather than throwing
their work away.
Only to one that exists: creating one is ``POST /flows``, and neither the
id nor the key changes here. This writes a draft: the running pipeline
keeps the published version until someone publishes. ``version`` is the
one the editor last saw — a mismatch means another client saved in
between and answers 409 rather than throwing their work away.
"""
if definition.name != name:
raise HTTPException(
status_code=400, detail="The flow name in the body must match the URL"
)
if definition.id and definition.id != name:
if definition.id and definition.id != flow_id:
raise HTTPException(
status_code=400, detail="The flow id in the body must match the URL"
)
duplicates = {n.id for n in definition.nodes}
if len(duplicates) != len(definition.nodes):
raise HTTPException(status_code=400, detail="Node names must be unique")
_check_flavors(definition)
_check_origin(definition)
definition = definition.model_copy(update={"id": flow_id})
_check_document(definition)
before = _read_flow(controller, flow_id)
if not user.is_superuser:
before = _read_flow(controller, name) if controller.store.exists(name) else None
_refuse_code_changes(definition, before)
try:
stored = await run_in_threadpool(
controller.store.write_draft, definition, definition.version
)
except FlowNotFound:
raise _missing(flow_id)
except StaleVersion as exc:
raise HTTPException(
status_code=409,
detail={"message": str(exc), "current_version": exc.current},
)
except FlowExists as exc:
raise HTTPException(status_code=409, detail=str(exc))
return await run_in_threadpool(_detail, controller, stored)
@router.post("/{name}/publish", response_model=FlowDetail)
@router.post("/{flow_id}/publish", response_model=FlowDetail)
async def publish_flow(
name: str,
flow_id: str,
body: PublishRequest,
controller: FlowControllerDep,
user: CurrentUser,
) -> Any:
"""Deploy the unpublished changes: the engine picks them up from here."""
_read_flow(controller, name)
if not controller.store.has_draft(name):
definition = _read_flow(controller, flow_id)
if not controller.store.has_draft(flow_id):
raise HTTPException(
status_code=400, detail=f"Flow '{name}' has no unpublished changes"
status_code=400, detail=f"Flow '{flow_id}' has no unpublished changes"
)
try:
published = await run_in_threadpool(
controller.store.publish_flow, name, body.version
controller.store.publish_flow, flow_id, body.version
)
except StaleVersion as exc:
raise HTTPException(
status_code=409,
detail={"message": str(exc), "current_version": exc.current},
)
_audit("published", name, user)
await controller.reload_flow(name)
_audit("published", definition.name, user)
await controller.reload_flow(flow_id)
return await run_in_threadpool(_detail, controller, published)
@router.post("/{name}/discard-draft", response_model=FlowDetail)
async def discard_draft(name: str, controller: FlowControllerDep) -> Any:
@router.post("/{flow_id}/discard-draft", response_model=FlowDetail)
async def discard_draft(flow_id: str, controller: FlowControllerDep) -> Any:
"""Throw the unpublished changes away and go back to what is running."""
_read_flow(controller, name)
if not controller.store.has_draft(name):
_read_flow(controller, flow_id)
if not controller.store.has_draft(flow_id):
raise HTTPException(
status_code=400, detail=f"Flow '{name}' has no unpublished changes"
status_code=400, detail=f"Flow '{flow_id}' has no unpublished changes"
)
if not controller.store.is_published(name):
if not controller.store.is_published(flow_id):
raise HTTPException(
status_code=400,
detail=(
f"Flow '{name}' has never been published. Delete it instead of "
f"Flow '{flow_id}' has never been published. Delete it instead of "
"discarding it"
),
)
published = await run_in_threadpool(controller.store.discard_draft, name)
published = await run_in_threadpool(controller.store.discard_draft, flow_id)
return await run_in_threadpool(_detail, controller, published)
@router.delete("/{name}", response_model=Message)
@router.delete("/{flow_id}", response_model=Message)
async def delete_flow(
name: str, controller: FlowControllerDep, user: CurrentUser, session: SessionDep
flow_id: str, controller: FlowControllerDep, user: CurrentUser, session: SessionDep
) -> Any:
"""Delete a flow and everything in it, except the record of what it ran."""
"""Delete a flow and everything in it, except the record of what it ran.
Its key is kept with its id (`fluksio.flow.tombstones`) for as long as
that record is, and what is left under the key, values and queued work,
goes with the flow so the key is free for a new one.
"""
live = session.exec(
select(Run.id).where(
col(Run.flow) == name, col(Run.status).in_(("running", "queued"))
col(Run.flow) == flow_id, col(Run.status).in_(("running", "queued"))
)
).first()
if live is not None:
raise HTTPException(
status_code=409,
detail=(
f"Flow '{name}' has a run in progress ({live}). Cancel it, or "
f"Flow '{flow_id}' has a run in progress ({live}). Cancel it, or "
"wait for it to finish, before deleting the flow."
),
)
try:
await run_in_threadpool(controller.store.delete_flow, name)
key = await run_in_threadpool(controller.store.delete_flow, flow_id)
except FlowNotFound:
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
_audit("deleted", name, user)
# Its files are gone; its values and queued work would otherwise linger.
raise _missing(flow_id)
_audit("deleted", key, user)
# Its nodes first, or they would put values and work straight back.
# Its runs stay: a run record is a record of what ran, and surviving the
# flow it belonged to is the point of keeping one. `DELETE /runs?flow=`
# is what clears them, and with `history=true` its cascades too.
await run_in_threadpool(controller.forget_flow, name)
await controller.reload_flow(name)
return Message(message=f"Deleted flow '{name}'")
await controller.reload_flow(flow_id)
await run_in_threadpool(controller.forget_flow, key)
return Message(message=f"Deleted flow '{key}'")
@router.post("/{name}/duplicate", response_model=FlowDetail)
@router.post("/{flow_id}/duplicate", response_model=FlowDetail)
async def duplicate_flow(
name: str, controller: FlowControllerDep, user: CurrentUser
flow_id: str, controller: FlowControllerDep, user: CurrentUser
) -> Any:
"""Copy a flow into a new one that is only a draft, named after it.
Nothing to reload: a draft does not run. Its code nodes are new code in
the copy, so copying them is a superuser's, like writing them.
The copy gets an id of its own, and the first ``_copy`` key with nothing
left behind it. Nothing to reload: a draft does not run. Its code nodes
are new code in the copy, so copying them is a superuser's, like writing
them.
"""
if not user.is_superuser:
_refuse_code_changes(_read_flow(controller, name), None)
_refuse_code_changes(_read_flow(controller, flow_id), None)
try:
copy = await run_in_threadpool(controller.store.duplicate_flow, name)
copy = await run_in_threadpool(
controller.store.duplicate_flow,
flow_id,
lambda key: bool(controller.key_residue(key)),
)
except FlowNotFound:
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
raise _missing(flow_id)
return await run_in_threadpool(_detail, controller, copy)
@@ -667,14 +734,14 @@ async def duplicate_flow(
# -----------------------------------------------------------------------------
@router.get("/{name}/nodes/{node_id}/source", response_model=NodeSource)
@router.get("/{flow_id}/nodes/{node_id}/source", response_model=NodeSource)
def read_node_source(
name: str,
flow_id: str,
node_id: str,
controller: FlowControllerDep,
) -> Any:
"""Read a node's Python source, including unpublished edits."""
definition = _read_flow(controller, name)
definition = _read_flow(controller, flow_id)
ref = _source_ref(definition, node_id)
if ref:
try:
@@ -682,13 +749,13 @@ def read_node_source(
except LibNotFound:
raise HTTPException(status_code=404, detail=f"No shared node '{ref}'")
return NodeSource(
code=controller.store.read_node_source(name, node_id, draft=True),
missing=not controller.store.has_node_source(name, node_id, draft=True),
code=controller.store.read_node_source(flow_id, node_id, draft=True),
missing=not controller.store.has_node_source(flow_id, node_id, draft=True),
)
def _follow_header(
controller: FlowController, name: str, node_id: str, code: str, drop: bool
controller: FlowController, flow_id: str, node_id: str, code: str, drop: bool
) -> FlowDef | None:
"""Bring a function node's ports and settings in line with its header.
@@ -697,7 +764,7 @@ def _follow_header(
and a synced flow's ports are declared in its repository. Returns the
draft when it changed.
"""
definition = controller.store.read_flow(name, draft=True)
definition = controller.store.read_flow(flow_id, draft=True)
node = next((n for n in definition.nodes if n.id == node_id), None)
spec = NODE_TYPES.get(node.type) if node is not None else None
if (
@@ -720,12 +787,12 @@ def _follow_header(
@router.put(
"/{name}/nodes/{node_id}/source",
"/{flow_id}/nodes/{node_id}/source",
response_model=NodeSourceSaved,
dependencies=[Depends(get_code_author)],
)
async def save_node_source(
name: str,
flow_id: str,
node_id: str,
source: NodeSource,
controller: FlowControllerDep,
@@ -749,7 +816,8 @@ async def save_node_source(
adds and renames from the header but keeps what it no longer names until
the editor is left.
"""
definition = _read_flow(controller, name)
definition = _read_flow(controller, flow_id)
name = definition.name
ref = _source_ref(definition, node_id)
if ref:
changed = await run_in_threadpool(
@@ -760,10 +828,10 @@ async def save_node_source(
await controller.reload_lib_users(ref)
else:
await run_in_threadpool(
controller.store.write_node_source, name, node_id, source.code, True
controller.store.write_node_source, flow_id, node_id, source.code, True
)
followed = await run_in_threadpool(
_follow_header, controller, name, node_id, source.code, drop
_follow_header, controller, flow_id, node_id, source.code, drop
)
node_def = next((n for n in definition.nodes if n.id == node_id), None)
worker = (
@@ -785,15 +853,15 @@ async def save_node_source(
)
@router.post("/{name}/nodes/{node_id}/share", response_model=FlowDetail)
@router.post("/{flow_id}/nodes/{node_id}/share", response_model=FlowDetail)
async def share_node(
name: str,
flow_id: str,
node_id: str,
body: ShareRequest,
controller: FlowControllerDep,
) -> Any:
"""Move this node's code into the library so other flows can run it too."""
_read_flow(controller, name)
_read_flow(controller, flow_id)
if not NAME_PATTERN.match(body.lib_name):
raise HTTPException(
status_code=400,
@@ -803,26 +871,26 @@ async def share_node(
)
try:
await run_in_threadpool(
controller.store.share_node, name, node_id, body.lib_name
controller.store.share_node, flow_id, node_id, body.lib_name
)
except LibExists as exc:
raise HTTPException(status_code=409, detail=str(exc))
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
return await run_in_threadpool(_detail, controller, _read_flow(controller, flow_id))
@router.post("/{name}/nodes/{node_id}/unshare", response_model=FlowDetail)
@router.post("/{flow_id}/nodes/{node_id}/unshare", response_model=FlowDetail)
async def unshare_node(
name: str,
flow_id: str,
node_id: str,
controller: FlowControllerDep,
) -> Any:
"""Take a private copy of the shared code back into this flow."""
_read_flow(controller, name)
_read_flow(controller, flow_id)
try:
await run_in_threadpool(controller.store.unshare_node, name, node_id)
await run_in_threadpool(controller.store.unshare_node, flow_id, node_id)
except LibNotFound as exc:
raise HTTPException(status_code=404, detail=str(exc))
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
return await run_in_threadpool(_detail, controller, _read_flow(controller, flow_id))
# -----------------------------------------------------------------------------
@@ -830,49 +898,50 @@ async def unshare_node(
# -----------------------------------------------------------------------------
@router.post("/{name}/start", response_model=FlowDetail)
@router.post("/{flow_id}/start", response_model=FlowDetail)
async def start_flow(
name: str, controller: FlowControllerDep, user: CurrentUser
flow_id: str, controller: FlowControllerDep, user: CurrentUser
) -> Any:
"""Let the engine run this flow again."""
_read_flow(controller, name)
await controller.set_enabled(name, True)
_audit("started", name, user)
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
definition = _read_flow(controller, flow_id)
await controller.set_enabled(flow_id, definition.name, True)
_audit("started", definition.name, user)
return await run_in_threadpool(_detail, controller, definition)
@router.post("/{name}/stop", response_model=FlowDetail)
async def stop_flow(name: str, controller: FlowControllerDep, user: CurrentUser) -> Any:
@router.post("/{flow_id}/stop", response_model=FlowDetail)
async def stop_flow(
flow_id: str, controller: FlowControllerDep, user: CurrentUser
) -> Any:
"""Take this flow off the engine: no subscriptions, schedules or webhooks."""
_read_flow(controller, name)
await controller.set_enabled(name, False)
_audit("stopped", name, user)
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
definition = _read_flow(controller, flow_id)
await controller.set_enabled(flow_id, definition.name, False)
_audit("stopped", definition.name, user)
return await run_in_threadpool(_detail, controller, definition)
@router.post("/{name}/pause", response_model=Message)
def pause_flow(name: str, controller: FlowControllerDep) -> Any:
@router.post("/{flow_id}/pause", response_model=Message)
def pause_flow(flow_id: str, controller: FlowControllerDep) -> Any:
"""Hold the flow's nodes so its messages can be stepped through."""
_read_flow(controller, name)
controller.pause_flow(name)
return Message(message=f"Paused flow '{name}'")
controller.pause_flow(_read_flow(controller, flow_id).name)
return Message(message=f"Paused flow '{flow_id}'")
@router.post("/{name}/resume", response_model=Message)
async def resume_flow(name: str, controller: FlowControllerDep) -> Any:
@router.post("/{flow_id}/resume", response_model=Message)
async def resume_flow(flow_id: str, controller: FlowControllerDep) -> Any:
"""Let the flow carry on, running whatever was held back."""
_read_flow(controller, name)
name = _read_flow(controller, flow_id).name
await run_in_threadpool(controller.resume_flow, name)
return Message(message=f"Resumed flow '{name}'")
return Message(message=f"Resumed flow '{flow_id}'")
@router.post("/{name}/step", response_model=Message)
async def step_flow(name: str, controller: FlowControllerDep) -> Any:
@router.post("/{flow_id}/step", response_model=Message)
async def step_flow(flow_id: str, controller: FlowControllerDep) -> Any:
"""Run one message a pause is holding back, leaving the flow paused."""
_read_flow(controller, name)
name = _read_flow(controller, flow_id).name
node = await run_in_threadpool(controller.step_flow, name)
if node is None:
return Message(message=f"Nothing held back in flow '{name}'")
return Message(message=f"Nothing held back in flow '{flow_id}'")
return Message(message=f"Stepped '{node}'")
@@ -881,18 +950,18 @@ async def step_flow(name: str, controller: FlowControllerDep) -> Any:
# -----------------------------------------------------------------------------
@router.post("/{name}/validate", response_model=ValidationResult)
def validate_flow(name: str, controller: FlowControllerDep) -> Any:
@router.post("/{flow_id}/validate", response_model=ValidationResult)
def validate_flow(flow_id: str, controller: FlowControllerDep) -> Any:
"""Report what would keep this flow from running."""
_read_flow(controller, name)
if controller.store.has_draft(name):
return ValidationResult(issues=controller.preview(name).issues)
return ValidationResult(issues=controller.flow_issues(name))
definition = _read_flow(controller, flow_id)
if controller.store.has_draft(flow_id):
return ValidationResult(issues=controller.preview(flow_id).issues)
return ValidationResult(issues=controller.flow_issues(definition.name))
@router.post("/{name}/run", response_model=FlowStatePublic)
@router.post("/{flow_id}/run", response_model=FlowStatePublic)
async def run_flow(
name: str,
flow_id: str,
body: RunRequest,
request: Request,
controller: FlowControllerDep,
@@ -908,7 +977,8 @@ async def run_flow(
one means: it is what keeps the parameters, the series and the result, and
a button that quietly did something else would be a trap.
"""
definition = _read_flow(controller, name)
definition = _read_flow(controller, flow_id)
name = definition.name
_require_enabled(controller, name)
if definition.mode == "batch":
service = getattr(request.app.state, "run_service", None)
@@ -917,33 +987,34 @@ async def run_flow(
try:
await run_in_threadpool(
service.submit,
name,
flow_id,
params=body.inputs,
cause="api",
actor=user.email,
draft=controller.store.has_draft(name),
draft=controller.store.has_draft(flow_id),
)
except RunRejected as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
return _flow_state(controller, name, hidden)
inputs = {qualify(name, key): value for key, value in body.inputs.items()}
if controller.store.has_draft(name):
await run_in_threadpool(controller.run_preview, name, inputs)
if controller.store.has_draft(flow_id):
await run_in_threadpool(controller.run_preview, flow_id, inputs)
else:
await run_in_threadpool(controller.run_flow, name, inputs)
return _flow_state(controller, name, hidden)
@router.post("/{name}/nodes/{node_id}/trigger", response_model=FlowStatePublic)
@router.post("/{flow_id}/nodes/{node_id}/trigger", response_model=FlowStatePublic)
async def trigger_node(
name: str,
flow_id: str,
node_id: str,
body: TriggerRequest,
controller: FlowControllerDep,
hidden: HiddenDep,
) -> Any:
"""Feed values into a single node."""
name = _read_flow(controller, flow_id).name
_require_enabled(controller, name)
try:
error = await run_in_threadpool(
@@ -951,7 +1022,7 @@ async def trigger_node(
)
except KeyError:
raise HTTPException(
status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
status_code=404, detail=f"No node named '{node_id}' in flow '{flow_id}'"
)
if error:
# The canvas already has the failure from the bus; the person who
@@ -960,22 +1031,23 @@ async def trigger_node(
return _flow_state(controller, name, hidden)
@router.post("/{name}/nodes/{node_id}/cancel", response_model=Message)
def cancel_node(name: str, node_id: str, controller: FlowControllerDep) -> Any:
@router.post("/{flow_id}/nodes/{node_id}/cancel", response_model=Message)
def cancel_node(flow_id: str, node_id: str, controller: FlowControllerDep) -> Any:
"""Stop a node that is running right now, by killing the worker running it.
Idempotent on purpose: by the time a click reaches here the node may well
have finished, and that is the outcome that was asked for.
"""
name = _read_flow(controller, flow_id).name
pool = controller.workers
if pool is not None and pool.cancel(f"{name}.{node_id}"):
return Message(message=f"Stopped '{node_id}'")
return Message(message=f"'{node_id}' was not running")
@router.post("/{name}/nodes/{node_id}/acknowledge", response_model=Message)
@router.post("/{flow_id}/nodes/{node_id}/acknowledge", response_model=Message)
def acknowledge_node_error(
name: str, node_id: str, controller: FlowControllerDep
flow_id: str, node_id: str, controller: FlowControllerDep
) -> Any:
"""Dismiss what a node last failed with, so the canvas stops marking it.
@@ -983,18 +1055,19 @@ def acknowledge_node_error(
an alert leaves no trace by the time anyone looks. Reading the traceback is
what says it has been seen.
"""
node = f"{_read_flow(controller, flow_id).name}.{node_id}"
try:
controller.acknowledge_error(f"{name}.{node_id}")
controller.acknowledge_error(node)
except KeyError:
raise HTTPException(
status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
status_code=404, detail=f"No node named '{node_id}' in flow '{flow_id}'"
) from None
# Another browser holds the same marker and has no reason to refetch, so it
# would keep showing a failure that is gone until its next snapshot.
event_bus.publish(
{
"type": "node_error_acknowledged",
"node": f"{name}.{node_id}",
"node": node,
"ts": time.time(),
}
)
@@ -1006,21 +1079,23 @@ def acknowledge_node_error(
# -----------------------------------------------------------------------------
def _node_def(controller: FlowController, name: str, node_id: str) -> NodeDef:
def _node_def(controller: FlowController, flow_id: str, node_id: str) -> NodeDef:
"""A node as the editor shows it: from the draft when there is one."""
node = next(
(n for n in _read_flow(controller, name).nodes if n.id == node_id), None
(n for n in _read_flow(controller, flow_id).nodes if n.id == node_id), None
)
if node is None:
raise HTTPException(
status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
status_code=404, detail=f"No node named '{node_id}' in flow '{flow_id}'"
)
return node
@router.post("/{name}/nodes/{node_id}/train", response_model=Training, status_code=202)
@router.post(
"/{flow_id}/nodes/{node_id}/train", response_model=Training, status_code=202
)
async def train_node(
name: str,
flow_id: str,
node_id: str,
request: Request,
controller: FlowControllerDep,
@@ -1030,9 +1105,10 @@ async def train_node(
It learns the node's targets, its ``examples`` and its ``tests``, as the
editor shows them. The node keeps running its model until the new one is
set as its ``model``.
set as its ``model``. The run names the node it trains by the flow's id,
``<flow id>.<node>``, which is what its panel finds it by.
"""
node = _node_def(controller, name, node_id)
node = _node_def(controller, flow_id, node_id)
if node.type != "wren":
raise HTTPException(status_code=400, detail=f"'{node_id}' is not a wren node")
try:
@@ -1054,7 +1130,7 @@ async def train_node(
cause="train",
actor=user.email,
no_cache=True,
for_node=f"{name}.{node_id}",
for_node=f"{flow_id}.{node_id}",
)
except FlowNotFound:
raise HTTPException(
@@ -1069,19 +1145,19 @@ async def train_node(
return Training(id=run.id, status=run.status)
@router.get("/{name}/nodes/{node_id}/train", response_model=Training | None)
@router.get("/{flow_id}/nodes/{node_id}/train", response_model=Training | None)
def read_training(
name: str,
flow_id: str,
node_id: str,
request: Request,
controller: FlowControllerDep,
session: SessionDep,
) -> Any:
"""A wren node's latest training run, or null when it has none."""
_node_def(controller, name, node_id)
_node_def(controller, flow_id, node_id)
run = session.exec(
select(Run)
.where(col(Run.flow) == TRAIN_FLOW, col(Run.for_node) == f"{name}.{node_id}")
.where(col(Run.flow) == TRAIN_FLOW, col(Run.for_node) == f"{flow_id}.{node_id}")
.order_by(col(Run.created_at).desc())
).first()
if run is None:
@@ -1106,27 +1182,27 @@ def read_training(
return training
@router.get("/{name}/state", response_model=FlowStatePublic)
def read_flow_state(name: str, controller: FlowControllerDep, hidden: HiddenDep) -> Any:
@router.get("/{flow_id}/state", response_model=FlowStatePublic)
def read_flow_state(
flow_id: str, controller: FlowControllerDep, hidden: HiddenDep
) -> Any:
"""The last value seen on every message of this flow."""
_read_flow(controller, name)
return _flow_state(controller, name, hidden)
return _flow_state(controller, _read_flow(controller, flow_id).name, hidden)
@router.get("/{name}/history/{message}", response_model=MessageHistory)
@router.get("/{flow_id}/history/{message}", response_model=MessageHistory)
def read_message_history(
name: str,
flow_id: str,
message: str,
controller: FlowControllerDep,
hidden: HiddenDep,
) -> Any:
"""The recent values of one message, for plotting.
``message`` may be given bare or qualified; a message that never carried a
number or a flag comes back with an empty series.
``message`` may be given bare, or qualified with the flow's key; a message
that never carried a number or a flag comes back with an empty series.
"""
_read_flow(controller, name)
key = qualify(name, message)
key = qualify(_read_flow(controller, flow_id).name, message)
if key in hidden:
raise HTTPException(status_code=403, detail=CAMERA_IS_SUPERUSERS)
return MessageHistory(
+2 -1
View File
@@ -204,7 +204,8 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
if queue.get("error"):
problems.append("work queue unreachable")
names = await run_in_threadpool(controller.store.list_flows)
# Keys, which is what the engine's gates and quarantine go by.
names = list(await run_in_threadpool(controller.store.key_index))
quarantined = controller.quarantined
paused = set(controller.paused_flows())
entries = list(controller.loaded.values())