Flows and dashboards are created by POST under an issued or superuser-pinned id; PUT only updates; deleted flows leave a tombstone with their key; the controller stores by id and runs by key, hooks live under /hooks/<flow id>; a key with state or queued work behind it is refused until cleared
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
"""flow_tombstone
|
||||
|
||||
The last key of a deleted flow, so its history still reads as something.
|
||||
|
||||
Revision ID: c4f1a8e2d7b3
|
||||
Revises: b9e4d2a7c1f6
|
||||
Create Date: 2026-11-02
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
import sqlmodel.sql.sqltypes
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "c4f1a8e2d7b3"
|
||||
down_revision = "b9e4d2a7c1f6"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"flow_tombstone",
|
||||
sa.Column("id", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False),
|
||||
sa.Column(
|
||||
"name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False
|
||||
),
|
||||
sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("flow_tombstone")
|
||||
@@ -4,7 +4,7 @@ from typing import Annotated, Any
|
||||
from urllib.parse import unquote
|
||||
|
||||
import jwt
|
||||
from fastapi import Depends, HTTPException, Request, status
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from fastapi.security import OAuth2PasswordBearer
|
||||
from jwt.exceptions import InvalidTokenError
|
||||
from pydantic import ValidationError
|
||||
@@ -615,3 +615,33 @@ def get_code_author(current_user: CurrentUser) -> User:
|
||||
if not current_user.is_superuser:
|
||||
raise HTTPException(status_code=403, detail=CODE_IS_SUPERUSERS)
|
||||
return current_user
|
||||
|
||||
|
||||
def check_pin(pin: str, user: User, *routers: APIRouter) -> None:
|
||||
"""Refuse a pinned id from anyone but a superuser, or one a route shadows.
|
||||
|
||||
Creating a flow or a dashboard gives it an opaque id. A pinned one is how
|
||||
the Demo, a test or a restore keeps the id it already knows. *routers* are
|
||||
the ones serving that kind's paths: a pin equal to one of their literal
|
||||
segments, such as ``graph``, would be unreachable behind that route.
|
||||
"""
|
||||
if not pin:
|
||||
return
|
||||
if not user.is_superuser:
|
||||
raise HTTPException(status_code=403, detail="Only a superuser can choose an id")
|
||||
shadowed = {
|
||||
segment
|
||||
for router in routers
|
||||
for route in router.routes
|
||||
if "{"
|
||||
not in (
|
||||
segment := getattr(route, "path", "")[len(router.prefix) :]
|
||||
.strip("/")
|
||||
.split("/")[0]
|
||||
)
|
||||
}
|
||||
if pin in shadowed:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"'{pin}' is a path of the API, so it cannot be an id",
|
||||
)
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
"""Dashboards: documents of widgets bound to message names."""
|
||||
"""Dashboards: documents of widgets bound to message names.
|
||||
|
||||
A dashboard is addressed by its id in every path. Its key, ``name``, is a
|
||||
label unique among dashboards and what ``GET /dashboards?name=`` looks up.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
@@ -7,18 +11,24 @@ from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from pydantic import BaseModel
|
||||
|
||||
from fluksio.api.deps import DashboardStoreDep, FlowControllerDep, get_current_user
|
||||
from fluksio.api.deps import (
|
||||
CurrentUser,
|
||||
DashboardStoreDep,
|
||||
FlowControllerDep,
|
||||
check_pin,
|
||||
get_current_user,
|
||||
)
|
||||
from fluksio.flow.dashboards import (
|
||||
DashboardCreate,
|
||||
DashboardDef,
|
||||
DashboardExists,
|
||||
DashboardNotFound,
|
||||
DashboardsPublic,
|
||||
default_dashboard,
|
||||
results_dashboard,
|
||||
results_name,
|
||||
)
|
||||
from fluksio.flow.events import event_bus
|
||||
from fluksio.flow.store import FlowNotFound, StaleVersion
|
||||
from fluksio.flow.store import FlowNotFound, IdTaken, StaleVersion
|
||||
from fluksio.models import Message
|
||||
|
||||
router = APIRouter(
|
||||
@@ -35,97 +45,118 @@ def _apply_history_limits(store: Any, controller: Any) -> None:
|
||||
controller.set_history_limits(store.history_requirements())
|
||||
|
||||
|
||||
def _missing(dashboard_id: str) -> HTTPException:
|
||||
return HTTPException(
|
||||
status_code=404, detail=f"No dashboard with the id '{dashboard_id}'"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/", response_model=DashboardsPublic)
|
||||
async def read_dashboards(store: DashboardStoreDep) -> Any:
|
||||
"""Every dashboard, without its contents."""
|
||||
async def read_dashboards(store: DashboardStoreDep, name: str | None = None) -> Any:
|
||||
"""Every dashboard, without its contents, or with ``name`` the one with
|
||||
that key."""
|
||||
summaries = await run_in_threadpool(store.list)
|
||||
if name is not None:
|
||||
summaries = [summary for summary in summaries if summary.name == name]
|
||||
return DashboardsPublic(data=summaries, count=len(summaries))
|
||||
|
||||
|
||||
@router.get("/{name}", response_model=DashboardDef)
|
||||
@router.post("/", response_model=DashboardDef, status_code=201)
|
||||
async def create_dashboard(
|
||||
body: DashboardCreate, store: DashboardStoreDep, user: CurrentUser
|
||||
) -> Any:
|
||||
"""Start a dashboard: one empty grid under a key of its own.
|
||||
|
||||
The server gives it an opaque id; a superuser may pin one with ``id``. A
|
||||
draft, like every edit that follows it — a dashboard reaches a panel only
|
||||
once someone publishes it, so an empty one never does.
|
||||
"""
|
||||
check_pin(body.id, user, router)
|
||||
try:
|
||||
# No history limits to apply: they are read from the published
|
||||
# documents, and this one is not one of them yet.
|
||||
return await run_in_threadpool(
|
||||
store.create, default_dashboard(body.name, body.title), body.id
|
||||
)
|
||||
except (DashboardExists, IdTaken) as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
|
||||
|
||||
@router.get("/{dashboard_id}", response_model=DashboardDef)
|
||||
async def read_dashboard(
|
||||
name: str, store: DashboardStoreDep, draft: bool = False
|
||||
dashboard_id: str, store: DashboardStoreDep, draft: bool = False
|
||||
) -> Any:
|
||||
"""What a panel shows, or with ``draft`` the copy the editor is on."""
|
||||
try:
|
||||
return await run_in_threadpool(store.read, name, draft)
|
||||
return await run_in_threadpool(store.read, dashboard_id, draft)
|
||||
except DashboardNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
|
||||
raise _missing(dashboard_id)
|
||||
|
||||
|
||||
@router.post("/{name}", response_model=DashboardDef)
|
||||
async def create_dashboard(name: str, store: DashboardStoreDep) -> Any:
|
||||
"""Start a dashboard: one page, one section, nothing on it yet.
|
||||
|
||||
A draft, like every edit that follows it — a dashboard reaches a panel
|
||||
only once someone publishes it, so an empty one never does.
|
||||
"""
|
||||
if await run_in_threadpool(store.exists, name):
|
||||
raise HTTPException(status_code=409, detail=f"'{name}' already exists")
|
||||
try:
|
||||
defn = default_dashboard(name)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc))
|
||||
# No history limits to apply: they are read from the published documents,
|
||||
# and this one is not one of them yet.
|
||||
return await run_in_threadpool(store.write_draft, defn, 0)
|
||||
|
||||
|
||||
@router.post("/from-flow/{flow}", response_model=DashboardDef)
|
||||
@router.post("/from-flow/{flow_id}", response_model=DashboardDef, status_code=201)
|
||||
async def generate_results_dashboard(
|
||||
flow: str,
|
||||
flow_id: str,
|
||||
store: DashboardStoreDep,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Draw a batch flow's results as a dashboard, from the ports it declares.
|
||||
|
||||
Published straight away rather than left as a draft: there is nothing to
|
||||
review that the flow did not already say, and what makes it useful is
|
||||
being able to open it against a run immediately. It is an ordinary
|
||||
dashboard afterwards — editing it is how it stops being generic.
|
||||
Keyed after the flow's key, ``<key>_results``, which a second one refuses
|
||||
(409) rather than numbering. Published straight away rather than left as a
|
||||
draft: there is nothing to review that the flow did not already say, and
|
||||
what makes it useful is being able to open it against a run immediately.
|
||||
It is an ordinary dashboard afterwards — editing it is how it stops being
|
||||
generic.
|
||||
"""
|
||||
try:
|
||||
definition = await run_in_threadpool(controller.store.read_flow, flow)
|
||||
definition = await run_in_threadpool(controller.store.read_flow, flow_id)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No flow named '{flow}'")
|
||||
raise HTTPException(status_code=404, detail=f"No flow with the id '{flow_id}'")
|
||||
if definition.mode != "batch":
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"'{flow}' is a live flow; a results dashboard is a run's view",
|
||||
detail=(
|
||||
f"'{definition.name}' is a live flow; a results dashboard is a "
|
||||
"run's view"
|
||||
),
|
||||
)
|
||||
name = results_name(flow)
|
||||
if await run_in_threadpool(store.exists, name):
|
||||
raise HTTPException(status_code=409, detail=f"'{name}' already exists")
|
||||
|
||||
try:
|
||||
written = await run_in_threadpool(store.write, results_dashboard(definition))
|
||||
written = await run_in_threadpool(
|
||||
store.create, results_dashboard(definition), "", True
|
||||
)
|
||||
except DashboardExists as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
await run_in_threadpool(_apply_history_limits, store, controller)
|
||||
event_bus.publish(
|
||||
{"type": "dashboard_changed", "dashboard": name, "ts": time.time()}
|
||||
{"type": "dashboard_changed", "dashboard": written.id, "ts": time.time()}
|
||||
)
|
||||
return written
|
||||
|
||||
|
||||
@router.put("/{name}", response_model=DashboardDef)
|
||||
@router.put("/{dashboard_id}", response_model=DashboardDef)
|
||||
async def save_dashboard(
|
||||
name: str,
|
||||
dashboard_id: str,
|
||||
body: DashboardDef,
|
||||
store: DashboardStoreDep,
|
||||
) -> Any:
|
||||
"""Save unpublished changes, refusing a save someone else has moved past.
|
||||
|
||||
This writes a draft: panels keep showing the published document until
|
||||
someone publishes, so nothing here can change what a wall is displaying —
|
||||
which is also why the engine's history limits are left alone.
|
||||
Only to a dashboard that exists: creating one is ``POST /dashboards``, and
|
||||
neither the id nor the key changes here. This writes a draft: panels keep
|
||||
showing the published document until someone publishes, so nothing here
|
||||
can change what a wall is displaying — which is also why the engine's
|
||||
history limits are left alone.
|
||||
"""
|
||||
if body.name != name:
|
||||
raise HTTPException(status_code=422, detail="The name in the body must match")
|
||||
if body.id and body.id != name:
|
||||
if body.id and body.id != dashboard_id:
|
||||
raise HTTPException(status_code=422, detail="The id in the body must match")
|
||||
try:
|
||||
saved = await run_in_threadpool(store.write_draft, body, body.version)
|
||||
return await run_in_threadpool(
|
||||
store.write_draft,
|
||||
body.model_copy(update={"id": dashboard_id}),
|
||||
body.version,
|
||||
)
|
||||
except DashboardNotFound:
|
||||
raise _missing(dashboard_id)
|
||||
except StaleVersion as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
@@ -134,27 +165,25 @@ async def save_dashboard(
|
||||
"current_version": exc.current,
|
||||
},
|
||||
)
|
||||
except DashboardExists as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
return saved
|
||||
|
||||
|
||||
@router.post("/{name}/publish", response_model=DashboardDef)
|
||||
@router.post("/{dashboard_id}/publish", response_model=DashboardDef)
|
||||
async def publish_dashboard(
|
||||
name: str,
|
||||
dashboard_id: str,
|
||||
body: PublishRequest,
|
||||
store: DashboardStoreDep,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Put the unpublished changes on the panels."""
|
||||
if not await run_in_threadpool(store.exists, name):
|
||||
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
|
||||
if not await run_in_threadpool(store.has_draft, name):
|
||||
if not await run_in_threadpool(store.exists, dashboard_id):
|
||||
raise _missing(dashboard_id)
|
||||
if not await run_in_threadpool(store.has_draft, dashboard_id):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Dashboard '{name}' has no unpublished changes"
|
||||
status_code=400,
|
||||
detail=f"Dashboard '{dashboard_id}' has no unpublished changes",
|
||||
)
|
||||
try:
|
||||
published = await run_in_threadpool(store.publish, name, body.version)
|
||||
published = await run_in_threadpool(store.publish, dashboard_id, body.version)
|
||||
except StaleVersion as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
@@ -167,50 +196,52 @@ async def publish_dashboard(
|
||||
# What a panel is showing has changed. Panels watch the flow socket, and
|
||||
# the tile values alone cannot tell them the document itself moved.
|
||||
event_bus.publish(
|
||||
{"type": "dashboard_changed", "dashboard": name, "ts": time.time()}
|
||||
{"type": "dashboard_changed", "dashboard": dashboard_id, "ts": time.time()}
|
||||
)
|
||||
return published
|
||||
|
||||
|
||||
@router.post("/{name}/discard", response_model=DashboardDef)
|
||||
async def discard_dashboard_draft(name: str, store: DashboardStoreDep) -> Any:
|
||||
@router.post("/{dashboard_id}/discard", response_model=DashboardDef)
|
||||
async def discard_dashboard_draft(dashboard_id: str, store: DashboardStoreDep) -> Any:
|
||||
"""Throw the unpublished changes away and go back to what is shown."""
|
||||
if not await run_in_threadpool(store.exists, name):
|
||||
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
|
||||
if not await run_in_threadpool(store.has_draft, name):
|
||||
if not await run_in_threadpool(store.exists, dashboard_id):
|
||||
raise _missing(dashboard_id)
|
||||
if not await run_in_threadpool(store.has_draft, dashboard_id):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Dashboard '{name}' has no unpublished changes"
|
||||
status_code=400,
|
||||
detail=f"Dashboard '{dashboard_id}' has no unpublished changes",
|
||||
)
|
||||
if not await run_in_threadpool(store.is_published, name):
|
||||
if not await run_in_threadpool(store.is_published, dashboard_id):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Dashboard '{name}' has never been published. Delete it instead "
|
||||
"of discarding it"
|
||||
f"Dashboard '{dashboard_id}' has never been published. Delete it "
|
||||
"instead of discarding it"
|
||||
),
|
||||
)
|
||||
return await run_in_threadpool(store.discard_draft, name)
|
||||
return await run_in_threadpool(store.discard_draft, dashboard_id)
|
||||
|
||||
|
||||
@router.delete("/{name}", response_model=Message)
|
||||
@router.delete("/{dashboard_id}", response_model=Message)
|
||||
async def delete_dashboard(
|
||||
name: str, store: DashboardStoreDep, controller: FlowControllerDep
|
||||
dashboard_id: str, store: DashboardStoreDep, controller: FlowControllerDep
|
||||
) -> Any:
|
||||
try:
|
||||
await run_in_threadpool(store.delete, name)
|
||||
await run_in_threadpool(store.delete, dashboard_id)
|
||||
except DashboardNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
|
||||
raise _missing(dashboard_id)
|
||||
await run_in_threadpool(_apply_history_limits, store, controller)
|
||||
return Message(message=f"Deleted dashboard '{name}'")
|
||||
return Message(message=f"Deleted dashboard '{dashboard_id}'")
|
||||
|
||||
|
||||
@router.post("/{name}/duplicate", response_model=DashboardDef)
|
||||
async def duplicate_dashboard(name: str, store: DashboardStoreDep) -> Any:
|
||||
@router.post("/{dashboard_id}/duplicate", response_model=DashboardDef)
|
||||
async def duplicate_dashboard(dashboard_id: str, store: DashboardStoreDep) -> Any:
|
||||
"""Copy a dashboard into a new one that is only a draft, named after it.
|
||||
|
||||
No history limits to apply, as for a new dashboard: nothing published moved.
|
||||
The copy gets an id of its own. No history limits to apply, as for a new
|
||||
dashboard: nothing published moved.
|
||||
"""
|
||||
try:
|
||||
return await run_in_threadpool(store.duplicate, name)
|
||||
return await run_in_threadpool(store.duplicate, dashboard_id)
|
||||
except DashboardNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
|
||||
raise _missing(dashboard_id)
|
||||
|
||||
+250
-174
@@ -1,4 +1,10 @@
|
||||
"""The flow API. Everything the editor can do is available here first."""
|
||||
"""The flow API. Everything the editor can do is available here first.
|
||||
|
||||
A flow is addressed by its id in every path, and never by its key: the key is
|
||||
the namespace of its messages, which is what the engine goes by, so each route
|
||||
reads the flow and runs, pauses or reads values under the key it holds. Node
|
||||
ids in paths are the flow's own, unqualified.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
@@ -30,6 +36,7 @@ from fluksio.api.deps import (
|
||||
HiddenDep,
|
||||
SessionDep,
|
||||
TokenDep,
|
||||
check_pin,
|
||||
decode_token,
|
||||
get_code_author,
|
||||
get_current_user,
|
||||
@@ -63,6 +70,7 @@ from fluksio.flow.runs import RunRejected
|
||||
from fluksio.flow.schemas import (
|
||||
NAME_PATTERN,
|
||||
BrainGraph,
|
||||
FlowCreate,
|
||||
FlowDef,
|
||||
FlowsPublic,
|
||||
FlowStatePublic,
|
||||
@@ -81,6 +89,7 @@ from fluksio.flow.state import as_number
|
||||
from fluksio.flow.store import (
|
||||
FlowExists,
|
||||
FlowNotFound,
|
||||
IdTaken,
|
||||
LibExists,
|
||||
LibNotFound,
|
||||
StaleVersion,
|
||||
@@ -247,10 +256,10 @@ def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail:
|
||||
"paused": controller.is_paused(name),
|
||||
}
|
||||
endpoints = _endpoints(controller, definition)
|
||||
if controller.store.has_draft(name):
|
||||
if controller.store.has_draft(definition.id):
|
||||
# Report the draft the editor is showing, not the version running
|
||||
# underneath it — otherwise a node the author just broke looks fine.
|
||||
preview = controller.preview(name)
|
||||
preview = controller.preview(definition.id)
|
||||
return FlowDetail(
|
||||
definition=definition,
|
||||
nodes=preview.nodes,
|
||||
@@ -268,16 +277,21 @@ def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail:
|
||||
)
|
||||
|
||||
|
||||
def _read_flow(controller: FlowController, name: str) -> FlowDef:
|
||||
def _missing(flow_id: str) -> HTTPException:
|
||||
return HTTPException(status_code=404, detail=f"No flow with the id '{flow_id}'")
|
||||
|
||||
|
||||
def _read_flow(controller: FlowController, flow_id: str) -> FlowDef:
|
||||
"""The working copy: the draft when there is one, else what is published."""
|
||||
try:
|
||||
return controller.store.read_flow(name, draft=True)
|
||||
return controller.store.read_flow(flow_id, draft=True)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
|
||||
raise _missing(flow_id)
|
||||
|
||||
|
||||
def _audit(action: str, flow: str, user: CurrentUser) -> None:
|
||||
"""Record who changed what. The collector writes it down; the bus carries it."""
|
||||
"""Record who changed what, by the flow's key. The collector writes it
|
||||
down; the bus carries it."""
|
||||
event_bus.publish(
|
||||
{
|
||||
"type": "audit",
|
||||
@@ -296,6 +310,7 @@ def _source_ref(definition: FlowDef, node_id: str) -> str | None:
|
||||
|
||||
|
||||
def _require_enabled(controller: FlowController, name: str) -> None:
|
||||
"""Refuse running a stopped flow, *name* being its key."""
|
||||
if not controller.is_enabled(name):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
@@ -306,7 +321,8 @@ def _require_enabled(controller: FlowController, name: str) -> None:
|
||||
def _flow_state(
|
||||
controller: FlowController, name: str, hidden: frozenset[str]
|
||||
) -> FlowStatePublic:
|
||||
"""The flow's values, but those of a camera the caller may not see."""
|
||||
"""The values of the flow with key *name*, but a camera's the caller may
|
||||
not see."""
|
||||
return FlowStatePublic(
|
||||
values={
|
||||
key: MessageValue(**value)
|
||||
@@ -323,29 +339,34 @@ def _flow_state(
|
||||
|
||||
|
||||
@router.get("/", response_model=FlowsPublic)
|
||||
def read_flows(controller: FlowControllerDep) -> Any:
|
||||
"""List every flow."""
|
||||
def read_flows(controller: FlowControllerDep, name: str | None = None) -> Any:
|
||||
"""List every flow, or with ``name`` the one holding that key."""
|
||||
ids = controller.store.list_flows()
|
||||
if name is not None:
|
||||
found = controller.store.key_index().get(name)
|
||||
ids = [found] if found else []
|
||||
summaries = []
|
||||
for name in controller.store.list_flows():
|
||||
for flow_id in ids:
|
||||
try:
|
||||
definition = controller.store.read_flow(name, draft=True)
|
||||
definition = controller.store.read_flow(flow_id, draft=True)
|
||||
except FlowNotFound:
|
||||
continue
|
||||
statuses = controller.node_statuses(name)
|
||||
key = definition.name
|
||||
statuses = controller.node_statuses(key)
|
||||
summaries.append(
|
||||
FlowSummary(
|
||||
id=definition.id,
|
||||
name=definition.name,
|
||||
name=key,
|
||||
title=definition.title,
|
||||
mode=definition.mode,
|
||||
node_count=len(definition.nodes),
|
||||
error_count=sum(1 for s in statuses if s.status == "error"),
|
||||
has_draft=controller.store.has_draft(name),
|
||||
enabled=controller.is_enabled(name),
|
||||
paused=controller.is_paused(name),
|
||||
quarantined=controller.is_quarantined(name),
|
||||
has_draft=controller.store.has_draft(flow_id),
|
||||
enabled=controller.is_enabled(key),
|
||||
paused=controller.is_paused(key),
|
||||
quarantined=controller.is_quarantined(key),
|
||||
version=definition.version,
|
||||
updated_at=controller.store.updated_at(name),
|
||||
updated_at=controller.store.updated_at(flow_id),
|
||||
)
|
||||
)
|
||||
return FlowsPublic(data=summaries, count=len(summaries))
|
||||
@@ -359,7 +380,7 @@ def read_node_types() -> Any:
|
||||
return node_type_info()
|
||||
|
||||
|
||||
# Above "/{name}" for the same reason "library" is: a flow called "graph" would
|
||||
# Above "/{flow_id}" for the same reason "library" is: a flow called "graph" would
|
||||
# otherwise be unreachable.
|
||||
@router.get("/graph", response_model=BrainGraph)
|
||||
def read_graph(controller: FlowControllerDep) -> Any:
|
||||
@@ -370,7 +391,7 @@ def read_graph(controller: FlowControllerDep) -> Any:
|
||||
# -----------------------------------------------------------------------------
|
||||
# Shared nodes
|
||||
#
|
||||
# Declared above the "/{name}" routes: "library" would otherwise be read as a
|
||||
# Declared above the "/{flow_id}" routes: "library" would otherwise be read as a
|
||||
# flow name.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
@@ -399,8 +420,9 @@ async def delete_shared_node(
|
||||
await run_in_threadpool(controller.store.detach_lib, lib_name)
|
||||
except LibNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No shared node '{lib_name}'")
|
||||
for flow in sorted({usage.split(".", 1)[0] for usage in running}):
|
||||
await controller.reload_flow(flow)
|
||||
# `usages` qualifies with the flow's id.
|
||||
for flow_id in sorted({usage.split(".", 1)[0] for usage in running}):
|
||||
await controller.reload_flow(flow_id)
|
||||
return Message(message=f"Deleted shared node '{lib_name}'")
|
||||
# The published flows too: a draft that unshared a node still runs it.
|
||||
used_by = sorted(
|
||||
@@ -418,10 +440,10 @@ async def delete_shared_node(
|
||||
return Message(message=f"Deleted shared node '{lib_name}'")
|
||||
|
||||
|
||||
@router.get("/{name}", response_model=FlowDetail)
|
||||
def read_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
@router.get("/{flow_id}", response_model=FlowDetail)
|
||||
def read_flow(flow_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Read one flow, with the state of its nodes."""
|
||||
return _detail(controller, _read_flow(controller, name))
|
||||
return _detail(controller, _read_flow(controller, flow_id))
|
||||
|
||||
|
||||
def _check_flavors(definition: FlowDef) -> None:
|
||||
@@ -518,147 +540,192 @@ def _refuse_code_changes(definition: FlowDef, before: FlowDef | None) -> None:
|
||||
)
|
||||
|
||||
|
||||
@router.put("/{name}", response_model=FlowDetail)
|
||||
def _check_document(definition: FlowDef) -> None:
|
||||
"""What a create and a save both refuse in a document."""
|
||||
if len({n.id for n in definition.nodes}) != len(definition.nodes):
|
||||
raise HTTPException(status_code=400, detail="Node names must be unique")
|
||||
_check_flavors(definition)
|
||||
_check_origin(definition)
|
||||
|
||||
|
||||
@router.post("/", response_model=FlowDetail, status_code=201)
|
||||
async def create_flow(
|
||||
body: FlowCreate, controller: FlowControllerDep, user: CurrentUser
|
||||
) -> Any:
|
||||
"""Create a flow. It is only a draft until someone publishes it.
|
||||
|
||||
The server gives it an opaque id. A superuser may pin one with ``id``: an
|
||||
id no flow holds, or one whose history this flow takes up again, under the
|
||||
key that history was kept under. A key with state or queued work still
|
||||
behind it is refused until that is gone, so a new flow never inherits what
|
||||
another left.
|
||||
"""
|
||||
check_pin(body.id, user, router, ws_router)
|
||||
definition = body.document()
|
||||
_check_document(definition)
|
||||
if not user.is_superuser:
|
||||
_refuse_code_changes(definition, None)
|
||||
residue = await run_in_threadpool(controller.key_residue, definition.name)
|
||||
if residue:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
f"The key '{definition.name}' still has values or queued work "
|
||||
f"behind it ({', '.join(residue[:3])}). Wait for the work to "
|
||||
"finish, or stop what still writes to it."
|
||||
),
|
||||
)
|
||||
try:
|
||||
stored = await run_in_threadpool(controller.store.create, definition, body.id)
|
||||
except (FlowExists, IdTaken) as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
return await run_in_threadpool(_detail, controller, stored)
|
||||
|
||||
|
||||
@router.put("/{flow_id}", response_model=FlowDetail)
|
||||
async def save_flow(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
definition: FlowDef,
|
||||
controller: FlowControllerDep,
|
||||
user: CurrentUser,
|
||||
) -> Any:
|
||||
"""Save unpublished changes to a flow.
|
||||
|
||||
This writes a draft: the running pipeline keeps the published version until
|
||||
someone publishes. ``version`` is the one the editor last saw — a mismatch
|
||||
means another client saved in between and answers 409 rather than throwing
|
||||
their work away.
|
||||
Only to one that exists: creating one is ``POST /flows``, and neither the
|
||||
id nor the key changes here. This writes a draft: the running pipeline
|
||||
keeps the published version until someone publishes. ``version`` is the
|
||||
one the editor last saw — a mismatch means another client saved in
|
||||
between and answers 409 rather than throwing their work away.
|
||||
"""
|
||||
if definition.name != name:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="The flow name in the body must match the URL"
|
||||
)
|
||||
if definition.id and definition.id != name:
|
||||
if definition.id and definition.id != flow_id:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="The flow id in the body must match the URL"
|
||||
)
|
||||
|
||||
duplicates = {n.id for n in definition.nodes}
|
||||
if len(duplicates) != len(definition.nodes):
|
||||
raise HTTPException(status_code=400, detail="Node names must be unique")
|
||||
|
||||
_check_flavors(definition)
|
||||
_check_origin(definition)
|
||||
definition = definition.model_copy(update={"id": flow_id})
|
||||
_check_document(definition)
|
||||
before = _read_flow(controller, flow_id)
|
||||
if not user.is_superuser:
|
||||
before = _read_flow(controller, name) if controller.store.exists(name) else None
|
||||
_refuse_code_changes(definition, before)
|
||||
|
||||
try:
|
||||
stored = await run_in_threadpool(
|
||||
controller.store.write_draft, definition, definition.version
|
||||
)
|
||||
except FlowNotFound:
|
||||
raise _missing(flow_id)
|
||||
except StaleVersion as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"message": str(exc), "current_version": exc.current},
|
||||
)
|
||||
except FlowExists as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
return await run_in_threadpool(_detail, controller, stored)
|
||||
|
||||
|
||||
@router.post("/{name}/publish", response_model=FlowDetail)
|
||||
@router.post("/{flow_id}/publish", response_model=FlowDetail)
|
||||
async def publish_flow(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
body: PublishRequest,
|
||||
controller: FlowControllerDep,
|
||||
user: CurrentUser,
|
||||
) -> Any:
|
||||
"""Deploy the unpublished changes: the engine picks them up from here."""
|
||||
_read_flow(controller, name)
|
||||
if not controller.store.has_draft(name):
|
||||
definition = _read_flow(controller, flow_id)
|
||||
if not controller.store.has_draft(flow_id):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Flow '{name}' has no unpublished changes"
|
||||
status_code=400, detail=f"Flow '{flow_id}' has no unpublished changes"
|
||||
)
|
||||
try:
|
||||
published = await run_in_threadpool(
|
||||
controller.store.publish_flow, name, body.version
|
||||
controller.store.publish_flow, flow_id, body.version
|
||||
)
|
||||
except StaleVersion as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"message": str(exc), "current_version": exc.current},
|
||||
)
|
||||
_audit("published", name, user)
|
||||
await controller.reload_flow(name)
|
||||
_audit("published", definition.name, user)
|
||||
await controller.reload_flow(flow_id)
|
||||
return await run_in_threadpool(_detail, controller, published)
|
||||
|
||||
|
||||
@router.post("/{name}/discard-draft", response_model=FlowDetail)
|
||||
async def discard_draft(name: str, controller: FlowControllerDep) -> Any:
|
||||
@router.post("/{flow_id}/discard-draft", response_model=FlowDetail)
|
||||
async def discard_draft(flow_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Throw the unpublished changes away and go back to what is running."""
|
||||
_read_flow(controller, name)
|
||||
if not controller.store.has_draft(name):
|
||||
_read_flow(controller, flow_id)
|
||||
if not controller.store.has_draft(flow_id):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Flow '{name}' has no unpublished changes"
|
||||
status_code=400, detail=f"Flow '{flow_id}' has no unpublished changes"
|
||||
)
|
||||
if not controller.store.is_published(name):
|
||||
if not controller.store.is_published(flow_id):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Flow '{name}' has never been published. Delete it instead of "
|
||||
f"Flow '{flow_id}' has never been published. Delete it instead of "
|
||||
"discarding it"
|
||||
),
|
||||
)
|
||||
published = await run_in_threadpool(controller.store.discard_draft, name)
|
||||
published = await run_in_threadpool(controller.store.discard_draft, flow_id)
|
||||
return await run_in_threadpool(_detail, controller, published)
|
||||
|
||||
|
||||
@router.delete("/{name}", response_model=Message)
|
||||
@router.delete("/{flow_id}", response_model=Message)
|
||||
async def delete_flow(
|
||||
name: str, controller: FlowControllerDep, user: CurrentUser, session: SessionDep
|
||||
flow_id: str, controller: FlowControllerDep, user: CurrentUser, session: SessionDep
|
||||
) -> Any:
|
||||
"""Delete a flow and everything in it, except the record of what it ran."""
|
||||
"""Delete a flow and everything in it, except the record of what it ran.
|
||||
|
||||
Its key is kept with its id (`fluksio.flow.tombstones`) for as long as
|
||||
that record is, and what is left under the key, values and queued work,
|
||||
goes with the flow so the key is free for a new one.
|
||||
"""
|
||||
live = session.exec(
|
||||
select(Run.id).where(
|
||||
col(Run.flow) == name, col(Run.status).in_(("running", "queued"))
|
||||
col(Run.flow) == flow_id, col(Run.status).in_(("running", "queued"))
|
||||
)
|
||||
).first()
|
||||
if live is not None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
f"Flow '{name}' has a run in progress ({live}). Cancel it, or "
|
||||
f"Flow '{flow_id}' has a run in progress ({live}). Cancel it, or "
|
||||
"wait for it to finish, before deleting the flow."
|
||||
),
|
||||
)
|
||||
try:
|
||||
await run_in_threadpool(controller.store.delete_flow, name)
|
||||
key = await run_in_threadpool(controller.store.delete_flow, flow_id)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
|
||||
_audit("deleted", name, user)
|
||||
# Its files are gone; its values and queued work would otherwise linger.
|
||||
raise _missing(flow_id)
|
||||
_audit("deleted", key, user)
|
||||
# Its nodes first, or they would put values and work straight back.
|
||||
# Its runs stay: a run record is a record of what ran, and surviving the
|
||||
# flow it belonged to is the point of keeping one. `DELETE /runs?flow=`
|
||||
# is what clears them, and with `history=true` its cascades too.
|
||||
await run_in_threadpool(controller.forget_flow, name)
|
||||
await controller.reload_flow(name)
|
||||
return Message(message=f"Deleted flow '{name}'")
|
||||
await controller.reload_flow(flow_id)
|
||||
await run_in_threadpool(controller.forget_flow, key)
|
||||
return Message(message=f"Deleted flow '{key}'")
|
||||
|
||||
|
||||
@router.post("/{name}/duplicate", response_model=FlowDetail)
|
||||
@router.post("/{flow_id}/duplicate", response_model=FlowDetail)
|
||||
async def duplicate_flow(
|
||||
name: str, controller: FlowControllerDep, user: CurrentUser
|
||||
flow_id: str, controller: FlowControllerDep, user: CurrentUser
|
||||
) -> Any:
|
||||
"""Copy a flow into a new one that is only a draft, named after it.
|
||||
|
||||
Nothing to reload: a draft does not run. Its code nodes are new code in
|
||||
the copy, so copying them is a superuser's, like writing them.
|
||||
The copy gets an id of its own, and the first ``_copy`` key with nothing
|
||||
left behind it. Nothing to reload: a draft does not run. Its code nodes
|
||||
are new code in the copy, so copying them is a superuser's, like writing
|
||||
them.
|
||||
"""
|
||||
if not user.is_superuser:
|
||||
_refuse_code_changes(_read_flow(controller, name), None)
|
||||
_refuse_code_changes(_read_flow(controller, flow_id), None)
|
||||
try:
|
||||
copy = await run_in_threadpool(controller.store.duplicate_flow, name)
|
||||
copy = await run_in_threadpool(
|
||||
controller.store.duplicate_flow,
|
||||
flow_id,
|
||||
lambda key: bool(controller.key_residue(key)),
|
||||
)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
|
||||
raise _missing(flow_id)
|
||||
return await run_in_threadpool(_detail, controller, copy)
|
||||
|
||||
|
||||
@@ -667,14 +734,14 @@ async def duplicate_flow(
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/{name}/nodes/{node_id}/source", response_model=NodeSource)
|
||||
@router.get("/{flow_id}/nodes/{node_id}/source", response_model=NodeSource)
|
||||
def read_node_source(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Read a node's Python source, including unpublished edits."""
|
||||
definition = _read_flow(controller, name)
|
||||
definition = _read_flow(controller, flow_id)
|
||||
ref = _source_ref(definition, node_id)
|
||||
if ref:
|
||||
try:
|
||||
@@ -682,13 +749,13 @@ def read_node_source(
|
||||
except LibNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No shared node '{ref}'")
|
||||
return NodeSource(
|
||||
code=controller.store.read_node_source(name, node_id, draft=True),
|
||||
missing=not controller.store.has_node_source(name, node_id, draft=True),
|
||||
code=controller.store.read_node_source(flow_id, node_id, draft=True),
|
||||
missing=not controller.store.has_node_source(flow_id, node_id, draft=True),
|
||||
)
|
||||
|
||||
|
||||
def _follow_header(
|
||||
controller: FlowController, name: str, node_id: str, code: str, drop: bool
|
||||
controller: FlowController, flow_id: str, node_id: str, code: str, drop: bool
|
||||
) -> FlowDef | None:
|
||||
"""Bring a function node's ports and settings in line with its header.
|
||||
|
||||
@@ -697,7 +764,7 @@ def _follow_header(
|
||||
and a synced flow's ports are declared in its repository. Returns the
|
||||
draft when it changed.
|
||||
"""
|
||||
definition = controller.store.read_flow(name, draft=True)
|
||||
definition = controller.store.read_flow(flow_id, draft=True)
|
||||
node = next((n for n in definition.nodes if n.id == node_id), None)
|
||||
spec = NODE_TYPES.get(node.type) if node is not None else None
|
||||
if (
|
||||
@@ -720,12 +787,12 @@ def _follow_header(
|
||||
|
||||
|
||||
@router.put(
|
||||
"/{name}/nodes/{node_id}/source",
|
||||
"/{flow_id}/nodes/{node_id}/source",
|
||||
response_model=NodeSourceSaved,
|
||||
dependencies=[Depends(get_code_author)],
|
||||
)
|
||||
async def save_node_source(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
source: NodeSource,
|
||||
controller: FlowControllerDep,
|
||||
@@ -749,7 +816,8 @@ async def save_node_source(
|
||||
adds and renames from the header but keeps what it no longer names until
|
||||
the editor is left.
|
||||
"""
|
||||
definition = _read_flow(controller, name)
|
||||
definition = _read_flow(controller, flow_id)
|
||||
name = definition.name
|
||||
ref = _source_ref(definition, node_id)
|
||||
if ref:
|
||||
changed = await run_in_threadpool(
|
||||
@@ -760,10 +828,10 @@ async def save_node_source(
|
||||
await controller.reload_lib_users(ref)
|
||||
else:
|
||||
await run_in_threadpool(
|
||||
controller.store.write_node_source, name, node_id, source.code, True
|
||||
controller.store.write_node_source, flow_id, node_id, source.code, True
|
||||
)
|
||||
followed = await run_in_threadpool(
|
||||
_follow_header, controller, name, node_id, source.code, drop
|
||||
_follow_header, controller, flow_id, node_id, source.code, drop
|
||||
)
|
||||
node_def = next((n for n in definition.nodes if n.id == node_id), None)
|
||||
worker = (
|
||||
@@ -785,15 +853,15 @@ async def save_node_source(
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/share", response_model=FlowDetail)
|
||||
@router.post("/{flow_id}/nodes/{node_id}/share", response_model=FlowDetail)
|
||||
async def share_node(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
body: ShareRequest,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Move this node's code into the library so other flows can run it too."""
|
||||
_read_flow(controller, name)
|
||||
_read_flow(controller, flow_id)
|
||||
if not NAME_PATTERN.match(body.lib_name):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
@@ -803,26 +871,26 @@ async def share_node(
|
||||
)
|
||||
try:
|
||||
await run_in_threadpool(
|
||||
controller.store.share_node, name, node_id, body.lib_name
|
||||
controller.store.share_node, flow_id, node_id, body.lib_name
|
||||
)
|
||||
except LibExists as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, flow_id))
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/unshare", response_model=FlowDetail)
|
||||
@router.post("/{flow_id}/nodes/{node_id}/unshare", response_model=FlowDetail)
|
||||
async def unshare_node(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Take a private copy of the shared code back into this flow."""
|
||||
_read_flow(controller, name)
|
||||
_read_flow(controller, flow_id)
|
||||
try:
|
||||
await run_in_threadpool(controller.store.unshare_node, name, node_id)
|
||||
await run_in_threadpool(controller.store.unshare_node, flow_id, node_id)
|
||||
except LibNotFound as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, flow_id))
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -830,49 +898,50 @@ async def unshare_node(
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/{name}/start", response_model=FlowDetail)
|
||||
@router.post("/{flow_id}/start", response_model=FlowDetail)
|
||||
async def start_flow(
|
||||
name: str, controller: FlowControllerDep, user: CurrentUser
|
||||
flow_id: str, controller: FlowControllerDep, user: CurrentUser
|
||||
) -> Any:
|
||||
"""Let the engine run this flow again."""
|
||||
_read_flow(controller, name)
|
||||
await controller.set_enabled(name, True)
|
||||
_audit("started", name, user)
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
|
||||
definition = _read_flow(controller, flow_id)
|
||||
await controller.set_enabled(flow_id, definition.name, True)
|
||||
_audit("started", definition.name, user)
|
||||
return await run_in_threadpool(_detail, controller, definition)
|
||||
|
||||
|
||||
@router.post("/{name}/stop", response_model=FlowDetail)
|
||||
async def stop_flow(name: str, controller: FlowControllerDep, user: CurrentUser) -> Any:
|
||||
@router.post("/{flow_id}/stop", response_model=FlowDetail)
|
||||
async def stop_flow(
|
||||
flow_id: str, controller: FlowControllerDep, user: CurrentUser
|
||||
) -> Any:
|
||||
"""Take this flow off the engine: no subscriptions, schedules or webhooks."""
|
||||
_read_flow(controller, name)
|
||||
await controller.set_enabled(name, False)
|
||||
_audit("stopped", name, user)
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
|
||||
definition = _read_flow(controller, flow_id)
|
||||
await controller.set_enabled(flow_id, definition.name, False)
|
||||
_audit("stopped", definition.name, user)
|
||||
return await run_in_threadpool(_detail, controller, definition)
|
||||
|
||||
|
||||
@router.post("/{name}/pause", response_model=Message)
|
||||
def pause_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
@router.post("/{flow_id}/pause", response_model=Message)
|
||||
def pause_flow(flow_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Hold the flow's nodes so its messages can be stepped through."""
|
||||
_read_flow(controller, name)
|
||||
controller.pause_flow(name)
|
||||
return Message(message=f"Paused flow '{name}'")
|
||||
controller.pause_flow(_read_flow(controller, flow_id).name)
|
||||
return Message(message=f"Paused flow '{flow_id}'")
|
||||
|
||||
|
||||
@router.post("/{name}/resume", response_model=Message)
|
||||
async def resume_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
@router.post("/{flow_id}/resume", response_model=Message)
|
||||
async def resume_flow(flow_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Let the flow carry on, running whatever was held back."""
|
||||
_read_flow(controller, name)
|
||||
name = _read_flow(controller, flow_id).name
|
||||
await run_in_threadpool(controller.resume_flow, name)
|
||||
return Message(message=f"Resumed flow '{name}'")
|
||||
return Message(message=f"Resumed flow '{flow_id}'")
|
||||
|
||||
|
||||
@router.post("/{name}/step", response_model=Message)
|
||||
async def step_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
@router.post("/{flow_id}/step", response_model=Message)
|
||||
async def step_flow(flow_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Run one message a pause is holding back, leaving the flow paused."""
|
||||
_read_flow(controller, name)
|
||||
name = _read_flow(controller, flow_id).name
|
||||
node = await run_in_threadpool(controller.step_flow, name)
|
||||
if node is None:
|
||||
return Message(message=f"Nothing held back in flow '{name}'")
|
||||
return Message(message=f"Nothing held back in flow '{flow_id}'")
|
||||
return Message(message=f"Stepped '{node}'")
|
||||
|
||||
|
||||
@@ -881,18 +950,18 @@ async def step_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/{name}/validate", response_model=ValidationResult)
|
||||
def validate_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
@router.post("/{flow_id}/validate", response_model=ValidationResult)
|
||||
def validate_flow(flow_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Report what would keep this flow from running."""
|
||||
_read_flow(controller, name)
|
||||
if controller.store.has_draft(name):
|
||||
return ValidationResult(issues=controller.preview(name).issues)
|
||||
return ValidationResult(issues=controller.flow_issues(name))
|
||||
definition = _read_flow(controller, flow_id)
|
||||
if controller.store.has_draft(flow_id):
|
||||
return ValidationResult(issues=controller.preview(flow_id).issues)
|
||||
return ValidationResult(issues=controller.flow_issues(definition.name))
|
||||
|
||||
|
||||
@router.post("/{name}/run", response_model=FlowStatePublic)
|
||||
@router.post("/{flow_id}/run", response_model=FlowStatePublic)
|
||||
async def run_flow(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
body: RunRequest,
|
||||
request: Request,
|
||||
controller: FlowControllerDep,
|
||||
@@ -908,7 +977,8 @@ async def run_flow(
|
||||
one means: it is what keeps the parameters, the series and the result, and
|
||||
a button that quietly did something else would be a trap.
|
||||
"""
|
||||
definition = _read_flow(controller, name)
|
||||
definition = _read_flow(controller, flow_id)
|
||||
name = definition.name
|
||||
_require_enabled(controller, name)
|
||||
if definition.mode == "batch":
|
||||
service = getattr(request.app.state, "run_service", None)
|
||||
@@ -917,33 +987,34 @@ async def run_flow(
|
||||
try:
|
||||
await run_in_threadpool(
|
||||
service.submit,
|
||||
name,
|
||||
flow_id,
|
||||
params=body.inputs,
|
||||
cause="api",
|
||||
actor=user.email,
|
||||
draft=controller.store.has_draft(name),
|
||||
draft=controller.store.has_draft(flow_id),
|
||||
)
|
||||
except RunRejected as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
return _flow_state(controller, name, hidden)
|
||||
|
||||
inputs = {qualify(name, key): value for key, value in body.inputs.items()}
|
||||
if controller.store.has_draft(name):
|
||||
await run_in_threadpool(controller.run_preview, name, inputs)
|
||||
if controller.store.has_draft(flow_id):
|
||||
await run_in_threadpool(controller.run_preview, flow_id, inputs)
|
||||
else:
|
||||
await run_in_threadpool(controller.run_flow, name, inputs)
|
||||
return _flow_state(controller, name, hidden)
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/trigger", response_model=FlowStatePublic)
|
||||
@router.post("/{flow_id}/nodes/{node_id}/trigger", response_model=FlowStatePublic)
|
||||
async def trigger_node(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
body: TriggerRequest,
|
||||
controller: FlowControllerDep,
|
||||
hidden: HiddenDep,
|
||||
) -> Any:
|
||||
"""Feed values into a single node."""
|
||||
name = _read_flow(controller, flow_id).name
|
||||
_require_enabled(controller, name)
|
||||
try:
|
||||
error = await run_in_threadpool(
|
||||
@@ -951,7 +1022,7 @@ async def trigger_node(
|
||||
)
|
||||
except KeyError:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{flow_id}'"
|
||||
)
|
||||
if error:
|
||||
# The canvas already has the failure from the bus; the person who
|
||||
@@ -960,22 +1031,23 @@ async def trigger_node(
|
||||
return _flow_state(controller, name, hidden)
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/cancel", response_model=Message)
|
||||
def cancel_node(name: str, node_id: str, controller: FlowControllerDep) -> Any:
|
||||
@router.post("/{flow_id}/nodes/{node_id}/cancel", response_model=Message)
|
||||
def cancel_node(flow_id: str, node_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Stop a node that is running right now, by killing the worker running it.
|
||||
|
||||
Idempotent on purpose: by the time a click reaches here the node may well
|
||||
have finished, and that is the outcome that was asked for.
|
||||
"""
|
||||
name = _read_flow(controller, flow_id).name
|
||||
pool = controller.workers
|
||||
if pool is not None and pool.cancel(f"{name}.{node_id}"):
|
||||
return Message(message=f"Stopped '{node_id}'")
|
||||
return Message(message=f"'{node_id}' was not running")
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/acknowledge", response_model=Message)
|
||||
@router.post("/{flow_id}/nodes/{node_id}/acknowledge", response_model=Message)
|
||||
def acknowledge_node_error(
|
||||
name: str, node_id: str, controller: FlowControllerDep
|
||||
flow_id: str, node_id: str, controller: FlowControllerDep
|
||||
) -> Any:
|
||||
"""Dismiss what a node last failed with, so the canvas stops marking it.
|
||||
|
||||
@@ -983,18 +1055,19 @@ def acknowledge_node_error(
|
||||
an alert leaves no trace by the time anyone looks. Reading the traceback is
|
||||
what says it has been seen.
|
||||
"""
|
||||
node = f"{_read_flow(controller, flow_id).name}.{node_id}"
|
||||
try:
|
||||
controller.acknowledge_error(f"{name}.{node_id}")
|
||||
controller.acknowledge_error(node)
|
||||
except KeyError:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{flow_id}'"
|
||||
) from None
|
||||
# Another browser holds the same marker and has no reason to refetch, so it
|
||||
# would keep showing a failure that is gone until its next snapshot.
|
||||
event_bus.publish(
|
||||
{
|
||||
"type": "node_error_acknowledged",
|
||||
"node": f"{name}.{node_id}",
|
||||
"node": node,
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
@@ -1006,21 +1079,23 @@ def acknowledge_node_error(
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _node_def(controller: FlowController, name: str, node_id: str) -> NodeDef:
|
||||
def _node_def(controller: FlowController, flow_id: str, node_id: str) -> NodeDef:
|
||||
"""A node as the editor shows it: from the draft when there is one."""
|
||||
node = next(
|
||||
(n for n in _read_flow(controller, name).nodes if n.id == node_id), None
|
||||
(n for n in _read_flow(controller, flow_id).nodes if n.id == node_id), None
|
||||
)
|
||||
if node is None:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{flow_id}'"
|
||||
)
|
||||
return node
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/train", response_model=Training, status_code=202)
|
||||
@router.post(
|
||||
"/{flow_id}/nodes/{node_id}/train", response_model=Training, status_code=202
|
||||
)
|
||||
async def train_node(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
request: Request,
|
||||
controller: FlowControllerDep,
|
||||
@@ -1030,9 +1105,10 @@ async def train_node(
|
||||
|
||||
It learns the node's targets, its ``examples`` and its ``tests``, as the
|
||||
editor shows them. The node keeps running its model until the new one is
|
||||
set as its ``model``.
|
||||
set as its ``model``. The run names the node it trains by the flow's id,
|
||||
``<flow id>.<node>``, which is what its panel finds it by.
|
||||
"""
|
||||
node = _node_def(controller, name, node_id)
|
||||
node = _node_def(controller, flow_id, node_id)
|
||||
if node.type != "wren":
|
||||
raise HTTPException(status_code=400, detail=f"'{node_id}' is not a wren node")
|
||||
try:
|
||||
@@ -1054,7 +1130,7 @@ async def train_node(
|
||||
cause="train",
|
||||
actor=user.email,
|
||||
no_cache=True,
|
||||
for_node=f"{name}.{node_id}",
|
||||
for_node=f"{flow_id}.{node_id}",
|
||||
)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(
|
||||
@@ -1069,19 +1145,19 @@ async def train_node(
|
||||
return Training(id=run.id, status=run.status)
|
||||
|
||||
|
||||
@router.get("/{name}/nodes/{node_id}/train", response_model=Training | None)
|
||||
@router.get("/{flow_id}/nodes/{node_id}/train", response_model=Training | None)
|
||||
def read_training(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
request: Request,
|
||||
controller: FlowControllerDep,
|
||||
session: SessionDep,
|
||||
) -> Any:
|
||||
"""A wren node's latest training run, or null when it has none."""
|
||||
_node_def(controller, name, node_id)
|
||||
_node_def(controller, flow_id, node_id)
|
||||
run = session.exec(
|
||||
select(Run)
|
||||
.where(col(Run.flow) == TRAIN_FLOW, col(Run.for_node) == f"{name}.{node_id}")
|
||||
.where(col(Run.flow) == TRAIN_FLOW, col(Run.for_node) == f"{flow_id}.{node_id}")
|
||||
.order_by(col(Run.created_at).desc())
|
||||
).first()
|
||||
if run is None:
|
||||
@@ -1106,27 +1182,27 @@ def read_training(
|
||||
return training
|
||||
|
||||
|
||||
@router.get("/{name}/state", response_model=FlowStatePublic)
|
||||
def read_flow_state(name: str, controller: FlowControllerDep, hidden: HiddenDep) -> Any:
|
||||
@router.get("/{flow_id}/state", response_model=FlowStatePublic)
|
||||
def read_flow_state(
|
||||
flow_id: str, controller: FlowControllerDep, hidden: HiddenDep
|
||||
) -> Any:
|
||||
"""The last value seen on every message of this flow."""
|
||||
_read_flow(controller, name)
|
||||
return _flow_state(controller, name, hidden)
|
||||
return _flow_state(controller, _read_flow(controller, flow_id).name, hidden)
|
||||
|
||||
|
||||
@router.get("/{name}/history/{message}", response_model=MessageHistory)
|
||||
@router.get("/{flow_id}/history/{message}", response_model=MessageHistory)
|
||||
def read_message_history(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
message: str,
|
||||
controller: FlowControllerDep,
|
||||
hidden: HiddenDep,
|
||||
) -> Any:
|
||||
"""The recent values of one message, for plotting.
|
||||
|
||||
``message`` may be given bare or qualified; a message that never carried a
|
||||
number or a flag comes back with an empty series.
|
||||
``message`` may be given bare, or qualified with the flow's key; a message
|
||||
that never carried a number or a flag comes back with an empty series.
|
||||
"""
|
||||
_read_flow(controller, name)
|
||||
key = qualify(name, message)
|
||||
key = qualify(_read_flow(controller, flow_id).name, message)
|
||||
if key in hidden:
|
||||
raise HTTPException(status_code=403, detail=CAMERA_IS_SUPERUSERS)
|
||||
return MessageHistory(
|
||||
|
||||
@@ -204,7 +204,8 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
|
||||
if queue.get("error"):
|
||||
problems.append("work queue unreachable")
|
||||
|
||||
names = await run_in_threadpool(controller.store.list_flows)
|
||||
# Keys, which is what the engine's gates and quarantine go by.
|
||||
names = list(await run_in_threadpool(controller.store.key_index))
|
||||
quarantined = controller.quarantined
|
||||
paused = set(controller.paused_flows())
|
||||
entries = list(controller.loaded.values())
|
||||
|
||||
@@ -72,6 +72,7 @@ from fluksio.flow.pipeline import (
|
||||
ValueSource,
|
||||
)
|
||||
from fluksio.flow.placement import Placer
|
||||
from fluksio.flow.queue import WorkItem, owned_by
|
||||
from fluksio.flow.remote import RemoteWorkerHub
|
||||
from fluksio.flow.resources import (
|
||||
ResourceAccountant,
|
||||
@@ -160,7 +161,10 @@ class LoadedNode:
|
||||
"""
|
||||
|
||||
id: str
|
||||
#: The flow's key, which ``id`` is qualified with.
|
||||
flow: str
|
||||
#: The flow's id, which its store paths and API routes use.
|
||||
flow_id: str = ""
|
||||
status: NodeStatus = NodeStatus.ACTIVE
|
||||
node: Node | None = None
|
||||
error: str | None = None
|
||||
@@ -556,6 +560,9 @@ class FlowController:
|
||||
self.disabled: set[str] = set()
|
||||
#: Flows that only run when a run asks them to.
|
||||
self.batch: set[str] = set()
|
||||
#: The key each built flow runs under, by its id. What a rebuild of a
|
||||
#: flow that is gone takes out, since its document can no longer say.
|
||||
self._flow_keys: dict[str, str] = {}
|
||||
#: Each flow's declared inputs, kept so rebuilding one flow can
|
||||
#: validate the whole graph without re-reading every flow off the disk.
|
||||
self._flow_inputs: dict[str, dict[str, bool]] = {}
|
||||
@@ -642,15 +649,16 @@ class FlowController:
|
||||
entry.last_error_ts = None
|
||||
entry.failures_in_a_row = 0
|
||||
|
||||
async def set_enabled(self, flow: str, enabled: bool) -> None:
|
||||
async def set_enabled(self, flow_id: str, flow: str, enabled: bool) -> None:
|
||||
"""Stop or start one flow, without building anything.
|
||||
|
||||
Its nodes are built whether or not it runs — being stopped means
|
||||
having no subscriptions, schedules or webhooks, not being absent — so
|
||||
all this has to do is the lifecycle call and the gate that goes with
|
||||
it. Nothing is compiled and no other flow is touched.
|
||||
it. Nothing is compiled and no other flow is touched. *flow_id* is
|
||||
where that is stored, *flow* the key the gate goes by.
|
||||
"""
|
||||
await run_in_threadpool(self.store.write_enabled, flow, enabled)
|
||||
await run_in_threadpool(self.store.write_enabled, flow_id, enabled)
|
||||
if self.pipeline is None:
|
||||
# Nothing built yet, so there are no nodes to start or stop.
|
||||
await self.reload()
|
||||
@@ -726,10 +734,9 @@ class FlowController:
|
||||
self.supervisor = Supervisor(self.events)
|
||||
|
||||
published = self.store.read_all()
|
||||
self._flow_keys = {flow.id: flow.name for flow in published}
|
||||
self.disabled = {
|
||||
flow.name
|
||||
for flow in published
|
||||
if not self.store.read_enabled(flow.name)
|
||||
flow.name for flow in published if not self.store.read_enabled(flow.id)
|
||||
}
|
||||
# A batch flow is built — the canvas draws it and validation covers
|
||||
# it — but never activated: it runs when a run asks it to, and a
|
||||
@@ -795,7 +802,7 @@ class FlowController:
|
||||
self._rewired()
|
||||
|
||||
async def reload_flow(
|
||||
self, name: str, wait: float | None = FLOW_REBUILD_WAIT
|
||||
self, flow_id: str, wait: float | None = FLOW_REBUILD_WAIT
|
||||
) -> None:
|
||||
"""Rebuild one flow, leaving every other flow's nodes connected.
|
||||
|
||||
@@ -806,7 +813,8 @@ class FlowController:
|
||||
on a populated instance is every node opening its socket again,
|
||||
and only one flow's have changed.
|
||||
|
||||
A flow the store no longer has is taken out instead of replaced.
|
||||
A flow the store no longer has is taken out instead of replaced. The
|
||||
flow is named by its id; everything about running it goes by its key.
|
||||
|
||||
The failure shape is the whole-pipeline one narrowed to a flow, and
|
||||
deliberately no better: a build that raises changes nothing, and
|
||||
@@ -826,11 +834,14 @@ class FlowController:
|
||||
# flow is not running is its own teardown and nothing more. Off
|
||||
# the loop for the same reason the full build is: compiling a
|
||||
# python node waits for a worker slot.
|
||||
built = await run_in_threadpool(self._build_one, name)
|
||||
built = await run_in_threadpool(self._build_one, flow_id)
|
||||
# A flow never built has nothing to take out, so no key either.
|
||||
name = built[0].name if built else self._flow_keys.pop(flow_id, "")
|
||||
await self._teardown(name)
|
||||
|
||||
if built is None:
|
||||
pipeline.remove_flow(name)
|
||||
if name:
|
||||
pipeline.remove_flow(name)
|
||||
self.loaded = {
|
||||
node_id: entry
|
||||
for node_id, entry in self.loaded.items()
|
||||
@@ -843,6 +854,7 @@ class FlowController:
|
||||
self.batch = self.batch - {name}
|
||||
else:
|
||||
flow, nodes, loaded, initial_values, enabled = built
|
||||
self._flow_keys[flow_id] = name
|
||||
pipeline.replace_flow(name, nodes, initial_values)
|
||||
|
||||
# A rebuild is a fresh set of nodes, but not a fresh history:
|
||||
@@ -953,7 +965,7 @@ class FlowController:
|
||||
"""
|
||||
broken = sorted(
|
||||
{
|
||||
entry.flow
|
||||
entry.flow_id
|
||||
for entry in self.loaded.values()
|
||||
if entry.status is NodeStatus.ERROR
|
||||
}
|
||||
@@ -963,7 +975,10 @@ class FlowController:
|
||||
return broken
|
||||
|
||||
async def reload_lib_users(self, ref: str) -> list[str]:
|
||||
"""Rebuild the flows running a shared node whose source has changed."""
|
||||
"""Rebuild the flows running a shared node whose source has changed.
|
||||
|
||||
Returns their ids, which is what `usages` qualifies with.
|
||||
"""
|
||||
usages = self.store.usages(ref, draft=False)
|
||||
flows = sorted({usage.split(".", 1)[0] for usage in usages})
|
||||
for flow in flows:
|
||||
@@ -971,18 +986,18 @@ class FlowController:
|
||||
return flows
|
||||
|
||||
def _build_one(
|
||||
self, name: str
|
||||
self, flow_id: str
|
||||
) -> tuple[FlowDef, list[Node], dict[str, LoadedNode], dict[str, Any], bool] | None:
|
||||
"""Read one published flow and build its nodes. None when it is gone.
|
||||
|
||||
Blocking: this is the part of a rebuild that talks to the worker pool.
|
||||
"""
|
||||
try:
|
||||
flow = self.store.read_flow(name)
|
||||
flow = self.store.read_flow(flow_id)
|
||||
except FlowNotFound:
|
||||
return None
|
||||
nodes, loaded, initial_values, _ = self._build_flows([(flow, False)])
|
||||
return flow, nodes, loaded, initial_values, self.store.read_enabled(name)
|
||||
return flow, nodes, loaded, initial_values, self.store.read_enabled(flow_id)
|
||||
|
||||
def _all_flow_inputs(self) -> dict[str, bool]:
|
||||
"""Every flow's declared inputs, merged the way one build would see them.
|
||||
@@ -1145,7 +1160,7 @@ class FlowController:
|
||||
|
||||
for flow, draft in flows:
|
||||
for node_def in flow.nodes:
|
||||
entry = self._build_node(flow.name, node_def, draft=draft, run=run)
|
||||
entry = self._build_node(flow, node_def, draft=draft, run=run)
|
||||
loaded[entry.id] = entry
|
||||
if entry.node is not None:
|
||||
nodes.append(entry.node)
|
||||
@@ -1157,13 +1172,16 @@ class FlowController:
|
||||
|
||||
def _build_node(
|
||||
self,
|
||||
flow: str,
|
||||
flow_def: FlowDef,
|
||||
node_def: NodeDef,
|
||||
draft: bool = False,
|
||||
run: RunContext | None = None,
|
||||
) -> LoadedNode:
|
||||
"""One node of *flow_def*: its code read by the flow's id, and run,
|
||||
named and wired by its key."""
|
||||
flow, flow_id = flow_def.name, flow_def.id
|
||||
node_id = f"{flow}.{node_def.id}"
|
||||
entry = LoadedNode(id=node_id, flow=flow)
|
||||
entry = LoadedNode(id=node_id, flow=flow, flow_id=flow_id)
|
||||
try:
|
||||
node_type = NODE_TYPES.get(node_def.type)
|
||||
if node_type is None:
|
||||
@@ -1192,19 +1210,21 @@ class FlowController:
|
||||
code = self.store.read_lib_source(node_def.source_ref)
|
||||
else:
|
||||
owner, local = flow, node_def.id
|
||||
code = self.store.read_node_source(flow, node_def.id, draft=draft)
|
||||
code = self.store.read_node_source(
|
||||
flow_id, node_def.id, draft=draft
|
||||
)
|
||||
# The store answers the new-node template when nothing was
|
||||
# written, which is what an editor should open with and not
|
||||
# something to run. A draft legitimately has none yet.
|
||||
entry.missing_source = not draft and not self.store.has_node_source(
|
||||
flow, node_def.id
|
||||
flow_id, node_def.id
|
||||
)
|
||||
|
||||
# Only a body somebody wrote: the template a new node opens
|
||||
# with takes nothing, and a node declared before it is written
|
||||
# is reported as missing its source rather than as wrong.
|
||||
written = bool(node_def.source_ref) or self.store.has_node_source(
|
||||
flow, node_def.id, draft=draft
|
||||
flow_id, node_def.id, draft=draft
|
||||
)
|
||||
if written:
|
||||
gap = _signature_gap(
|
||||
@@ -1376,8 +1396,10 @@ class FlowController:
|
||||
node._on_health = self._health_changed
|
||||
|
||||
if _is_hook(node):
|
||||
# Hooks live under their flow, away from the JSON API.
|
||||
node.url = f"{HOOK_PREFIX}/{flow}/{node.url.lstrip('/')}"
|
||||
# Hooks live under their flow, away from the JSON API: under
|
||||
# its id, so a URL handed out keeps working when the key
|
||||
# changes.
|
||||
node.url = f"{HOOK_PREFIX}/{flow_id}/{node.url.lstrip('/')}"
|
||||
|
||||
entry.node = node
|
||||
except (SecretNotFound, LibNotFound) as exc:
|
||||
@@ -1493,7 +1515,7 @@ class FlowController:
|
||||
issue for issue in self.issues if not issue.flow or issue.flow == flow
|
||||
] + self._health_issues(flow)
|
||||
|
||||
def preview(self, name: str) -> Preview:
|
||||
def preview(self, flow_id: str) -> Preview:
|
||||
"""Build a flow's unpublished draft without deploying it.
|
||||
|
||||
The draft stands in for its published self among the other flows, so
|
||||
@@ -1501,13 +1523,15 @@ class FlowController:
|
||||
graph is thrown away afterwards and seeded into a throwaway state, so
|
||||
nothing here touches what is running.
|
||||
"""
|
||||
flows: list[tuple[FlowDef, bool]] = [
|
||||
(flow, False) for flow in self.store.read_all() if flow.name != name
|
||||
]
|
||||
try:
|
||||
flows.append((self.store.read_flow(name, draft=True), True))
|
||||
draft = self.store.read_flow(flow_id, draft=True)
|
||||
except FlowNotFound:
|
||||
return Preview(nodes=[], issues=[])
|
||||
name = draft.name
|
||||
flows: list[tuple[FlowDef, bool]] = [
|
||||
(flow, False) for flow in self.store.read_all() if flow.id != flow_id
|
||||
]
|
||||
flows.append((draft, True))
|
||||
|
||||
nodes, loaded, initial_values, flow_inputs = self._build_flows(flows)
|
||||
pipeline = Pipeline(
|
||||
@@ -1856,21 +1880,35 @@ class FlowController:
|
||||
return self.execution.stats() if self.execution is not None else {}
|
||||
|
||||
def forget_flow(self, flow: str) -> None:
|
||||
"""Drop what a deleted flow left behind. Blocking."""
|
||||
"""Drop what a deleted flow left behind, by its key. Blocking.
|
||||
|
||||
After its nodes are torn down, or they put some of it straight back.
|
||||
"""
|
||||
if self.execution is not None:
|
||||
self.execution.queue.clear_flow(flow)
|
||||
prefix = f"{flow}."
|
||||
with self.state.lock():
|
||||
stale = [
|
||||
key
|
||||
for key in self.state.keys()
|
||||
# Both the messages themselves and the engine's own bookkeeping
|
||||
# about them, which is keyed by message name too.
|
||||
if key.startswith(prefix) or f":{prefix}" in key
|
||||
]
|
||||
for key in stale:
|
||||
for key in [k for k in self.state.keys() if _in_namespace(k, flow)]:
|
||||
self.state.delete(key)
|
||||
|
||||
def key_residue(self, flow: str) -> list[str]:
|
||||
"""What still lives under the flow key *flow*. Blocking.
|
||||
|
||||
State and live queue work in its namespace: left by a flow that held
|
||||
the key before, or put there by anything still writing to it. A new
|
||||
flow taking the key would inherit it, so creating one is refused while
|
||||
this is not empty, and so is renaming a flow onto it. Work that was
|
||||
acknowledged, dead-lettered or marked done is history and not counted.
|
||||
Names what it found, state keys first.
|
||||
"""
|
||||
found = [key for key in self.state.keys() if _in_namespace(key, flow)]
|
||||
if self.execution is not None:
|
||||
found += [
|
||||
f"queued work of '{item.node}'"
|
||||
for item in self.execution.queue.live_items()
|
||||
if _touches(item, flow)
|
||||
]
|
||||
return found
|
||||
|
||||
def run_flow(self, flow: str, inputs: dict[str, Any] | None = None) -> None:
|
||||
"""Run every node of one flow. Blocking — call from a worker thread."""
|
||||
if self.pipeline is None:
|
||||
@@ -1912,7 +1950,7 @@ class FlowController:
|
||||
pipeline.history_limits = self.history_limits
|
||||
return pipeline
|
||||
|
||||
def run_preview(self, flow_name: str, inputs: dict[str, Any] | None = None) -> None:
|
||||
def run_preview(self, flow_id: str, inputs: dict[str, Any] | None = None) -> None:
|
||||
"""Run a flow's unpublished draft once, as the editor shows it.
|
||||
|
||||
The nodes are built and run but never activated, so a draft never
|
||||
@@ -1920,7 +1958,7 @@ class FlowController:
|
||||
that. State and events are the live ones, so values land on the canvas
|
||||
exactly as they do for a published run.
|
||||
"""
|
||||
flow = self.store.read_flow(flow_name, draft=True)
|
||||
flow = self.store.read_flow(flow_id, draft=True)
|
||||
nodes, _, initial_values, _ = self._build_flows([(flow, True)])
|
||||
pipeline = Pipeline(
|
||||
nodes=nodes,
|
||||
@@ -1960,6 +1998,20 @@ class FlowController:
|
||||
self.events.publish(event)
|
||||
|
||||
|
||||
def _in_namespace(state_key: str, flow: str) -> bool:
|
||||
"""Whether a state key is about the flow key *flow*: one of its messages,
|
||||
or the engine's own bookkeeping on one of its messages or nodes, which is
|
||||
keyed by those names too."""
|
||||
prefix = f"{flow}."
|
||||
return state_key.startswith(prefix) or f":{prefix}" in state_key
|
||||
|
||||
|
||||
def _touches(item: WorkItem, flow: str) -> bool:
|
||||
"""Whether a queued item names anything of the flow key *flow*: as its
|
||||
flow, its node or one of the messages it carries."""
|
||||
return owned_by(item, flow) or any(flow_of(name) == flow for name in item.outputs)
|
||||
|
||||
|
||||
def _bound(specs: list[MessageSpec]) -> list[MessageSpec]:
|
||||
"""Ports without a message name are not wired into the graph."""
|
||||
return [spec for spec in specs if spec.name]
|
||||
|
||||
@@ -28,11 +28,15 @@ from fluksio.flow.messages import DType, qualify
|
||||
from fluksio.flow.schemas import FlowDef, _validate_id, _validate_name
|
||||
from fluksio.flow.store import (
|
||||
FlowStore,
|
||||
IdTaken,
|
||||
StaleVersion,
|
||||
_updated_at,
|
||||
_version,
|
||||
checked,
|
||||
claim,
|
||||
copy_name,
|
||||
named,
|
||||
new_id,
|
||||
read_keys,
|
||||
)
|
||||
|
||||
@@ -619,6 +623,27 @@ class DashboardDef(BaseModel):
|
||||
return shapes
|
||||
|
||||
|
||||
class DashboardCreate(BaseModel):
|
||||
"""A new dashboard: one empty grid under a key, filled in by saves after."""
|
||||
|
||||
#: Empty for an id the server issues, or one a superuser pins, as for a
|
||||
#: flow (`FlowCreate`). Dashboards keep no history, so it only has to be free.
|
||||
id: str = ""
|
||||
name: str
|
||||
#: Empty reads the key as words.
|
||||
title: str = ""
|
||||
|
||||
@field_validator("id")
|
||||
@classmethod
|
||||
def _check_id(cls, value: str) -> str:
|
||||
return _validate_id(value)
|
||||
|
||||
@field_validator("name")
|
||||
@classmethod
|
||||
def _check_name(cls, value: str) -> str:
|
||||
return _validate_name(value)
|
||||
|
||||
|
||||
class DashboardSummary(BaseModel):
|
||||
"""A dashboard in a list, without its contents."""
|
||||
|
||||
@@ -766,14 +791,17 @@ class DashboardStore:
|
||||
self._keys = (at, keys)
|
||||
return keys
|
||||
|
||||
def _claim(self, defn: DashboardDef) -> DashboardDef:
|
||||
def _claim(self, defn: DashboardDef, create: bool = False) -> DashboardDef:
|
||||
"""*defn* with its id, once it is certain to be a save the store allows.
|
||||
|
||||
Under the flow store's write lock; see `claim`. An id left out is the
|
||||
key, which is how every dashboard is created for now.
|
||||
Under the flow store's write lock; see `claim`. A save is to a
|
||||
dashboard that exists, named by its id or, from before ids, its key.
|
||||
Only *create* may make a new one.
|
||||
"""
|
||||
dash_id = checked(defn.id or defn.name)
|
||||
current = self.read(dash_id, draft=True) if self.exists(dash_id) else None
|
||||
if current is None and not create:
|
||||
raise DashboardNotFound(dash_id)
|
||||
claim(
|
||||
dash_id,
|
||||
defn.name,
|
||||
@@ -807,21 +835,50 @@ class DashboardStore:
|
||||
"""Publish a dashboard directly, skipping the draft.
|
||||
|
||||
The API never does: it creates a draft and promotes it. This is for a
|
||||
caller that already has the finished document — a test, or a seed.
|
||||
caller that already has the finished document, a test or a seed, and
|
||||
creates one too, under the id it carries or else its key.
|
||||
"""
|
||||
with self._lock, self.flows._write_lock:
|
||||
defn = self._claim(defn)
|
||||
defn = self._claim(defn, create=True)
|
||||
path = self._file(defn.id)
|
||||
current = 0
|
||||
if path.exists():
|
||||
current = DashboardDef.model_validate_json(path.read_text()).version
|
||||
current = _version(path)
|
||||
if base_version is not None and base_version != current:
|
||||
raise StaleVersion(defn.name, current)
|
||||
|
||||
saved = defn.model_copy(update={"version": current + 1, "has_draft": False})
|
||||
# Above the draft too, which an editor may be holding.
|
||||
top = max(current, _version(self._draft_file(defn.id)))
|
||||
saved = defn.model_copy(update={"version": top + 1, "has_draft": False})
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(self._dump(saved))
|
||||
self.flows._commit(f"Save dashboard '{defn.name}'")
|
||||
self.flows._commit(f"Save dashboard {named(defn.name, defn.id)}")
|
||||
return saved
|
||||
|
||||
def create(
|
||||
self, defn: DashboardDef, pin: str = "", published: bool = False
|
||||
) -> DashboardDef:
|
||||
"""A new dashboard, only a draft unless *published* says otherwise.
|
||||
|
||||
Its id is issued here unless *pin* names one, a superuser's to ask for
|
||||
as for a flow (`FlowStore.create`). Dashboards have no history to
|
||||
adopt, so a pin only has to be free.
|
||||
"""
|
||||
with self._lock, self.flows._write_lock:
|
||||
keys = self.key_index()
|
||||
if pin:
|
||||
dash_id = checked(pin)
|
||||
if (self.root / dash_id).exists():
|
||||
raise IdTaken(dash_id, "dashboard")
|
||||
else:
|
||||
dash_id = new_id("d", lambda c: (self.root / c).exists() or c in keys)
|
||||
defn = self._claim(defn.model_copy(update={"id": dash_id}), create=True)
|
||||
if keys.get(dash_id, dash_id) != dash_id:
|
||||
raise IdTaken(dash_id, "dashboard")
|
||||
|
||||
saved = defn.model_copy(update={"version": 1, "has_draft": not published})
|
||||
path = self._file(dash_id) if published else self._draft_file(dash_id)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(self._dump(saved))
|
||||
self.flows._commit(f"Create dashboard {named(defn.name, dash_id)}")
|
||||
return saved
|
||||
|
||||
def write_draft(
|
||||
@@ -829,23 +886,19 @@ class DashboardStore:
|
||||
) -> DashboardDef:
|
||||
"""Save unpublished changes, refusing to overwrite someone else's.
|
||||
|
||||
``base_version`` is the version the editor last saw — of the working
|
||||
copy, which is the draft once there is one, and 0 for a dashboard that
|
||||
does not exist yet: creating one is its first draft.
|
||||
``base_version`` is the version the editor last saw, of the working
|
||||
copy, which is the draft once there is one. The dashboard has to
|
||||
exist: creating one is `create`.
|
||||
"""
|
||||
with self._lock, self.flows._write_lock:
|
||||
defn = self._claim(defn)
|
||||
current = 0
|
||||
if self.exists(defn.id):
|
||||
current = self.read(defn.id, draft=True).version
|
||||
current = self.read(defn.id, draft=True).version
|
||||
if base_version is not None and base_version != current:
|
||||
raise StaleVersion(defn.name, current)
|
||||
|
||||
saved = defn.model_copy(update={"version": current + 1, "has_draft": True})
|
||||
path = self._draft_file(defn.id)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(self._dump(saved))
|
||||
self.flows._commit(f"Update draft of dashboard '{defn.name}'")
|
||||
self._draft_file(defn.id).write_text(self._dump(saved))
|
||||
self.flows._commit(f"Update draft of dashboard {named(defn.name, defn.id)}")
|
||||
return saved
|
||||
|
||||
def publish(self, name: str, base_version: int | None = None) -> DashboardDef:
|
||||
@@ -859,14 +912,24 @@ class DashboardStore:
|
||||
if draft.exists():
|
||||
self._file(name).write_text(self._dump(current))
|
||||
draft.unlink()
|
||||
self.flows._commit(f"Publish dashboard '{name}'")
|
||||
self.flows._commit(f"Publish dashboard {named(current.name, name)}")
|
||||
return current.model_copy(update={"has_draft": False})
|
||||
|
||||
def discard_draft(self, name: str) -> DashboardDef:
|
||||
"""Throw the unpublished changes away and go back to what is shown."""
|
||||
"""Throw the unpublished changes away and go back to what is shown.
|
||||
|
||||
What is shown comes back a version above the draft, as a flow's does
|
||||
(`FlowStore.discard_draft`).
|
||||
"""
|
||||
with self._lock, self.flows._write_lock:
|
||||
draft = self._draft_file(name)
|
||||
if draft.exists():
|
||||
shown = self.read(name)
|
||||
top = _version(draft)
|
||||
if shown.version <= top:
|
||||
self._file(name).write_text(
|
||||
self._dump(shown.model_copy(update={"version": top + 1}))
|
||||
)
|
||||
draft.unlink()
|
||||
self.flows._commit(f"Discard draft of dashboard '{name}'")
|
||||
return self.read(name)
|
||||
@@ -887,27 +950,32 @@ class DashboardStore:
|
||||
def duplicate(self, name: str) -> DashboardDef:
|
||||
"""Copy the working copy into a new dashboard, as a draft only.
|
||||
|
||||
Named ``{name}_copy``, then ``_copy_2`` and on. Unpublished like a new
|
||||
dashboard, so no panel shows it until someone publishes it.
|
||||
Under an id of its own and the key ``{name}_copy``, then ``_copy_2``
|
||||
and on. Unpublished like a new dashboard, so no panel shows it until
|
||||
someone publishes it.
|
||||
"""
|
||||
with self._lock, self.flows._write_lock:
|
||||
source = self.read(name, draft=True)
|
||||
# The copy's id is its key, so it has to be free as both.
|
||||
keys = self.key_index()
|
||||
new_name = copy_name(source.name, lambda n: self.exists(n) or n in keys)
|
||||
new_name = copy_name(source.name, lambda n: n in keys)
|
||||
copy_id = new_id("d", lambda c: (self.root / c).exists() or c in keys)
|
||||
copy = source.model_copy(
|
||||
update={
|
||||
"id": new_name,
|
||||
"id": copy_id,
|
||||
"name": new_name,
|
||||
"title": f"{source.title} (copy)" if source.title else "",
|
||||
"version": 1,
|
||||
"has_draft": True,
|
||||
}
|
||||
)
|
||||
path = self._draft_file(new_name)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
copy = self._claim(copy, create=True)
|
||||
path = self._draft_file(copy_id)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(self._dump(copy))
|
||||
self.flows._commit(f"Duplicate dashboard '{name}' as '{new_name}'")
|
||||
self.flows._commit(
|
||||
f"Duplicate dashboard {named(source.name, name)} as "
|
||||
f"{named(new_name, copy_id)}"
|
||||
)
|
||||
return copy
|
||||
|
||||
def bindings_for(self, flow: str) -> Bindings:
|
||||
@@ -984,11 +1052,11 @@ class DashboardStore:
|
||||
return limits
|
||||
|
||||
|
||||
def default_dashboard(name: str) -> DashboardDef:
|
||||
"""A new dashboard: one page, one section, nothing in it yet."""
|
||||
def default_dashboard(name: str, title: str = "") -> DashboardDef:
|
||||
"""A new dashboard: one empty grid, titled after its key unless told."""
|
||||
return DashboardDef(
|
||||
name=name,
|
||||
title=name.replace("_", " ").capitalize(),
|
||||
title=title or name.replace("_", " ").capitalize(),
|
||||
)
|
||||
|
||||
|
||||
@@ -1094,6 +1162,7 @@ __all__ = [
|
||||
"INPUT_WIDGETS",
|
||||
"SETTING_DTYPES",
|
||||
"WIDGET_DTYPES",
|
||||
"DashboardCreate",
|
||||
"DashboardDef",
|
||||
"DashboardExists",
|
||||
"DashboardNotFound",
|
||||
|
||||
@@ -445,7 +445,7 @@ class DeviceStore:
|
||||
if self._wiring_at != revision:
|
||||
found: dict[str, list[Binding]] = {}
|
||||
for flow in self.flows.read_all():
|
||||
if not self.flows.read_enabled(flow.name):
|
||||
if not self.flows.read_enabled(flow.id):
|
||||
continue
|
||||
for binding in bindings_of(flow):
|
||||
found.setdefault(binding.device, []).append(binding)
|
||||
|
||||
@@ -24,6 +24,7 @@ from sqlmodel import Session, col
|
||||
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.core.db import engine
|
||||
from fluksio.flow import tombstones
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.models import EngineEvent, FlowRun, MetricBucket
|
||||
|
||||
@@ -456,6 +457,9 @@ class MetricsCollector:
|
||||
session.execute(delete(MetricBucket).where(col(MetricBucket.bucket) < cutoff))
|
||||
session.execute(delete(EngineEvent).where(col(EngineEvent.ts) < cutoff))
|
||||
session.execute(delete(FlowRun).where(col(FlowRun.started_at) < cutoff))
|
||||
# After the rows above: a deleted flow's name is kept as long as any
|
||||
# of its history is.
|
||||
tombstones.prune(session)
|
||||
# A cascade still open long after it started did not finish; saying so
|
||||
# is more honest than leaving it running forever. A batch run is not
|
||||
# swept: it is allowed to take hours, and `Run` has its own lease sweep
|
||||
|
||||
@@ -161,7 +161,7 @@ class HttpNode(Node):
|
||||
|
||||
url: str = Field(
|
||||
description=(
|
||||
"A path such as /tick, served as a webhook at /hooks/<flow>/<path>, "
|
||||
"A path such as /tick, served as a webhook at /hooks/<flow id>/<path>, "
|
||||
"or the full URL a sender requests."
|
||||
)
|
||||
)
|
||||
@@ -197,7 +197,7 @@ class HttpNode(Node):
|
||||
default="",
|
||||
description=(
|
||||
"Shared secret for a webhook, appended to its URL: "
|
||||
"/hooks/<flow>/<url>/<secret>. Empty leaves the webhook open "
|
||||
"/hooks/<flow id>/<url>/<secret>. Empty leaves the webhook open "
|
||||
"to anyone."
|
||||
),
|
||||
json_schema_extra={"x-secret": True},
|
||||
|
||||
@@ -31,7 +31,7 @@ class WebSocketNode(Node):
|
||||
|
||||
The URL picks the role: ``ws://`` or ``wss://`` dials out and holds the
|
||||
connection, a path such as ``/ticker`` is served at
|
||||
``/hooks/<flow>/<path>[/<secret>]`` while the flow runs. The ports pick the
|
||||
``/hooks/<flow id>/<path>[/<secret>]`` while the flow runs. The ports pick the
|
||||
direction, as on the mqtt node: outputs only make a **listener**, inputs a
|
||||
**sender**.
|
||||
|
||||
@@ -79,7 +79,7 @@ class WebSocketNode(Node):
|
||||
url: str = Field(
|
||||
description=(
|
||||
"ws:// or wss:// to connect to a server, or a path such as "
|
||||
"/ticker to serve one at /hooks/<flow>/<path>."
|
||||
"/ticker to serve one at /hooks/<flow id>/<path>."
|
||||
)
|
||||
)
|
||||
headers: dict[str, str] = Field(
|
||||
@@ -100,7 +100,7 @@ class WebSocketNode(Node):
|
||||
default="",
|
||||
description=(
|
||||
"Shared secret for a served socket, appended to its URL: "
|
||||
"/hooks/<flow>/<url>/<secret>. Empty leaves it open to anyone."
|
||||
"/hooks/<flow id>/<url>/<secret>. Empty leaves it open to anyone."
|
||||
),
|
||||
json_schema_extra={"x-secret": True},
|
||||
)
|
||||
|
||||
@@ -151,6 +151,17 @@ class WorkItem:
|
||||
)
|
||||
|
||||
|
||||
def owned_by(item: WorkItem, flow: str) -> bool:
|
||||
"""Whether *item* is work of the flow with key *flow*, not one reaching it.
|
||||
|
||||
A run item names its run's flow by identity rather than by key, and a
|
||||
run is never removed with a flow's queued work, so it is nobody's here.
|
||||
"""
|
||||
return item.kind != "run" and (
|
||||
item.flow == flow or item.node.split(".", 1)[0] == flow
|
||||
)
|
||||
|
||||
|
||||
class WorkQueue(ABC):
|
||||
"""What the execution service pulls from."""
|
||||
|
||||
@@ -236,7 +247,21 @@ class WorkQueue(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def clear_flow(self, flow: str) -> None:
|
||||
"""Forget anything held for a flow that no longer exists."""
|
||||
"""Drop the work of a flow that no longer exists (`owned_by`).
|
||||
|
||||
What it parked, what waits on a timer, and what is queued or claimed
|
||||
and not yet acknowledged, so a new flow under the same key never runs
|
||||
it. A cascade already running finishes; its acknowledgement then has
|
||||
nothing left to remove.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def live_items(self) -> list[WorkItem]:
|
||||
"""Every item still to run: queued, claimed, delayed or parked.
|
||||
|
||||
Not what was acknowledged, dead-lettered or marked done, which is
|
||||
history. Reads the whole queue, so it is for rare questions.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def stats(self) -> dict[str, Any]:
|
||||
@@ -388,6 +413,23 @@ class MemoryWorkQueue(WorkQueue):
|
||||
def clear_flow(self, flow: str) -> None:
|
||||
with self._lock:
|
||||
self._parked.pop(flow, None)
|
||||
for lane in (self._items, self._due):
|
||||
kept = [item for item in lane if not owned_by(item, flow)]
|
||||
lane.clear()
|
||||
lane.extend(kept)
|
||||
self._delayed = [e for e in self._delayed if not owned_by(e[2], flow)]
|
||||
heapq.heapify(self._delayed)
|
||||
|
||||
def live_items(self) -> list[WorkItem]:
|
||||
"""What is waiting. A claimed item is gone from here once it is handed
|
||||
out, so unlike Redis this cannot say what is running."""
|
||||
with self._lock:
|
||||
return [
|
||||
*self._items,
|
||||
*self._due,
|
||||
*(entry[2] for entry in self._delayed),
|
||||
*(item for held in self._parked.values() for item in held),
|
||||
]
|
||||
|
||||
def backlog(self) -> int:
|
||||
with self._lock:
|
||||
@@ -683,8 +725,68 @@ class RedisWorkQueue(WorkQueue):
|
||||
pipe = self._redis.pipeline()
|
||||
pipe.delete(self._parked_key(flow))
|
||||
pipe.srem(self._parked_flows_key, flow)
|
||||
# Timers before the streams: one promoted in between is in the due
|
||||
# stream by the time that is read.
|
||||
for raw in cast(list[str], self._redis.zrange(self._delayed_key, 0, -1)):
|
||||
if owned_by(WorkItem.from_fields(orjson.loads(raw), ""), flow):
|
||||
pipe.zrem(self._delayed_key, raw)
|
||||
for stream in (self._stream, self._due_stream):
|
||||
ids = [
|
||||
entry_id
|
||||
for entry_id, fields in self._live_entries(stream)
|
||||
if owned_by(WorkItem.from_fields(fields, entry_id), flow)
|
||||
]
|
||||
if ids:
|
||||
pipe.xack(stream, GROUP, *ids)
|
||||
pipe.xdel(stream, *ids)
|
||||
pipe.execute()
|
||||
|
||||
def _live_entries(self, stream: str) -> list[tuple[str, dict[str, str]]]:
|
||||
"""The entries of *stream* still to run: claimed and unacknowledged,
|
||||
then never handed out. Acknowledged ones stay in a stream until it is
|
||||
trimmed, and are history."""
|
||||
group = self._group_info(stream)
|
||||
if not group:
|
||||
return []
|
||||
last = str(group.get("last-delivered-id") or "0-0")
|
||||
live: list[tuple[str, dict[str, str]]] = []
|
||||
pending = int(group.get("pending") or 0)
|
||||
if pending:
|
||||
rows = cast(
|
||||
list[dict[str, Any]],
|
||||
self._redis.xpending_range(
|
||||
stream, GROUP, min="-", max="+", count=pending
|
||||
),
|
||||
)
|
||||
ids = {row["message_id"] for row in rows}
|
||||
if ids:
|
||||
# In id order, so the first is the oldest still held.
|
||||
entries = self._redis.xrange(
|
||||
stream, min=rows[0]["message_id"], max=last
|
||||
)
|
||||
live += [(i, f) for i, f in cast(Any, entries) if i in ids]
|
||||
live += cast(Any, self._redis.xrange(stream, min=f"({last}", max="+"))
|
||||
return live
|
||||
|
||||
def live_items(self) -> list[WorkItem]:
|
||||
items = [
|
||||
WorkItem.from_fields(fields, prefix + entry_id)
|
||||
for stream, prefix in ((self._due_stream, DUE_PREFIX), (self._stream, ""))
|
||||
for entry_id, fields in self._live_entries(stream)
|
||||
]
|
||||
items += [
|
||||
WorkItem.from_fields(orjson.loads(raw), "")
|
||||
for raw in cast(list[str], self._redis.zrange(self._delayed_key, 0, -1))
|
||||
]
|
||||
for flow in cast(set[str], self._redis.smembers(self._parked_flows_key)):
|
||||
items += [
|
||||
WorkItem.from_fields(orjson.loads(raw), "")
|
||||
for raw in cast(
|
||||
list[str], self._redis.lrange(self._parked_key(flow), 0, -1)
|
||||
)
|
||||
]
|
||||
return items
|
||||
|
||||
def _group_info(self, stream: str = "") -> dict[str, Any]:
|
||||
"""This consumer group's row of ``XINFO GROUPS``, or an empty one."""
|
||||
try:
|
||||
|
||||
@@ -155,7 +155,7 @@ def batch_issues(
|
||||
"the value would be dropped. Remove the interval, or mark "
|
||||
"the port as streaming if it is a curve being thinned out."
|
||||
)
|
||||
if store is not None and _has_no_body(store, flow.name, node, draft):
|
||||
if store is not None and _has_no_body(store, flow.id, node, draft):
|
||||
issues.append(
|
||||
f"Node '{node.id}' has no stored code. It would run as an "
|
||||
"empty node and publish nothing, so the run is refused."
|
||||
|
||||
@@ -352,6 +352,36 @@ class FlowDef(BaseModel):
|
||||
return _validate_name(value)
|
||||
|
||||
|
||||
class FlowCreate(BaseModel):
|
||||
"""A new flow as a client hands it over: a document but for what the
|
||||
server owns. The fields mean what they do on `FlowDef`, but ``id``."""
|
||||
|
||||
#: Empty for an id the server issues. Anything else pins it, which only a
|
||||
#: superuser may: an id that is free, or whose history this flow resumes.
|
||||
id: str = ""
|
||||
name: str
|
||||
title: str = ""
|
||||
nodes: list[NodeDef] = Field(default_factory=list)
|
||||
inputs: list[FlowInput] = Field(default_factory=list)
|
||||
mode: Literal["live", "batch"] = "live"
|
||||
outputs: list[str] = Field(default_factory=list)
|
||||
origin: FlowOrigin | None = None
|
||||
|
||||
@field_validator("id")
|
||||
@classmethod
|
||||
def _check_id(cls, value: str) -> str:
|
||||
return _validate_id(value)
|
||||
|
||||
@field_validator("name")
|
||||
@classmethod
|
||||
def _check_name(cls, value: str) -> str:
|
||||
return _validate_name(value)
|
||||
|
||||
def document(self) -> FlowDef:
|
||||
"""The flow to store, before the store gives it an id."""
|
||||
return FlowDef.model_validate(self.model_dump(exclude={"id"}))
|
||||
|
||||
|
||||
class NodeSource(BaseModel):
|
||||
"""The Python source of a node."""
|
||||
|
||||
|
||||
+208
-68
@@ -14,8 +14,9 @@ unpublished — which is what every flow written before this existed looks like.
|
||||
A flow is two names. Its id is which flow it is: the directory, the argument
|
||||
every method here takes, and never changed once given. Its key, ``name``, is
|
||||
the namespace of its messages and unique among flows. A flow from before ids
|
||||
existed has its key as its id (`fluksio.flow.identity`), and so does every new
|
||||
one for now. Published and draft documents always share both.
|
||||
existed has its key as its id (`fluksio.flow.identity`). A new one is given an
|
||||
opaque id by `create`, or the one a superuser pinned. Published and draft
|
||||
documents always share both.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -31,10 +32,13 @@ import threading
|
||||
from collections.abc import Callable, Iterable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
from fluksio.flow.schemas import FlowDef, _validate_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fluksio.flow.tombstones import Tombstones
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: Where the sources of nodes shared between flows live.
|
||||
@@ -97,6 +101,37 @@ class IdentityChange(ValueError):
|
||||
"""
|
||||
|
||||
|
||||
class RecordedKey(IdentityChange):
|
||||
"""A pinned id restored under another key than its history is filed under.
|
||||
|
||||
The history names the old key's messages, so taking a new key with it is
|
||||
a rename, not a create.
|
||||
"""
|
||||
|
||||
def __init__(self, obj_id: str, key: str) -> None:
|
||||
super().__init__(obj_id)
|
||||
self.id = obj_id
|
||||
self.key = key
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (
|
||||
f"'{self.id}' has history filed under the key '{self.key}', so it "
|
||||
"can only be restored with that key"
|
||||
)
|
||||
|
||||
|
||||
class IdTaken(ValueError):
|
||||
"""A pinned id that is in use: an object's, a directory's or another key."""
|
||||
|
||||
def __init__(self, obj_id: str, kind: str = "flow") -> None:
|
||||
super().__init__(obj_id)
|
||||
self.id = obj_id
|
||||
self.kind = kind
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"The id '{self.id}' is already taken by a {self.kind}"
|
||||
|
||||
|
||||
def claim(
|
||||
obj_id: str,
|
||||
key: str,
|
||||
@@ -146,6 +181,11 @@ def read_keys(directories: Iterable[Path], document: str) -> dict[str, str]:
|
||||
return keys
|
||||
|
||||
|
||||
def named(key: str, obj_id: str) -> str:
|
||||
"""How a commit message names an object: by key, and by id when it differs."""
|
||||
return f"'{key}'" if key == obj_id else f"'{key}' ({obj_id})"
|
||||
|
||||
|
||||
def new_id(kind: Literal["f", "d"], taken: Callable[[str], bool]) -> str:
|
||||
"""A fresh id for a flow (``f``) or a dashboard (``d``), e.g. ``f_k3q7m2xa``.
|
||||
|
||||
@@ -222,6 +262,14 @@ def _updated_at(paths: Iterable[Path]) -> datetime | None:
|
||||
return datetime.fromtimestamp(max(stamps), UTC) if stamps else None
|
||||
|
||||
|
||||
def _version(path: Path) -> int:
|
||||
"""The version a stored document carries, 0 for none or an unreadable one."""
|
||||
try:
|
||||
return int(json.loads(path.read_text()).get("version", 1))
|
||||
except (OSError, ValueError, TypeError, AttributeError):
|
||||
return 0
|
||||
|
||||
|
||||
def _same_content(left: FlowDef, right: FlowDef) -> bool:
|
||||
"""Equal but for the version counter, which the server owns."""
|
||||
return left.model_copy(update={"version": 0}) == right.model_copy(
|
||||
@@ -252,8 +300,11 @@ class FlowStore:
|
||||
#: a store reached 1898 commits, 5863 loose objects and no pack at all.
|
||||
_GC_EVERY = 500
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
def __init__(self, root: Path, tombstones: Tombstones | None = None) -> None:
|
||||
self.root = root
|
||||
#: Where a deleted flow's key is kept, and what history an id has.
|
||||
#: None leaves both out: a store without a database, as in tests.
|
||||
self.tombstones = tombstones
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
# Draft writes are check-and-set, so two clients saving at once must not
|
||||
# interleave between reading the current version and writing the next.
|
||||
@@ -679,14 +730,16 @@ class FlowStore:
|
||||
self._keys = (at, keys)
|
||||
return keys
|
||||
|
||||
def _claim(self, flow: FlowDef) -> FlowDef:
|
||||
def _claim(self, flow: FlowDef, create: bool = False) -> FlowDef:
|
||||
"""*flow* with its id, once it is certain to be a save the store allows.
|
||||
|
||||
Under `_write_lock`. An id left out is the key, which is how every
|
||||
flow is created for now.
|
||||
Under `_write_lock`. A save is to a flow that exists, named by its id
|
||||
or, from before ids, by its key. Only *create* may make a new one.
|
||||
"""
|
||||
flow_id = checked(flow.id or flow.name)
|
||||
current = self.read_flow(flow_id, draft=True) if self.exists(flow_id) else None
|
||||
if current is None and not create:
|
||||
raise FlowNotFound(flow_id)
|
||||
claim(
|
||||
flow_id,
|
||||
flow.name,
|
||||
@@ -696,6 +749,22 @@ class FlowStore:
|
||||
)
|
||||
return flow.model_copy(update={"id": flow_id})
|
||||
|
||||
def _issue(self, keys: dict[str, str]) -> str:
|
||||
"""A fresh flow id. Under `_write_lock`, with *keys* the key index.
|
||||
|
||||
Free as a directory, a key and in history: an id a deleted flow's runs
|
||||
are still filed under would hand them to the new one.
|
||||
"""
|
||||
graves = self.tombstones
|
||||
return new_id(
|
||||
"f",
|
||||
lambda c: (
|
||||
self._flow_dir(c).exists()
|
||||
or c in keys
|
||||
or (graves is not None and graves.recorded(c) is not None)
|
||||
),
|
||||
)
|
||||
|
||||
def read_all(self) -> list[FlowDef]:
|
||||
"""Every published flow — what the engine runs.
|
||||
|
||||
@@ -718,9 +787,14 @@ class FlowStore:
|
||||
return list(self._read_all)
|
||||
|
||||
def write_flow(self, flow: FlowDef) -> bool:
|
||||
"""Publish a flow directly. Returns False when nothing actually changed."""
|
||||
"""Publish a flow directly. Returns False when nothing actually changed.
|
||||
|
||||
Creates one too, under the id it carries or else its key: what a test
|
||||
or a seed holding a finished document wants. The API creates only
|
||||
through `create`.
|
||||
"""
|
||||
with self._write_lock:
|
||||
flow = self._claim(flow)
|
||||
flow = self._claim(flow, create=True)
|
||||
path = self._flow_file(flow.id)
|
||||
content = flow.model_dump_json(indent=2) + "\n"
|
||||
if path.exists() and path.read_text() == content:
|
||||
@@ -728,122 +802,188 @@ class FlowStore:
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content)
|
||||
self._commit(f"Update flow '{flow.name}'")
|
||||
self._commit(f"Update flow {named(flow.name, flow.id)}")
|
||||
return True
|
||||
|
||||
def create(self, flow: FlowDef, pin: str = "") -> FlowDef:
|
||||
"""A new flow, only a draft until someone publishes it.
|
||||
|
||||
Its id is issued here unless *pin* names one, which is a superuser's
|
||||
to ask for: the Demo, a test, a restore. A pinned id whose history
|
||||
outlived its flow adopts that history, so it has to take the key the
|
||||
history is filed under, and its tombstone goes once the document is on
|
||||
disk.
|
||||
"""
|
||||
with self._write_lock:
|
||||
keys = self.key_index()
|
||||
if pin:
|
||||
flow_id = checked(pin)
|
||||
# The directory rather than `exists`: anything left in it would
|
||||
# become the new flow's.
|
||||
if self._flow_dir(flow_id).exists():
|
||||
raise IdTaken(flow_id)
|
||||
recorded = (
|
||||
self.tombstones.recorded(flow_id) if self.tombstones else None
|
||||
)
|
||||
if recorded is not None and recorded != flow.name:
|
||||
raise RecordedKey(flow_id, recorded)
|
||||
else:
|
||||
flow_id = self._issue(keys)
|
||||
flow = self._claim(flow.model_copy(update={"id": flow_id}), create=True)
|
||||
# Another flow's key as this one's id would make naming either
|
||||
# ambiguous. Checked after the key, which is the likelier mistake.
|
||||
if keys.get(flow_id, flow_id) != flow_id:
|
||||
raise IdTaken(flow_id)
|
||||
|
||||
stored = flow.model_copy(update={"version": 1})
|
||||
path = self._draft_file(flow_id)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(stored.model_dump_json(indent=2) + "\n")
|
||||
self._commit(f"Create flow {named(flow.name, flow_id)}")
|
||||
if pin and self.tombstones is not None:
|
||||
self.tombstones.lift(flow_id)
|
||||
return stored
|
||||
|
||||
def write_draft(self, flow: FlowDef, base_version: int) -> FlowDef:
|
||||
"""Save unpublished changes, refusing to overwrite someone else's.
|
||||
|
||||
``base_version`` is the version the editor last saw. It has to match
|
||||
what is stored, otherwise another client saved in between and this
|
||||
write would silently discard their work.
|
||||
write would silently discard their work. The flow has to exist:
|
||||
creating one is `create`.
|
||||
"""
|
||||
with self._write_lock:
|
||||
flow = self._claim(flow)
|
||||
current: FlowDef | None = None
|
||||
if self.exists(flow.id):
|
||||
current = self.read_flow(flow.id, draft=True)
|
||||
if base_version != current.version:
|
||||
raise StaleVersion(flow.name, current.version)
|
||||
if _same_content(current, flow):
|
||||
return current
|
||||
current = self.read_flow(flow.id, draft=True)
|
||||
if base_version != current.version:
|
||||
raise StaleVersion(flow.name, current.version)
|
||||
if _same_content(current, flow):
|
||||
return current
|
||||
|
||||
stored = flow.model_copy(
|
||||
update={"version": (current.version if current else 0) + 1}
|
||||
)
|
||||
stored = flow.model_copy(update={"version": current.version + 1})
|
||||
path = self._draft_file(flow.id)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(stored.model_dump_json(indent=2) + "\n")
|
||||
self._commit(f"Update draft of flow '{flow.name}'")
|
||||
self._commit(f"Update draft of flow {named(flow.name, flow.id)}")
|
||||
return stored
|
||||
|
||||
def publish_flow(self, name: str, base_version: int) -> FlowDef:
|
||||
def publish_flow(self, flow_id: str, base_version: int) -> FlowDef:
|
||||
"""Promote the working copy to what the engine runs."""
|
||||
with self._write_lock:
|
||||
current = self.read_flow(name, draft=True)
|
||||
current = self.read_flow(flow_id, draft=True)
|
||||
if base_version != current.version:
|
||||
raise StaleVersion(name, current.version)
|
||||
raise StaleVersion(current.name, current.version)
|
||||
|
||||
draft = self._draft_file(name)
|
||||
draft = self._draft_file(flow_id)
|
||||
if draft.exists():
|
||||
self._flow_file(name).write_text(
|
||||
self._flow_file(flow_id).write_text(
|
||||
current.model_dump_json(indent=2) + "\n"
|
||||
)
|
||||
draft.unlink()
|
||||
|
||||
drafts = self._draft_nodes_dir(name)
|
||||
drafts = self._draft_nodes_dir(flow_id)
|
||||
if drafts.exists():
|
||||
target = self._flow_dir(name) / "nodes"
|
||||
target = self._flow_dir(flow_id) / "nodes"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for source in sorted(drafts.glob("*.py")):
|
||||
shutil.copy2(source, target / source.name)
|
||||
shutil.rmtree(drafts)
|
||||
|
||||
self._commit(f"Publish flow '{name}'")
|
||||
self._commit(f"Publish flow {named(current.name, flow_id)}")
|
||||
return current
|
||||
|
||||
def discard_draft(self, name: str) -> FlowDef:
|
||||
"""Throw the unpublished changes away and go back to what is running."""
|
||||
with self._write_lock:
|
||||
draft = self._draft_file(name)
|
||||
if draft.exists():
|
||||
draft.unlink()
|
||||
drafts = self._draft_nodes_dir(name)
|
||||
if drafts.exists():
|
||||
shutil.rmtree(drafts)
|
||||
self._commit(f"Discard draft of flow '{name}'")
|
||||
return self.read_flow(name)
|
||||
def discard_draft(self, flow_id: str) -> FlowDef:
|
||||
"""Throw the unpublished changes away and go back to what is running.
|
||||
|
||||
def delete_flow(self, name: str) -> None:
|
||||
with self._write_lock:
|
||||
directory = self._flow_dir(name)
|
||||
if not directory.exists():
|
||||
raise FlowNotFound(name)
|
||||
shutil.rmtree(directory)
|
||||
self._commit(f"Delete flow '{name}'")
|
||||
|
||||
def duplicate_flow(self, name: str) -> FlowDef:
|
||||
"""Copy a flow's working copy into a new flow that is only a draft.
|
||||
|
||||
Like a new flow, the copy runs nothing until someone publishes it. It is
|
||||
named ``{name}_copy``, then ``_copy_2`` and on. ``origin`` is dropped,
|
||||
since the copy is a local flow ``fluksio sync`` must not claim, and
|
||||
``runtime.json`` stays behind with everything else about running.
|
||||
What is running comes back a version above the draft, so an editor
|
||||
still holding the draft is refused rather than matching the next save
|
||||
made from the published copy.
|
||||
"""
|
||||
with self._write_lock:
|
||||
source = self.read_flow(name, draft=True)
|
||||
# The directory rather than `exists`: anything left in it would
|
||||
# otherwise end up in the copy. The copy's id is its key, so that
|
||||
# has to be free among the keys too.
|
||||
draft = self._draft_file(flow_id)
|
||||
if draft.exists():
|
||||
running = self.read_flow(flow_id)
|
||||
top = _version(draft)
|
||||
if running.version <= top:
|
||||
running.version = top + 1
|
||||
self._flow_file(flow_id).write_text(
|
||||
running.model_dump_json(indent=2) + "\n"
|
||||
)
|
||||
draft.unlink()
|
||||
drafts = self._draft_nodes_dir(flow_id)
|
||||
if drafts.exists():
|
||||
shutil.rmtree(drafts)
|
||||
published = self.read_flow(flow_id)
|
||||
self._commit(f"Discard draft of flow {named(published.name, flow_id)}")
|
||||
return published
|
||||
|
||||
def delete_flow(self, flow_id: str) -> str:
|
||||
"""Remove a flow's files. Returns the key it had.
|
||||
|
||||
The key is buried first, so a delete that stops half-way never leaves
|
||||
history without a name, and the live document wins while it is there.
|
||||
"""
|
||||
with self._write_lock:
|
||||
directory = self._flow_dir(flow_id)
|
||||
if not directory.exists():
|
||||
raise FlowNotFound(flow_id)
|
||||
key = next(
|
||||
(k for k, i in self.key_index().items() if i == flow_id), flow_id
|
||||
)
|
||||
if self.tombstones is not None:
|
||||
self.tombstones.bury(flow_id, key)
|
||||
shutil.rmtree(directory)
|
||||
self._commit(f"Delete flow {named(key, flow_id)}")
|
||||
return key
|
||||
|
||||
def duplicate_flow(
|
||||
self, flow_id: str, busy: Callable[[str], bool] | None = None
|
||||
) -> FlowDef:
|
||||
"""Copy a flow's working copy into a new flow that is only a draft.
|
||||
|
||||
Like a new flow, the copy runs nothing until someone publishes it. It
|
||||
gets an id of its own and the key ``{name}_copy``, then ``_copy_2`` and
|
||||
on, past any key *busy* says something still holds. ``origin`` is
|
||||
dropped, since the copy is a local flow ``fluksio sync`` must not
|
||||
claim, and ``runtime.json`` stays behind with everything else about
|
||||
running.
|
||||
"""
|
||||
with self._write_lock:
|
||||
source = self.read_flow(flow_id, draft=True)
|
||||
keys = self.key_index()
|
||||
new_name = copy_name(
|
||||
source.name, lambda n: self._flow_dir(n).exists() or n in keys
|
||||
source.name, lambda n: n in keys or (busy is not None and busy(n))
|
||||
)
|
||||
|
||||
copy = source.model_copy(
|
||||
update={
|
||||
"id": new_name,
|
||||
"id": self._issue(keys),
|
||||
"name": new_name,
|
||||
"title": f"{source.title} (copy)" if source.title else "",
|
||||
"version": 1,
|
||||
"origin": None,
|
||||
}
|
||||
)
|
||||
copy = self._claim(copy, create=True)
|
||||
# Its own messages named with its own key are the copy's now.
|
||||
self._retarget(copy, f"{source.name}.", f"{new_name}.")
|
||||
path = self._draft_file(new_name)
|
||||
path = self._draft_file(copy.id)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(copy.model_dump_json(indent=2) + "\n")
|
||||
|
||||
# The bodies the source's editor shows: published, edits over them.
|
||||
# As drafts, so publishing the copy promotes them like a new flow's.
|
||||
target = self._draft_nodes_dir(new_name)
|
||||
for folder in (self._flow_dir(name) / "nodes", self._draft_nodes_dir(name)):
|
||||
target = self._draft_nodes_dir(copy.id)
|
||||
for folder in (
|
||||
self._flow_dir(flow_id) / "nodes",
|
||||
self._draft_nodes_dir(flow_id),
|
||||
):
|
||||
for body in sorted(folder.glob("*.py")):
|
||||
target.mkdir(exist_ok=True)
|
||||
shutil.copy2(body, target / body.name)
|
||||
|
||||
self._commit(f"Duplicate flow '{name}' as '{new_name}'")
|
||||
self._commit(
|
||||
f"Duplicate flow {named(source.name, flow_id)} as "
|
||||
f"{named(new_name, copy.id)}"
|
||||
)
|
||||
return copy
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""What is left of a deleted flow: its last key, while its history is kept.
|
||||
|
||||
A flow's runs, events and metrics are filed under its id and outlive it. A new
|
||||
flow's id is opaque, so a tombstone keeps the key it last had, and `names_of`
|
||||
reads that back for anything showing the history. A flow deleted before
|
||||
tombstones existed has its old key as its id, which already reads right.
|
||||
|
||||
The flow store is the one writer: it buries a flow before removing its files,
|
||||
asks before issuing an id, and lifts the tombstone of a flow restored under a
|
||||
pinned id once the restored document is on disk.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import delete, or_
|
||||
from sqlmodel import Session, col, select
|
||||
|
||||
from fluksio.core.db import engine
|
||||
from fluksio.models import EngineEvent, FlowRun, FlowTombstone, MetricBucket, Run
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fluksio.flow.store import FlowStore
|
||||
|
||||
|
||||
def referenced(session: Session, flow_id: str) -> bool:
|
||||
"""Whether any history row names *flow_id*, as its flow or a node's.
|
||||
|
||||
A training run's ``for_node`` names the node it trains, in a flow other
|
||||
than the run's own, so the four ``flow`` columns are not enough.
|
||||
"""
|
||||
nodes = f"{flow_id}."
|
||||
queries = [
|
||||
select(Run.id).where(
|
||||
or_(
|
||||
col(Run.flow) == flow_id,
|
||||
col(Run.for_node).startswith(nodes, autoescape=True),
|
||||
)
|
||||
),
|
||||
select(FlowRun.id).where(col(FlowRun.flow) == flow_id),
|
||||
select(MetricBucket.flow).where(
|
||||
or_(
|
||||
col(MetricBucket.flow) == flow_id,
|
||||
col(MetricBucket.node).startswith(nodes, autoescape=True),
|
||||
)
|
||||
),
|
||||
select(EngineEvent.id).where(
|
||||
or_(
|
||||
col(EngineEvent.flow) == flow_id,
|
||||
col(EngineEvent.node).startswith(nodes, autoescape=True),
|
||||
)
|
||||
),
|
||||
]
|
||||
return any(session.exec(query.limit(1)).first() is not None for query in queries)
|
||||
|
||||
|
||||
class Tombstones:
|
||||
"""Tombstones and the history behind an id, as the flow store asks."""
|
||||
|
||||
def bury(self, flow_id: str, name: str) -> None:
|
||||
"""Keep *name* for *flow_id*, before its files go.
|
||||
|
||||
Written again by a retried delete, which may find another name if the
|
||||
first attempt never got as far as removing anything.
|
||||
"""
|
||||
with Session(engine) as session:
|
||||
stone = session.get(FlowTombstone, flow_id) or FlowTombstone(
|
||||
id=flow_id, name=name
|
||||
)
|
||||
stone.name = name
|
||||
session.add(stone)
|
||||
session.commit()
|
||||
|
||||
def lift(self, flow_id: str) -> None:
|
||||
"""Forget the tombstone of a flow that is live again."""
|
||||
with Session(engine) as session:
|
||||
session.execute(
|
||||
delete(FlowTombstone).where(col(FlowTombstone.id) == flow_id)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
def recorded(self, flow_id: str) -> str | None:
|
||||
"""The key the history of a flow that is not live is filed under.
|
||||
|
||||
Its tombstone's, or the id itself for history from before tombstones,
|
||||
whose id was its key. None when nothing names the id at all.
|
||||
"""
|
||||
with Session(engine) as session:
|
||||
stone = session.get(FlowTombstone, flow_id)
|
||||
if stone is not None:
|
||||
return stone.name
|
||||
return flow_id if referenced(session, flow_id) else None
|
||||
|
||||
|
||||
def names_of(ids: Iterable[str], store: FlowStore) -> dict[str, str]:
|
||||
"""Each flow id's key: the live flow's, else its tombstone's, else the id.
|
||||
|
||||
One read of the key index and one query, however many ids a response
|
||||
carries.
|
||||
"""
|
||||
wanted = set(ids)
|
||||
live = {flow_id: key for key, flow_id in store.key_index().items()}
|
||||
names = {flow_id: live[flow_id] for flow_id in wanted if flow_id in live}
|
||||
gone = wanted - names.keys()
|
||||
if gone:
|
||||
with Session(engine) as session:
|
||||
for stone in session.exec(
|
||||
select(FlowTombstone).where(col(FlowTombstone.id).in_(gone))
|
||||
):
|
||||
names[stone.id] = stone.name
|
||||
return {flow_id: names.get(flow_id, flow_id) for flow_id in wanted}
|
||||
|
||||
|
||||
def name_of(flow_id: str, store: FlowStore) -> str:
|
||||
"""One flow id's key, as `names_of`."""
|
||||
return names_of([flow_id], store)[flow_id]
|
||||
|
||||
|
||||
def prune(session: Session) -> int:
|
||||
"""Drop the tombstones no history row names any more. Returns how many."""
|
||||
stale = [
|
||||
stone.id
|
||||
for stone in session.exec(select(FlowTombstone))
|
||||
if not referenced(session, stone.id)
|
||||
]
|
||||
if stale:
|
||||
session.execute(delete(FlowTombstone).where(col(FlowTombstone.id).in_(stale)))
|
||||
return len(stale)
|
||||
@@ -60,6 +60,7 @@ from fluksio.flow.runs import RUN_STATE_TTL, RunService, sweep_artifacts
|
||||
from fluksio.flow.secrets import init_secrets
|
||||
from fluksio.flow.state import MemoryState, RedisState, StateBackend
|
||||
from fluksio.flow.store import FlowStore, IdentityChange, InvalidName
|
||||
from fluksio.flow.tombstones import Tombstones
|
||||
from fluksio.flow.watchdog import LoopWatchdog
|
||||
from fluksio.flow.workers import PythonWorkerPool
|
||||
|
||||
@@ -69,7 +70,7 @@ logger = logging.getLogger(__name__)
|
||||
#: worker's `?token=`, the pairing poll's `?secret=`.
|
||||
_CREDENTIAL_IN_QUERY = re.compile(r"([?&](?:[\w-]*token|[\w-]*secret|ticket)=)[^&\s]+")
|
||||
|
||||
#: A hook's secret, the last segment of `/hooks/<flow>/<path>/<secret>`. A hook
|
||||
#: A hook's secret, the last segment of `/hooks/<flow id>/<path>/<secret>`. A hook
|
||||
#: without one loses its last path segment from the log too, since the log
|
||||
#: cannot tell which hooks carry a secret.
|
||||
_SECRET_IN_HOOK_PATH = re.compile(r"(/hooks/[^/\s?]+/(?:[^/\s?]*/)*)[^/\s?]+")
|
||||
@@ -306,7 +307,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
# store's own `_migrate`, the device migrations below and every cache
|
||||
# after them write documents back with whatever id they read.
|
||||
await run_in_threadpool(assign_ids, settings.FLOWS_DIR)
|
||||
store = FlowStore(settings.FLOWS_DIR)
|
||||
store = FlowStore(settings.FLOWS_DIR, tombstones=Tombstones())
|
||||
# The packages node code imports. In the background, because a first
|
||||
# install of anything substantial takes minutes and `uv` is given five
|
||||
# of them twice over — held here, the container never answers its
|
||||
|
||||
@@ -607,6 +607,25 @@ class AlertSent(SQLModel, table=True):
|
||||
ts: float = Field(index=True)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Flow tombstones
|
||||
#
|
||||
# A deleted flow's runs, events and metrics stay, filed under its id. New ids
|
||||
# are opaque, so without the key the flow last had they would name nothing
|
||||
# anyone recognises.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FlowTombstone(SQLModel, table=True):
|
||||
"""A deleted flow's last key, kept while history names its id."""
|
||||
|
||||
__tablename__ = "flow_tombstone"
|
||||
|
||||
id: str = Field(primary_key=True, max_length=255)
|
||||
name: str = Field(max_length=255)
|
||||
deleted_at: datetime = Field(default_factory=get_datetime_utc, sa_type=UTCDateTime)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Worker tokens
|
||||
#
|
||||
|
||||
@@ -238,7 +238,7 @@ class Client:
|
||||
# -- flows -------------------------------------------------------------
|
||||
|
||||
def get_flow(self, name: str) -> dict[str, Any] | None:
|
||||
"""The stored flow, draft included, or ``None`` if there is none."""
|
||||
"""The stored flow with the id *name*, draft included, or ``None``."""
|
||||
try:
|
||||
result: dict[str, Any] = self._call("GET", f"/flows/{seg(name)}")
|
||||
except ApiError as exc:
|
||||
@@ -247,9 +247,23 @@ class Client:
|
||||
raise
|
||||
return result
|
||||
|
||||
def find_flow(self, key: str) -> dict[str, Any] | None:
|
||||
"""The listing of the flow holding the key *key*, or ``None``."""
|
||||
result = self._call("GET", "/flows/", params={"name": key})
|
||||
found = list(result.get("data") or [])
|
||||
return found[0] if found else None
|
||||
|
||||
def create_flow(self, document: dict[str, Any]) -> dict[str, Any]:
|
||||
"""A new flow, under the id the engine issues for it."""
|
||||
result: dict[str, Any] = self._call("POST", "/flows/", json=document)
|
||||
return result
|
||||
|
||||
def put_flow(self, document: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Save over the flow *document* names by its id, or by its key for
|
||||
one whose id is its key."""
|
||||
flow_id = document.get("id") or document["name"]
|
||||
result: dict[str, Any] = self._call(
|
||||
"PUT", f"/flows/{seg(document['name'])}", json=document
|
||||
"PUT", f"/flows/{seg(flow_id)}", json=document
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -821,7 +835,11 @@ def _sync_one(
|
||||
force: bool,
|
||||
) -> SyncReport:
|
||||
report = SyncReport(target.name)
|
||||
stored = client.get_flow(target.name)
|
||||
# The declared key is what an instance shares; the id is this engine's,
|
||||
# and every request after the lookup names the flow by it.
|
||||
found = client.find_flow(target.name)
|
||||
flow_id = str(found["id"]) if found else ""
|
||||
stored = client.get_flow(flow_id) if flow_id else None
|
||||
version = 1
|
||||
# What the drift check already read, so the write loop below does not ask
|
||||
# the engine for every node's body a second time.
|
||||
@@ -836,7 +854,11 @@ def _sync_one(
|
||||
|
||||
document = target.document(origin)
|
||||
moved = _moved_code(stored, document)
|
||||
saved = client.put_flow(document | {"version": version})
|
||||
if stored is None:
|
||||
saved = client.create_flow(document)
|
||||
flow_id = str((saved.get("definition") or {}).get("id") or target.name)
|
||||
else:
|
||||
saved = client.put_flow(document | {"id": flow_id, "version": version})
|
||||
# An engine older than the per-node digest parses the document without
|
||||
# those fields and stores it without them, silently: no error, no 404, and
|
||||
# a cache still keyed on the whole repository.
|
||||
@@ -860,16 +882,16 @@ def _sync_one(
|
||||
if not report.created:
|
||||
stored_code = known.get(node_id)
|
||||
if stored_code is None:
|
||||
stored_code = client.get_source(target.name, node_id)
|
||||
stored_code = client.get_source(flow_id, node_id)
|
||||
if stored_code == code:
|
||||
continue
|
||||
client.put_source(target.name, node_id, code)
|
||||
client.put_source(flow_id, node_id, code)
|
||||
if node_id not in report.changed:
|
||||
report.changed.append(node_id)
|
||||
|
||||
has_draft = bool((client.get_flow(target.name) or {}).get("has_draft"))
|
||||
has_draft = bool((client.get_flow(flow_id) or {}).get("has_draft"))
|
||||
if publish and has_draft:
|
||||
client.publish(target.name, version)
|
||||
client.publish(flow_id, version)
|
||||
report.published = True
|
||||
else:
|
||||
# Only when one is actually there. "Nothing needed publishing" and
|
||||
@@ -934,7 +956,9 @@ def _refuse_on_drift(
|
||||
for stored_node in definition.get("nodes") or []:
|
||||
node_id = str(stored_node.get("id"))
|
||||
try:
|
||||
entry = client.get_source_entry(target.name, node_id)
|
||||
entry = client.get_source_entry(
|
||||
str(definition.get("id") or target.name), node_id
|
||||
)
|
||||
except ApiError:
|
||||
continue
|
||||
if entry.get("missing"):
|
||||
|
||||
Reference in New Issue
Block a user