Flows and dashboards are created by POST under an issued or superuser-pinned id; PUT only updates; deleted flows leave a tombstone with their key; the controller stores by id and runs by key, hooks live under /hooks/<flow id>; a key with state or queued work behind it is refused until cleared
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
"""flow_tombstone
|
||||
|
||||
The last key of a deleted flow, so its history still reads as something.
|
||||
|
||||
Revision ID: c4f1a8e2d7b3
|
||||
Revises: b9e4d2a7c1f6
|
||||
Create Date: 2026-11-02
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
import sqlmodel.sql.sqltypes
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "c4f1a8e2d7b3"
|
||||
down_revision = "b9e4d2a7c1f6"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"flow_tombstone",
|
||||
sa.Column("id", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False),
|
||||
sa.Column(
|
||||
"name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False
|
||||
),
|
||||
sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("flow_tombstone")
|
||||
@@ -4,7 +4,7 @@ from typing import Annotated, Any
|
||||
from urllib.parse import unquote
|
||||
|
||||
import jwt
|
||||
from fastapi import Depends, HTTPException, Request, status
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, status
|
||||
from fastapi.security import OAuth2PasswordBearer
|
||||
from jwt.exceptions import InvalidTokenError
|
||||
from pydantic import ValidationError
|
||||
@@ -615,3 +615,33 @@ def get_code_author(current_user: CurrentUser) -> User:
|
||||
if not current_user.is_superuser:
|
||||
raise HTTPException(status_code=403, detail=CODE_IS_SUPERUSERS)
|
||||
return current_user
|
||||
|
||||
|
||||
def check_pin(pin: str, user: User, *routers: APIRouter) -> None:
|
||||
"""Refuse a pinned id from anyone but a superuser, or one a route shadows.
|
||||
|
||||
Creating a flow or a dashboard gives it an opaque id. A pinned one is how
|
||||
the Demo, a test or a restore keeps the id it already knows. *routers* are
|
||||
the ones serving that kind's paths: a pin equal to one of their literal
|
||||
segments, such as ``graph``, would be unreachable behind that route.
|
||||
"""
|
||||
if not pin:
|
||||
return
|
||||
if not user.is_superuser:
|
||||
raise HTTPException(status_code=403, detail="Only a superuser can choose an id")
|
||||
shadowed = {
|
||||
segment
|
||||
for router in routers
|
||||
for route in router.routes
|
||||
if "{"
|
||||
not in (
|
||||
segment := getattr(route, "path", "")[len(router.prefix) :]
|
||||
.strip("/")
|
||||
.split("/")[0]
|
||||
)
|
||||
}
|
||||
if pin in shadowed:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"'{pin}' is a path of the API, so it cannot be an id",
|
||||
)
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
"""Dashboards: documents of widgets bound to message names."""
|
||||
"""Dashboards: documents of widgets bound to message names.
|
||||
|
||||
A dashboard is addressed by its id in every path. Its key, ``name``, is a
|
||||
label unique among dashboards and what ``GET /dashboards?name=`` looks up.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
@@ -7,18 +11,24 @@ from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from pydantic import BaseModel
|
||||
|
||||
from fluksio.api.deps import DashboardStoreDep, FlowControllerDep, get_current_user
|
||||
from fluksio.api.deps import (
|
||||
CurrentUser,
|
||||
DashboardStoreDep,
|
||||
FlowControllerDep,
|
||||
check_pin,
|
||||
get_current_user,
|
||||
)
|
||||
from fluksio.flow.dashboards import (
|
||||
DashboardCreate,
|
||||
DashboardDef,
|
||||
DashboardExists,
|
||||
DashboardNotFound,
|
||||
DashboardsPublic,
|
||||
default_dashboard,
|
||||
results_dashboard,
|
||||
results_name,
|
||||
)
|
||||
from fluksio.flow.events import event_bus
|
||||
from fluksio.flow.store import FlowNotFound, StaleVersion
|
||||
from fluksio.flow.store import FlowNotFound, IdTaken, StaleVersion
|
||||
from fluksio.models import Message
|
||||
|
||||
router = APIRouter(
|
||||
@@ -35,97 +45,118 @@ def _apply_history_limits(store: Any, controller: Any) -> None:
|
||||
controller.set_history_limits(store.history_requirements())
|
||||
|
||||
|
||||
def _missing(dashboard_id: str) -> HTTPException:
|
||||
return HTTPException(
|
||||
status_code=404, detail=f"No dashboard with the id '{dashboard_id}'"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/", response_model=DashboardsPublic)
|
||||
async def read_dashboards(store: DashboardStoreDep) -> Any:
|
||||
"""Every dashboard, without its contents."""
|
||||
async def read_dashboards(store: DashboardStoreDep, name: str | None = None) -> Any:
|
||||
"""Every dashboard, without its contents, or with ``name`` the one with
|
||||
that key."""
|
||||
summaries = await run_in_threadpool(store.list)
|
||||
if name is not None:
|
||||
summaries = [summary for summary in summaries if summary.name == name]
|
||||
return DashboardsPublic(data=summaries, count=len(summaries))
|
||||
|
||||
|
||||
@router.get("/{name}", response_model=DashboardDef)
|
||||
@router.post("/", response_model=DashboardDef, status_code=201)
|
||||
async def create_dashboard(
|
||||
body: DashboardCreate, store: DashboardStoreDep, user: CurrentUser
|
||||
) -> Any:
|
||||
"""Start a dashboard: one empty grid under a key of its own.
|
||||
|
||||
The server gives it an opaque id; a superuser may pin one with ``id``. A
|
||||
draft, like every edit that follows it — a dashboard reaches a panel only
|
||||
once someone publishes it, so an empty one never does.
|
||||
"""
|
||||
check_pin(body.id, user, router)
|
||||
try:
|
||||
# No history limits to apply: they are read from the published
|
||||
# documents, and this one is not one of them yet.
|
||||
return await run_in_threadpool(
|
||||
store.create, default_dashboard(body.name, body.title), body.id
|
||||
)
|
||||
except (DashboardExists, IdTaken) as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
|
||||
|
||||
@router.get("/{dashboard_id}", response_model=DashboardDef)
|
||||
async def read_dashboard(
|
||||
name: str, store: DashboardStoreDep, draft: bool = False
|
||||
dashboard_id: str, store: DashboardStoreDep, draft: bool = False
|
||||
) -> Any:
|
||||
"""What a panel shows, or with ``draft`` the copy the editor is on."""
|
||||
try:
|
||||
return await run_in_threadpool(store.read, name, draft)
|
||||
return await run_in_threadpool(store.read, dashboard_id, draft)
|
||||
except DashboardNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
|
||||
raise _missing(dashboard_id)
|
||||
|
||||
|
||||
@router.post("/{name}", response_model=DashboardDef)
|
||||
async def create_dashboard(name: str, store: DashboardStoreDep) -> Any:
|
||||
"""Start a dashboard: one page, one section, nothing on it yet.
|
||||
|
||||
A draft, like every edit that follows it — a dashboard reaches a panel
|
||||
only once someone publishes it, so an empty one never does.
|
||||
"""
|
||||
if await run_in_threadpool(store.exists, name):
|
||||
raise HTTPException(status_code=409, detail=f"'{name}' already exists")
|
||||
try:
|
||||
defn = default_dashboard(name)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc))
|
||||
# No history limits to apply: they are read from the published documents,
|
||||
# and this one is not one of them yet.
|
||||
return await run_in_threadpool(store.write_draft, defn, 0)
|
||||
|
||||
|
||||
@router.post("/from-flow/{flow}", response_model=DashboardDef)
|
||||
@router.post("/from-flow/{flow_id}", response_model=DashboardDef, status_code=201)
|
||||
async def generate_results_dashboard(
|
||||
flow: str,
|
||||
flow_id: str,
|
||||
store: DashboardStoreDep,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Draw a batch flow's results as a dashboard, from the ports it declares.
|
||||
|
||||
Published straight away rather than left as a draft: there is nothing to
|
||||
review that the flow did not already say, and what makes it useful is
|
||||
being able to open it against a run immediately. It is an ordinary
|
||||
dashboard afterwards — editing it is how it stops being generic.
|
||||
Keyed after the flow's key, ``<key>_results``, which a second one refuses
|
||||
(409) rather than numbering. Published straight away rather than left as a
|
||||
draft: there is nothing to review that the flow did not already say, and
|
||||
what makes it useful is being able to open it against a run immediately.
|
||||
It is an ordinary dashboard afterwards — editing it is how it stops being
|
||||
generic.
|
||||
"""
|
||||
try:
|
||||
definition = await run_in_threadpool(controller.store.read_flow, flow)
|
||||
definition = await run_in_threadpool(controller.store.read_flow, flow_id)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No flow named '{flow}'")
|
||||
raise HTTPException(status_code=404, detail=f"No flow with the id '{flow_id}'")
|
||||
if definition.mode != "batch":
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"'{flow}' is a live flow; a results dashboard is a run's view",
|
||||
detail=(
|
||||
f"'{definition.name}' is a live flow; a results dashboard is a "
|
||||
"run's view"
|
||||
),
|
||||
)
|
||||
name = results_name(flow)
|
||||
if await run_in_threadpool(store.exists, name):
|
||||
raise HTTPException(status_code=409, detail=f"'{name}' already exists")
|
||||
|
||||
try:
|
||||
written = await run_in_threadpool(store.write, results_dashboard(definition))
|
||||
written = await run_in_threadpool(
|
||||
store.create, results_dashboard(definition), "", True
|
||||
)
|
||||
except DashboardExists as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
await run_in_threadpool(_apply_history_limits, store, controller)
|
||||
event_bus.publish(
|
||||
{"type": "dashboard_changed", "dashboard": name, "ts": time.time()}
|
||||
{"type": "dashboard_changed", "dashboard": written.id, "ts": time.time()}
|
||||
)
|
||||
return written
|
||||
|
||||
|
||||
@router.put("/{name}", response_model=DashboardDef)
|
||||
@router.put("/{dashboard_id}", response_model=DashboardDef)
|
||||
async def save_dashboard(
|
||||
name: str,
|
||||
dashboard_id: str,
|
||||
body: DashboardDef,
|
||||
store: DashboardStoreDep,
|
||||
) -> Any:
|
||||
"""Save unpublished changes, refusing a save someone else has moved past.
|
||||
|
||||
This writes a draft: panels keep showing the published document until
|
||||
someone publishes, so nothing here can change what a wall is displaying —
|
||||
which is also why the engine's history limits are left alone.
|
||||
Only to a dashboard that exists: creating one is ``POST /dashboards``, and
|
||||
neither the id nor the key changes here. This writes a draft: panels keep
|
||||
showing the published document until someone publishes, so nothing here
|
||||
can change what a wall is displaying — which is also why the engine's
|
||||
history limits are left alone.
|
||||
"""
|
||||
if body.name != name:
|
||||
raise HTTPException(status_code=422, detail="The name in the body must match")
|
||||
if body.id and body.id != name:
|
||||
if body.id and body.id != dashboard_id:
|
||||
raise HTTPException(status_code=422, detail="The id in the body must match")
|
||||
try:
|
||||
saved = await run_in_threadpool(store.write_draft, body, body.version)
|
||||
return await run_in_threadpool(
|
||||
store.write_draft,
|
||||
body.model_copy(update={"id": dashboard_id}),
|
||||
body.version,
|
||||
)
|
||||
except DashboardNotFound:
|
||||
raise _missing(dashboard_id)
|
||||
except StaleVersion as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
@@ -134,27 +165,25 @@ async def save_dashboard(
|
||||
"current_version": exc.current,
|
||||
},
|
||||
)
|
||||
except DashboardExists as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
return saved
|
||||
|
||||
|
||||
@router.post("/{name}/publish", response_model=DashboardDef)
|
||||
@router.post("/{dashboard_id}/publish", response_model=DashboardDef)
|
||||
async def publish_dashboard(
|
||||
name: str,
|
||||
dashboard_id: str,
|
||||
body: PublishRequest,
|
||||
store: DashboardStoreDep,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Put the unpublished changes on the panels."""
|
||||
if not await run_in_threadpool(store.exists, name):
|
||||
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
|
||||
if not await run_in_threadpool(store.has_draft, name):
|
||||
if not await run_in_threadpool(store.exists, dashboard_id):
|
||||
raise _missing(dashboard_id)
|
||||
if not await run_in_threadpool(store.has_draft, dashboard_id):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Dashboard '{name}' has no unpublished changes"
|
||||
status_code=400,
|
||||
detail=f"Dashboard '{dashboard_id}' has no unpublished changes",
|
||||
)
|
||||
try:
|
||||
published = await run_in_threadpool(store.publish, name, body.version)
|
||||
published = await run_in_threadpool(store.publish, dashboard_id, body.version)
|
||||
except StaleVersion as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
@@ -167,50 +196,52 @@ async def publish_dashboard(
|
||||
# What a panel is showing has changed. Panels watch the flow socket, and
|
||||
# the tile values alone cannot tell them the document itself moved.
|
||||
event_bus.publish(
|
||||
{"type": "dashboard_changed", "dashboard": name, "ts": time.time()}
|
||||
{"type": "dashboard_changed", "dashboard": dashboard_id, "ts": time.time()}
|
||||
)
|
||||
return published
|
||||
|
||||
|
||||
@router.post("/{name}/discard", response_model=DashboardDef)
|
||||
async def discard_dashboard_draft(name: str, store: DashboardStoreDep) -> Any:
|
||||
@router.post("/{dashboard_id}/discard", response_model=DashboardDef)
|
||||
async def discard_dashboard_draft(dashboard_id: str, store: DashboardStoreDep) -> Any:
|
||||
"""Throw the unpublished changes away and go back to what is shown."""
|
||||
if not await run_in_threadpool(store.exists, name):
|
||||
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
|
||||
if not await run_in_threadpool(store.has_draft, name):
|
||||
if not await run_in_threadpool(store.exists, dashboard_id):
|
||||
raise _missing(dashboard_id)
|
||||
if not await run_in_threadpool(store.has_draft, dashboard_id):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Dashboard '{name}' has no unpublished changes"
|
||||
status_code=400,
|
||||
detail=f"Dashboard '{dashboard_id}' has no unpublished changes",
|
||||
)
|
||||
if not await run_in_threadpool(store.is_published, name):
|
||||
if not await run_in_threadpool(store.is_published, dashboard_id):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Dashboard '{name}' has never been published. Delete it instead "
|
||||
"of discarding it"
|
||||
f"Dashboard '{dashboard_id}' has never been published. Delete it "
|
||||
"instead of discarding it"
|
||||
),
|
||||
)
|
||||
return await run_in_threadpool(store.discard_draft, name)
|
||||
return await run_in_threadpool(store.discard_draft, dashboard_id)
|
||||
|
||||
|
||||
@router.delete("/{name}", response_model=Message)
|
||||
@router.delete("/{dashboard_id}", response_model=Message)
|
||||
async def delete_dashboard(
|
||||
name: str, store: DashboardStoreDep, controller: FlowControllerDep
|
||||
dashboard_id: str, store: DashboardStoreDep, controller: FlowControllerDep
|
||||
) -> Any:
|
||||
try:
|
||||
await run_in_threadpool(store.delete, name)
|
||||
await run_in_threadpool(store.delete, dashboard_id)
|
||||
except DashboardNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
|
||||
raise _missing(dashboard_id)
|
||||
await run_in_threadpool(_apply_history_limits, store, controller)
|
||||
return Message(message=f"Deleted dashboard '{name}'")
|
||||
return Message(message=f"Deleted dashboard '{dashboard_id}'")
|
||||
|
||||
|
||||
@router.post("/{name}/duplicate", response_model=DashboardDef)
|
||||
async def duplicate_dashboard(name: str, store: DashboardStoreDep) -> Any:
|
||||
@router.post("/{dashboard_id}/duplicate", response_model=DashboardDef)
|
||||
async def duplicate_dashboard(dashboard_id: str, store: DashboardStoreDep) -> Any:
|
||||
"""Copy a dashboard into a new one that is only a draft, named after it.
|
||||
|
||||
No history limits to apply, as for a new dashboard: nothing published moved.
|
||||
The copy gets an id of its own. No history limits to apply, as for a new
|
||||
dashboard: nothing published moved.
|
||||
"""
|
||||
try:
|
||||
return await run_in_threadpool(store.duplicate, name)
|
||||
return await run_in_threadpool(store.duplicate, dashboard_id)
|
||||
except DashboardNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'")
|
||||
raise _missing(dashboard_id)
|
||||
|
||||
+250
-174
@@ -1,4 +1,10 @@
|
||||
"""The flow API. Everything the editor can do is available here first."""
|
||||
"""The flow API. Everything the editor can do is available here first.
|
||||
|
||||
A flow is addressed by its id in every path, and never by its key: the key is
|
||||
the namespace of its messages, which is what the engine goes by, so each route
|
||||
reads the flow and runs, pauses or reads values under the key it holds. Node
|
||||
ids in paths are the flow's own, unqualified.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
@@ -30,6 +36,7 @@ from fluksio.api.deps import (
|
||||
HiddenDep,
|
||||
SessionDep,
|
||||
TokenDep,
|
||||
check_pin,
|
||||
decode_token,
|
||||
get_code_author,
|
||||
get_current_user,
|
||||
@@ -63,6 +70,7 @@ from fluksio.flow.runs import RunRejected
|
||||
from fluksio.flow.schemas import (
|
||||
NAME_PATTERN,
|
||||
BrainGraph,
|
||||
FlowCreate,
|
||||
FlowDef,
|
||||
FlowsPublic,
|
||||
FlowStatePublic,
|
||||
@@ -81,6 +89,7 @@ from fluksio.flow.state import as_number
|
||||
from fluksio.flow.store import (
|
||||
FlowExists,
|
||||
FlowNotFound,
|
||||
IdTaken,
|
||||
LibExists,
|
||||
LibNotFound,
|
||||
StaleVersion,
|
||||
@@ -247,10 +256,10 @@ def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail:
|
||||
"paused": controller.is_paused(name),
|
||||
}
|
||||
endpoints = _endpoints(controller, definition)
|
||||
if controller.store.has_draft(name):
|
||||
if controller.store.has_draft(definition.id):
|
||||
# Report the draft the editor is showing, not the version running
|
||||
# underneath it — otherwise a node the author just broke looks fine.
|
||||
preview = controller.preview(name)
|
||||
preview = controller.preview(definition.id)
|
||||
return FlowDetail(
|
||||
definition=definition,
|
||||
nodes=preview.nodes,
|
||||
@@ -268,16 +277,21 @@ def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail:
|
||||
)
|
||||
|
||||
|
||||
def _read_flow(controller: FlowController, name: str) -> FlowDef:
|
||||
def _missing(flow_id: str) -> HTTPException:
|
||||
return HTTPException(status_code=404, detail=f"No flow with the id '{flow_id}'")
|
||||
|
||||
|
||||
def _read_flow(controller: FlowController, flow_id: str) -> FlowDef:
|
||||
"""The working copy: the draft when there is one, else what is published."""
|
||||
try:
|
||||
return controller.store.read_flow(name, draft=True)
|
||||
return controller.store.read_flow(flow_id, draft=True)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
|
||||
raise _missing(flow_id)
|
||||
|
||||
|
||||
def _audit(action: str, flow: str, user: CurrentUser) -> None:
|
||||
"""Record who changed what. The collector writes it down; the bus carries it."""
|
||||
"""Record who changed what, by the flow's key. The collector writes it
|
||||
down; the bus carries it."""
|
||||
event_bus.publish(
|
||||
{
|
||||
"type": "audit",
|
||||
@@ -296,6 +310,7 @@ def _source_ref(definition: FlowDef, node_id: str) -> str | None:
|
||||
|
||||
|
||||
def _require_enabled(controller: FlowController, name: str) -> None:
|
||||
"""Refuse running a stopped flow, *name* being its key."""
|
||||
if not controller.is_enabled(name):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
@@ -306,7 +321,8 @@ def _require_enabled(controller: FlowController, name: str) -> None:
|
||||
def _flow_state(
|
||||
controller: FlowController, name: str, hidden: frozenset[str]
|
||||
) -> FlowStatePublic:
|
||||
"""The flow's values, but those of a camera the caller may not see."""
|
||||
"""The values of the flow with key *name*, but a camera's the caller may
|
||||
not see."""
|
||||
return FlowStatePublic(
|
||||
values={
|
||||
key: MessageValue(**value)
|
||||
@@ -323,29 +339,34 @@ def _flow_state(
|
||||
|
||||
|
||||
@router.get("/", response_model=FlowsPublic)
|
||||
def read_flows(controller: FlowControllerDep) -> Any:
|
||||
"""List every flow."""
|
||||
def read_flows(controller: FlowControllerDep, name: str | None = None) -> Any:
|
||||
"""List every flow, or with ``name`` the one holding that key."""
|
||||
ids = controller.store.list_flows()
|
||||
if name is not None:
|
||||
found = controller.store.key_index().get(name)
|
||||
ids = [found] if found else []
|
||||
summaries = []
|
||||
for name in controller.store.list_flows():
|
||||
for flow_id in ids:
|
||||
try:
|
||||
definition = controller.store.read_flow(name, draft=True)
|
||||
definition = controller.store.read_flow(flow_id, draft=True)
|
||||
except FlowNotFound:
|
||||
continue
|
||||
statuses = controller.node_statuses(name)
|
||||
key = definition.name
|
||||
statuses = controller.node_statuses(key)
|
||||
summaries.append(
|
||||
FlowSummary(
|
||||
id=definition.id,
|
||||
name=definition.name,
|
||||
name=key,
|
||||
title=definition.title,
|
||||
mode=definition.mode,
|
||||
node_count=len(definition.nodes),
|
||||
error_count=sum(1 for s in statuses if s.status == "error"),
|
||||
has_draft=controller.store.has_draft(name),
|
||||
enabled=controller.is_enabled(name),
|
||||
paused=controller.is_paused(name),
|
||||
quarantined=controller.is_quarantined(name),
|
||||
has_draft=controller.store.has_draft(flow_id),
|
||||
enabled=controller.is_enabled(key),
|
||||
paused=controller.is_paused(key),
|
||||
quarantined=controller.is_quarantined(key),
|
||||
version=definition.version,
|
||||
updated_at=controller.store.updated_at(name),
|
||||
updated_at=controller.store.updated_at(flow_id),
|
||||
)
|
||||
)
|
||||
return FlowsPublic(data=summaries, count=len(summaries))
|
||||
@@ -359,7 +380,7 @@ def read_node_types() -> Any:
|
||||
return node_type_info()
|
||||
|
||||
|
||||
# Above "/{name}" for the same reason "library" is: a flow called "graph" would
|
||||
# Above "/{flow_id}" for the same reason "library" is: a flow called "graph" would
|
||||
# otherwise be unreachable.
|
||||
@router.get("/graph", response_model=BrainGraph)
|
||||
def read_graph(controller: FlowControllerDep) -> Any:
|
||||
@@ -370,7 +391,7 @@ def read_graph(controller: FlowControllerDep) -> Any:
|
||||
# -----------------------------------------------------------------------------
|
||||
# Shared nodes
|
||||
#
|
||||
# Declared above the "/{name}" routes: "library" would otherwise be read as a
|
||||
# Declared above the "/{flow_id}" routes: "library" would otherwise be read as a
|
||||
# flow name.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
@@ -399,8 +420,9 @@ async def delete_shared_node(
|
||||
await run_in_threadpool(controller.store.detach_lib, lib_name)
|
||||
except LibNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No shared node '{lib_name}'")
|
||||
for flow in sorted({usage.split(".", 1)[0] for usage in running}):
|
||||
await controller.reload_flow(flow)
|
||||
# `usages` qualifies with the flow's id.
|
||||
for flow_id in sorted({usage.split(".", 1)[0] for usage in running}):
|
||||
await controller.reload_flow(flow_id)
|
||||
return Message(message=f"Deleted shared node '{lib_name}'")
|
||||
# The published flows too: a draft that unshared a node still runs it.
|
||||
used_by = sorted(
|
||||
@@ -418,10 +440,10 @@ async def delete_shared_node(
|
||||
return Message(message=f"Deleted shared node '{lib_name}'")
|
||||
|
||||
|
||||
@router.get("/{name}", response_model=FlowDetail)
|
||||
def read_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
@router.get("/{flow_id}", response_model=FlowDetail)
|
||||
def read_flow(flow_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Read one flow, with the state of its nodes."""
|
||||
return _detail(controller, _read_flow(controller, name))
|
||||
return _detail(controller, _read_flow(controller, flow_id))
|
||||
|
||||
|
||||
def _check_flavors(definition: FlowDef) -> None:
|
||||
@@ -518,147 +540,192 @@ def _refuse_code_changes(definition: FlowDef, before: FlowDef | None) -> None:
|
||||
)
|
||||
|
||||
|
||||
@router.put("/{name}", response_model=FlowDetail)
|
||||
def _check_document(definition: FlowDef) -> None:
|
||||
"""What a create and a save both refuse in a document."""
|
||||
if len({n.id for n in definition.nodes}) != len(definition.nodes):
|
||||
raise HTTPException(status_code=400, detail="Node names must be unique")
|
||||
_check_flavors(definition)
|
||||
_check_origin(definition)
|
||||
|
||||
|
||||
@router.post("/", response_model=FlowDetail, status_code=201)
|
||||
async def create_flow(
|
||||
body: FlowCreate, controller: FlowControllerDep, user: CurrentUser
|
||||
) -> Any:
|
||||
"""Create a flow. It is only a draft until someone publishes it.
|
||||
|
||||
The server gives it an opaque id. A superuser may pin one with ``id``: an
|
||||
id no flow holds, or one whose history this flow takes up again, under the
|
||||
key that history was kept under. A key with state or queued work still
|
||||
behind it is refused until that is gone, so a new flow never inherits what
|
||||
another left.
|
||||
"""
|
||||
check_pin(body.id, user, router, ws_router)
|
||||
definition = body.document()
|
||||
_check_document(definition)
|
||||
if not user.is_superuser:
|
||||
_refuse_code_changes(definition, None)
|
||||
residue = await run_in_threadpool(controller.key_residue, definition.name)
|
||||
if residue:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
f"The key '{definition.name}' still has values or queued work "
|
||||
f"behind it ({', '.join(residue[:3])}). Wait for the work to "
|
||||
"finish, or stop what still writes to it."
|
||||
),
|
||||
)
|
||||
try:
|
||||
stored = await run_in_threadpool(controller.store.create, definition, body.id)
|
||||
except (FlowExists, IdTaken) as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
return await run_in_threadpool(_detail, controller, stored)
|
||||
|
||||
|
||||
@router.put("/{flow_id}", response_model=FlowDetail)
|
||||
async def save_flow(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
definition: FlowDef,
|
||||
controller: FlowControllerDep,
|
||||
user: CurrentUser,
|
||||
) -> Any:
|
||||
"""Save unpublished changes to a flow.
|
||||
|
||||
This writes a draft: the running pipeline keeps the published version until
|
||||
someone publishes. ``version`` is the one the editor last saw — a mismatch
|
||||
means another client saved in between and answers 409 rather than throwing
|
||||
their work away.
|
||||
Only to one that exists: creating one is ``POST /flows``, and neither the
|
||||
id nor the key changes here. This writes a draft: the running pipeline
|
||||
keeps the published version until someone publishes. ``version`` is the
|
||||
one the editor last saw — a mismatch means another client saved in
|
||||
between and answers 409 rather than throwing their work away.
|
||||
"""
|
||||
if definition.name != name:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="The flow name in the body must match the URL"
|
||||
)
|
||||
if definition.id and definition.id != name:
|
||||
if definition.id and definition.id != flow_id:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="The flow id in the body must match the URL"
|
||||
)
|
||||
|
||||
duplicates = {n.id for n in definition.nodes}
|
||||
if len(duplicates) != len(definition.nodes):
|
||||
raise HTTPException(status_code=400, detail="Node names must be unique")
|
||||
|
||||
_check_flavors(definition)
|
||||
_check_origin(definition)
|
||||
definition = definition.model_copy(update={"id": flow_id})
|
||||
_check_document(definition)
|
||||
before = _read_flow(controller, flow_id)
|
||||
if not user.is_superuser:
|
||||
before = _read_flow(controller, name) if controller.store.exists(name) else None
|
||||
_refuse_code_changes(definition, before)
|
||||
|
||||
try:
|
||||
stored = await run_in_threadpool(
|
||||
controller.store.write_draft, definition, definition.version
|
||||
)
|
||||
except FlowNotFound:
|
||||
raise _missing(flow_id)
|
||||
except StaleVersion as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"message": str(exc), "current_version": exc.current},
|
||||
)
|
||||
except FlowExists as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
return await run_in_threadpool(_detail, controller, stored)
|
||||
|
||||
|
||||
@router.post("/{name}/publish", response_model=FlowDetail)
|
||||
@router.post("/{flow_id}/publish", response_model=FlowDetail)
|
||||
async def publish_flow(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
body: PublishRequest,
|
||||
controller: FlowControllerDep,
|
||||
user: CurrentUser,
|
||||
) -> Any:
|
||||
"""Deploy the unpublished changes: the engine picks them up from here."""
|
||||
_read_flow(controller, name)
|
||||
if not controller.store.has_draft(name):
|
||||
definition = _read_flow(controller, flow_id)
|
||||
if not controller.store.has_draft(flow_id):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Flow '{name}' has no unpublished changes"
|
||||
status_code=400, detail=f"Flow '{flow_id}' has no unpublished changes"
|
||||
)
|
||||
try:
|
||||
published = await run_in_threadpool(
|
||||
controller.store.publish_flow, name, body.version
|
||||
controller.store.publish_flow, flow_id, body.version
|
||||
)
|
||||
except StaleVersion as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"message": str(exc), "current_version": exc.current},
|
||||
)
|
||||
_audit("published", name, user)
|
||||
await controller.reload_flow(name)
|
||||
_audit("published", definition.name, user)
|
||||
await controller.reload_flow(flow_id)
|
||||
return await run_in_threadpool(_detail, controller, published)
|
||||
|
||||
|
||||
@router.post("/{name}/discard-draft", response_model=FlowDetail)
|
||||
async def discard_draft(name: str, controller: FlowControllerDep) -> Any:
|
||||
@router.post("/{flow_id}/discard-draft", response_model=FlowDetail)
|
||||
async def discard_draft(flow_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Throw the unpublished changes away and go back to what is running."""
|
||||
_read_flow(controller, name)
|
||||
if not controller.store.has_draft(name):
|
||||
_read_flow(controller, flow_id)
|
||||
if not controller.store.has_draft(flow_id):
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Flow '{name}' has no unpublished changes"
|
||||
status_code=400, detail=f"Flow '{flow_id}' has no unpublished changes"
|
||||
)
|
||||
if not controller.store.is_published(name):
|
||||
if not controller.store.is_published(flow_id):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Flow '{name}' has never been published. Delete it instead of "
|
||||
f"Flow '{flow_id}' has never been published. Delete it instead of "
|
||||
"discarding it"
|
||||
),
|
||||
)
|
||||
published = await run_in_threadpool(controller.store.discard_draft, name)
|
||||
published = await run_in_threadpool(controller.store.discard_draft, flow_id)
|
||||
return await run_in_threadpool(_detail, controller, published)
|
||||
|
||||
|
||||
@router.delete("/{name}", response_model=Message)
|
||||
@router.delete("/{flow_id}", response_model=Message)
|
||||
async def delete_flow(
|
||||
name: str, controller: FlowControllerDep, user: CurrentUser, session: SessionDep
|
||||
flow_id: str, controller: FlowControllerDep, user: CurrentUser, session: SessionDep
|
||||
) -> Any:
|
||||
"""Delete a flow and everything in it, except the record of what it ran."""
|
||||
"""Delete a flow and everything in it, except the record of what it ran.
|
||||
|
||||
Its key is kept with its id (`fluksio.flow.tombstones`) for as long as
|
||||
that record is, and what is left under the key, values and queued work,
|
||||
goes with the flow so the key is free for a new one.
|
||||
"""
|
||||
live = session.exec(
|
||||
select(Run.id).where(
|
||||
col(Run.flow) == name, col(Run.status).in_(("running", "queued"))
|
||||
col(Run.flow) == flow_id, col(Run.status).in_(("running", "queued"))
|
||||
)
|
||||
).first()
|
||||
if live is not None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
f"Flow '{name}' has a run in progress ({live}). Cancel it, or "
|
||||
f"Flow '{flow_id}' has a run in progress ({live}). Cancel it, or "
|
||||
"wait for it to finish, before deleting the flow."
|
||||
),
|
||||
)
|
||||
try:
|
||||
await run_in_threadpool(controller.store.delete_flow, name)
|
||||
key = await run_in_threadpool(controller.store.delete_flow, flow_id)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
|
||||
_audit("deleted", name, user)
|
||||
# Its files are gone; its values and queued work would otherwise linger.
|
||||
raise _missing(flow_id)
|
||||
_audit("deleted", key, user)
|
||||
# Its nodes first, or they would put values and work straight back.
|
||||
# Its runs stay: a run record is a record of what ran, and surviving the
|
||||
# flow it belonged to is the point of keeping one. `DELETE /runs?flow=`
|
||||
# is what clears them, and with `history=true` its cascades too.
|
||||
await run_in_threadpool(controller.forget_flow, name)
|
||||
await controller.reload_flow(name)
|
||||
return Message(message=f"Deleted flow '{name}'")
|
||||
await controller.reload_flow(flow_id)
|
||||
await run_in_threadpool(controller.forget_flow, key)
|
||||
return Message(message=f"Deleted flow '{key}'")
|
||||
|
||||
|
||||
@router.post("/{name}/duplicate", response_model=FlowDetail)
|
||||
@router.post("/{flow_id}/duplicate", response_model=FlowDetail)
|
||||
async def duplicate_flow(
|
||||
name: str, controller: FlowControllerDep, user: CurrentUser
|
||||
flow_id: str, controller: FlowControllerDep, user: CurrentUser
|
||||
) -> Any:
|
||||
"""Copy a flow into a new one that is only a draft, named after it.
|
||||
|
||||
Nothing to reload: a draft does not run. Its code nodes are new code in
|
||||
the copy, so copying them is a superuser's, like writing them.
|
||||
The copy gets an id of its own, and the first ``_copy`` key with nothing
|
||||
left behind it. Nothing to reload: a draft does not run. Its code nodes
|
||||
are new code in the copy, so copying them is a superuser's, like writing
|
||||
them.
|
||||
"""
|
||||
if not user.is_superuser:
|
||||
_refuse_code_changes(_read_flow(controller, name), None)
|
||||
_refuse_code_changes(_read_flow(controller, flow_id), None)
|
||||
try:
|
||||
copy = await run_in_threadpool(controller.store.duplicate_flow, name)
|
||||
copy = await run_in_threadpool(
|
||||
controller.store.duplicate_flow,
|
||||
flow_id,
|
||||
lambda key: bool(controller.key_residue(key)),
|
||||
)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
|
||||
raise _missing(flow_id)
|
||||
return await run_in_threadpool(_detail, controller, copy)
|
||||
|
||||
|
||||
@@ -667,14 +734,14 @@ async def duplicate_flow(
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/{name}/nodes/{node_id}/source", response_model=NodeSource)
|
||||
@router.get("/{flow_id}/nodes/{node_id}/source", response_model=NodeSource)
|
||||
def read_node_source(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Read a node's Python source, including unpublished edits."""
|
||||
definition = _read_flow(controller, name)
|
||||
definition = _read_flow(controller, flow_id)
|
||||
ref = _source_ref(definition, node_id)
|
||||
if ref:
|
||||
try:
|
||||
@@ -682,13 +749,13 @@ def read_node_source(
|
||||
except LibNotFound:
|
||||
raise HTTPException(status_code=404, detail=f"No shared node '{ref}'")
|
||||
return NodeSource(
|
||||
code=controller.store.read_node_source(name, node_id, draft=True),
|
||||
missing=not controller.store.has_node_source(name, node_id, draft=True),
|
||||
code=controller.store.read_node_source(flow_id, node_id, draft=True),
|
||||
missing=not controller.store.has_node_source(flow_id, node_id, draft=True),
|
||||
)
|
||||
|
||||
|
||||
def _follow_header(
|
||||
controller: FlowController, name: str, node_id: str, code: str, drop: bool
|
||||
controller: FlowController, flow_id: str, node_id: str, code: str, drop: bool
|
||||
) -> FlowDef | None:
|
||||
"""Bring a function node's ports and settings in line with its header.
|
||||
|
||||
@@ -697,7 +764,7 @@ def _follow_header(
|
||||
and a synced flow's ports are declared in its repository. Returns the
|
||||
draft when it changed.
|
||||
"""
|
||||
definition = controller.store.read_flow(name, draft=True)
|
||||
definition = controller.store.read_flow(flow_id, draft=True)
|
||||
node = next((n for n in definition.nodes if n.id == node_id), None)
|
||||
spec = NODE_TYPES.get(node.type) if node is not None else None
|
||||
if (
|
||||
@@ -720,12 +787,12 @@ def _follow_header(
|
||||
|
||||
|
||||
@router.put(
|
||||
"/{name}/nodes/{node_id}/source",
|
||||
"/{flow_id}/nodes/{node_id}/source",
|
||||
response_model=NodeSourceSaved,
|
||||
dependencies=[Depends(get_code_author)],
|
||||
)
|
||||
async def save_node_source(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
source: NodeSource,
|
||||
controller: FlowControllerDep,
|
||||
@@ -749,7 +816,8 @@ async def save_node_source(
|
||||
adds and renames from the header but keeps what it no longer names until
|
||||
the editor is left.
|
||||
"""
|
||||
definition = _read_flow(controller, name)
|
||||
definition = _read_flow(controller, flow_id)
|
||||
name = definition.name
|
||||
ref = _source_ref(definition, node_id)
|
||||
if ref:
|
||||
changed = await run_in_threadpool(
|
||||
@@ -760,10 +828,10 @@ async def save_node_source(
|
||||
await controller.reload_lib_users(ref)
|
||||
else:
|
||||
await run_in_threadpool(
|
||||
controller.store.write_node_source, name, node_id, source.code, True
|
||||
controller.store.write_node_source, flow_id, node_id, source.code, True
|
||||
)
|
||||
followed = await run_in_threadpool(
|
||||
_follow_header, controller, name, node_id, source.code, drop
|
||||
_follow_header, controller, flow_id, node_id, source.code, drop
|
||||
)
|
||||
node_def = next((n for n in definition.nodes if n.id == node_id), None)
|
||||
worker = (
|
||||
@@ -785,15 +853,15 @@ async def save_node_source(
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/share", response_model=FlowDetail)
|
||||
@router.post("/{flow_id}/nodes/{node_id}/share", response_model=FlowDetail)
|
||||
async def share_node(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
body: ShareRequest,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Move this node's code into the library so other flows can run it too."""
|
||||
_read_flow(controller, name)
|
||||
_read_flow(controller, flow_id)
|
||||
if not NAME_PATTERN.match(body.lib_name):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
@@ -803,26 +871,26 @@ async def share_node(
|
||||
)
|
||||
try:
|
||||
await run_in_threadpool(
|
||||
controller.store.share_node, name, node_id, body.lib_name
|
||||
controller.store.share_node, flow_id, node_id, body.lib_name
|
||||
)
|
||||
except LibExists as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, flow_id))
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/unshare", response_model=FlowDetail)
|
||||
@router.post("/{flow_id}/nodes/{node_id}/unshare", response_model=FlowDetail)
|
||||
async def unshare_node(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
controller: FlowControllerDep,
|
||||
) -> Any:
|
||||
"""Take a private copy of the shared code back into this flow."""
|
||||
_read_flow(controller, name)
|
||||
_read_flow(controller, flow_id)
|
||||
try:
|
||||
await run_in_threadpool(controller.store.unshare_node, name, node_id)
|
||||
await run_in_threadpool(controller.store.unshare_node, flow_id, node_id)
|
||||
except LibNotFound as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc))
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, flow_id))
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -830,49 +898,50 @@ async def unshare_node(
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/{name}/start", response_model=FlowDetail)
|
||||
@router.post("/{flow_id}/start", response_model=FlowDetail)
|
||||
async def start_flow(
|
||||
name: str, controller: FlowControllerDep, user: CurrentUser
|
||||
flow_id: str, controller: FlowControllerDep, user: CurrentUser
|
||||
) -> Any:
|
||||
"""Let the engine run this flow again."""
|
||||
_read_flow(controller, name)
|
||||
await controller.set_enabled(name, True)
|
||||
_audit("started", name, user)
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
|
||||
definition = _read_flow(controller, flow_id)
|
||||
await controller.set_enabled(flow_id, definition.name, True)
|
||||
_audit("started", definition.name, user)
|
||||
return await run_in_threadpool(_detail, controller, definition)
|
||||
|
||||
|
||||
@router.post("/{name}/stop", response_model=FlowDetail)
|
||||
async def stop_flow(name: str, controller: FlowControllerDep, user: CurrentUser) -> Any:
|
||||
@router.post("/{flow_id}/stop", response_model=FlowDetail)
|
||||
async def stop_flow(
|
||||
flow_id: str, controller: FlowControllerDep, user: CurrentUser
|
||||
) -> Any:
|
||||
"""Take this flow off the engine: no subscriptions, schedules or webhooks."""
|
||||
_read_flow(controller, name)
|
||||
await controller.set_enabled(name, False)
|
||||
_audit("stopped", name, user)
|
||||
return await run_in_threadpool(_detail, controller, _read_flow(controller, name))
|
||||
definition = _read_flow(controller, flow_id)
|
||||
await controller.set_enabled(flow_id, definition.name, False)
|
||||
_audit("stopped", definition.name, user)
|
||||
return await run_in_threadpool(_detail, controller, definition)
|
||||
|
||||
|
||||
@router.post("/{name}/pause", response_model=Message)
|
||||
def pause_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
@router.post("/{flow_id}/pause", response_model=Message)
|
||||
def pause_flow(flow_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Hold the flow's nodes so its messages can be stepped through."""
|
||||
_read_flow(controller, name)
|
||||
controller.pause_flow(name)
|
||||
return Message(message=f"Paused flow '{name}'")
|
||||
controller.pause_flow(_read_flow(controller, flow_id).name)
|
||||
return Message(message=f"Paused flow '{flow_id}'")
|
||||
|
||||
|
||||
@router.post("/{name}/resume", response_model=Message)
|
||||
async def resume_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
@router.post("/{flow_id}/resume", response_model=Message)
|
||||
async def resume_flow(flow_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Let the flow carry on, running whatever was held back."""
|
||||
_read_flow(controller, name)
|
||||
name = _read_flow(controller, flow_id).name
|
||||
await run_in_threadpool(controller.resume_flow, name)
|
||||
return Message(message=f"Resumed flow '{name}'")
|
||||
return Message(message=f"Resumed flow '{flow_id}'")
|
||||
|
||||
|
||||
@router.post("/{name}/step", response_model=Message)
|
||||
async def step_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
@router.post("/{flow_id}/step", response_model=Message)
|
||||
async def step_flow(flow_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Run one message a pause is holding back, leaving the flow paused."""
|
||||
_read_flow(controller, name)
|
||||
name = _read_flow(controller, flow_id).name
|
||||
node = await run_in_threadpool(controller.step_flow, name)
|
||||
if node is None:
|
||||
return Message(message=f"Nothing held back in flow '{name}'")
|
||||
return Message(message=f"Nothing held back in flow '{flow_id}'")
|
||||
return Message(message=f"Stepped '{node}'")
|
||||
|
||||
|
||||
@@ -881,18 +950,18 @@ async def step_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/{name}/validate", response_model=ValidationResult)
|
||||
def validate_flow(name: str, controller: FlowControllerDep) -> Any:
|
||||
@router.post("/{flow_id}/validate", response_model=ValidationResult)
|
||||
def validate_flow(flow_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Report what would keep this flow from running."""
|
||||
_read_flow(controller, name)
|
||||
if controller.store.has_draft(name):
|
||||
return ValidationResult(issues=controller.preview(name).issues)
|
||||
return ValidationResult(issues=controller.flow_issues(name))
|
||||
definition = _read_flow(controller, flow_id)
|
||||
if controller.store.has_draft(flow_id):
|
||||
return ValidationResult(issues=controller.preview(flow_id).issues)
|
||||
return ValidationResult(issues=controller.flow_issues(definition.name))
|
||||
|
||||
|
||||
@router.post("/{name}/run", response_model=FlowStatePublic)
|
||||
@router.post("/{flow_id}/run", response_model=FlowStatePublic)
|
||||
async def run_flow(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
body: RunRequest,
|
||||
request: Request,
|
||||
controller: FlowControllerDep,
|
||||
@@ -908,7 +977,8 @@ async def run_flow(
|
||||
one means: it is what keeps the parameters, the series and the result, and
|
||||
a button that quietly did something else would be a trap.
|
||||
"""
|
||||
definition = _read_flow(controller, name)
|
||||
definition = _read_flow(controller, flow_id)
|
||||
name = definition.name
|
||||
_require_enabled(controller, name)
|
||||
if definition.mode == "batch":
|
||||
service = getattr(request.app.state, "run_service", None)
|
||||
@@ -917,33 +987,34 @@ async def run_flow(
|
||||
try:
|
||||
await run_in_threadpool(
|
||||
service.submit,
|
||||
name,
|
||||
flow_id,
|
||||
params=body.inputs,
|
||||
cause="api",
|
||||
actor=user.email,
|
||||
draft=controller.store.has_draft(name),
|
||||
draft=controller.store.has_draft(flow_id),
|
||||
)
|
||||
except RunRejected as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
return _flow_state(controller, name, hidden)
|
||||
|
||||
inputs = {qualify(name, key): value for key, value in body.inputs.items()}
|
||||
if controller.store.has_draft(name):
|
||||
await run_in_threadpool(controller.run_preview, name, inputs)
|
||||
if controller.store.has_draft(flow_id):
|
||||
await run_in_threadpool(controller.run_preview, flow_id, inputs)
|
||||
else:
|
||||
await run_in_threadpool(controller.run_flow, name, inputs)
|
||||
return _flow_state(controller, name, hidden)
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/trigger", response_model=FlowStatePublic)
|
||||
@router.post("/{flow_id}/nodes/{node_id}/trigger", response_model=FlowStatePublic)
|
||||
async def trigger_node(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
body: TriggerRequest,
|
||||
controller: FlowControllerDep,
|
||||
hidden: HiddenDep,
|
||||
) -> Any:
|
||||
"""Feed values into a single node."""
|
||||
name = _read_flow(controller, flow_id).name
|
||||
_require_enabled(controller, name)
|
||||
try:
|
||||
error = await run_in_threadpool(
|
||||
@@ -951,7 +1022,7 @@ async def trigger_node(
|
||||
)
|
||||
except KeyError:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{flow_id}'"
|
||||
)
|
||||
if error:
|
||||
# The canvas already has the failure from the bus; the person who
|
||||
@@ -960,22 +1031,23 @@ async def trigger_node(
|
||||
return _flow_state(controller, name, hidden)
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/cancel", response_model=Message)
|
||||
def cancel_node(name: str, node_id: str, controller: FlowControllerDep) -> Any:
|
||||
@router.post("/{flow_id}/nodes/{node_id}/cancel", response_model=Message)
|
||||
def cancel_node(flow_id: str, node_id: str, controller: FlowControllerDep) -> Any:
|
||||
"""Stop a node that is running right now, by killing the worker running it.
|
||||
|
||||
Idempotent on purpose: by the time a click reaches here the node may well
|
||||
have finished, and that is the outcome that was asked for.
|
||||
"""
|
||||
name = _read_flow(controller, flow_id).name
|
||||
pool = controller.workers
|
||||
if pool is not None and pool.cancel(f"{name}.{node_id}"):
|
||||
return Message(message=f"Stopped '{node_id}'")
|
||||
return Message(message=f"'{node_id}' was not running")
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/acknowledge", response_model=Message)
|
||||
@router.post("/{flow_id}/nodes/{node_id}/acknowledge", response_model=Message)
|
||||
def acknowledge_node_error(
|
||||
name: str, node_id: str, controller: FlowControllerDep
|
||||
flow_id: str, node_id: str, controller: FlowControllerDep
|
||||
) -> Any:
|
||||
"""Dismiss what a node last failed with, so the canvas stops marking it.
|
||||
|
||||
@@ -983,18 +1055,19 @@ def acknowledge_node_error(
|
||||
an alert leaves no trace by the time anyone looks. Reading the traceback is
|
||||
what says it has been seen.
|
||||
"""
|
||||
node = f"{_read_flow(controller, flow_id).name}.{node_id}"
|
||||
try:
|
||||
controller.acknowledge_error(f"{name}.{node_id}")
|
||||
controller.acknowledge_error(node)
|
||||
except KeyError:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{flow_id}'"
|
||||
) from None
|
||||
# Another browser holds the same marker and has no reason to refetch, so it
|
||||
# would keep showing a failure that is gone until its next snapshot.
|
||||
event_bus.publish(
|
||||
{
|
||||
"type": "node_error_acknowledged",
|
||||
"node": f"{name}.{node_id}",
|
||||
"node": node,
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
@@ -1006,21 +1079,23 @@ def acknowledge_node_error(
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _node_def(controller: FlowController, name: str, node_id: str) -> NodeDef:
|
||||
def _node_def(controller: FlowController, flow_id: str, node_id: str) -> NodeDef:
|
||||
"""A node as the editor shows it: from the draft when there is one."""
|
||||
node = next(
|
||||
(n for n in _read_flow(controller, name).nodes if n.id == node_id), None
|
||||
(n for n in _read_flow(controller, flow_id).nodes if n.id == node_id), None
|
||||
)
|
||||
if node is None:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{flow_id}'"
|
||||
)
|
||||
return node
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/train", response_model=Training, status_code=202)
|
||||
@router.post(
|
||||
"/{flow_id}/nodes/{node_id}/train", response_model=Training, status_code=202
|
||||
)
|
||||
async def train_node(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
request: Request,
|
||||
controller: FlowControllerDep,
|
||||
@@ -1030,9 +1105,10 @@ async def train_node(
|
||||
|
||||
It learns the node's targets, its ``examples`` and its ``tests``, as the
|
||||
editor shows them. The node keeps running its model until the new one is
|
||||
set as its ``model``.
|
||||
set as its ``model``. The run names the node it trains by the flow's id,
|
||||
``<flow id>.<node>``, which is what its panel finds it by.
|
||||
"""
|
||||
node = _node_def(controller, name, node_id)
|
||||
node = _node_def(controller, flow_id, node_id)
|
||||
if node.type != "wren":
|
||||
raise HTTPException(status_code=400, detail=f"'{node_id}' is not a wren node")
|
||||
try:
|
||||
@@ -1054,7 +1130,7 @@ async def train_node(
|
||||
cause="train",
|
||||
actor=user.email,
|
||||
no_cache=True,
|
||||
for_node=f"{name}.{node_id}",
|
||||
for_node=f"{flow_id}.{node_id}",
|
||||
)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(
|
||||
@@ -1069,19 +1145,19 @@ async def train_node(
|
||||
return Training(id=run.id, status=run.status)
|
||||
|
||||
|
||||
@router.get("/{name}/nodes/{node_id}/train", response_model=Training | None)
|
||||
@router.get("/{flow_id}/nodes/{node_id}/train", response_model=Training | None)
|
||||
def read_training(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
node_id: str,
|
||||
request: Request,
|
||||
controller: FlowControllerDep,
|
||||
session: SessionDep,
|
||||
) -> Any:
|
||||
"""A wren node's latest training run, or null when it has none."""
|
||||
_node_def(controller, name, node_id)
|
||||
_node_def(controller, flow_id, node_id)
|
||||
run = session.exec(
|
||||
select(Run)
|
||||
.where(col(Run.flow) == TRAIN_FLOW, col(Run.for_node) == f"{name}.{node_id}")
|
||||
.where(col(Run.flow) == TRAIN_FLOW, col(Run.for_node) == f"{flow_id}.{node_id}")
|
||||
.order_by(col(Run.created_at).desc())
|
||||
).first()
|
||||
if run is None:
|
||||
@@ -1106,27 +1182,27 @@ def read_training(
|
||||
return training
|
||||
|
||||
|
||||
@router.get("/{name}/state", response_model=FlowStatePublic)
|
||||
def read_flow_state(name: str, controller: FlowControllerDep, hidden: HiddenDep) -> Any:
|
||||
@router.get("/{flow_id}/state", response_model=FlowStatePublic)
|
||||
def read_flow_state(
|
||||
flow_id: str, controller: FlowControllerDep, hidden: HiddenDep
|
||||
) -> Any:
|
||||
"""The last value seen on every message of this flow."""
|
||||
_read_flow(controller, name)
|
||||
return _flow_state(controller, name, hidden)
|
||||
return _flow_state(controller, _read_flow(controller, flow_id).name, hidden)
|
||||
|
||||
|
||||
@router.get("/{name}/history/{message}", response_model=MessageHistory)
|
||||
@router.get("/{flow_id}/history/{message}", response_model=MessageHistory)
|
||||
def read_message_history(
|
||||
name: str,
|
||||
flow_id: str,
|
||||
message: str,
|
||||
controller: FlowControllerDep,
|
||||
hidden: HiddenDep,
|
||||
) -> Any:
|
||||
"""The recent values of one message, for plotting.
|
||||
|
||||
``message`` may be given bare or qualified; a message that never carried a
|
||||
number or a flag comes back with an empty series.
|
||||
``message`` may be given bare, or qualified with the flow's key; a message
|
||||
that never carried a number or a flag comes back with an empty series.
|
||||
"""
|
||||
_read_flow(controller, name)
|
||||
key = qualify(name, message)
|
||||
key = qualify(_read_flow(controller, flow_id).name, message)
|
||||
if key in hidden:
|
||||
raise HTTPException(status_code=403, detail=CAMERA_IS_SUPERUSERS)
|
||||
return MessageHistory(
|
||||
|
||||
@@ -204,7 +204,8 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
|
||||
if queue.get("error"):
|
||||
problems.append("work queue unreachable")
|
||||
|
||||
names = await run_in_threadpool(controller.store.list_flows)
|
||||
# Keys, which is what the engine's gates and quarantine go by.
|
||||
names = list(await run_in_threadpool(controller.store.key_index))
|
||||
quarantined = controller.quarantined
|
||||
paused = set(controller.paused_flows())
|
||||
entries = list(controller.loaded.values())
|
||||
|
||||
@@ -72,6 +72,7 @@ from fluksio.flow.pipeline import (
|
||||
ValueSource,
|
||||
)
|
||||
from fluksio.flow.placement import Placer
|
||||
from fluksio.flow.queue import WorkItem, owned_by
|
||||
from fluksio.flow.remote import RemoteWorkerHub
|
||||
from fluksio.flow.resources import (
|
||||
ResourceAccountant,
|
||||
@@ -160,7 +161,10 @@ class LoadedNode:
|
||||
"""
|
||||
|
||||
id: str
|
||||
#: The flow's key, which ``id`` is qualified with.
|
||||
flow: str
|
||||
#: The flow's id, which its store paths and API routes use.
|
||||
flow_id: str = ""
|
||||
status: NodeStatus = NodeStatus.ACTIVE
|
||||
node: Node | None = None
|
||||
error: str | None = None
|
||||
@@ -556,6 +560,9 @@ class FlowController:
|
||||
self.disabled: set[str] = set()
|
||||
#: Flows that only run when a run asks them to.
|
||||
self.batch: set[str] = set()
|
||||
#: The key each built flow runs under, by its id. What a rebuild of a
|
||||
#: flow that is gone takes out, since its document can no longer say.
|
||||
self._flow_keys: dict[str, str] = {}
|
||||
#: Each flow's declared inputs, kept so rebuilding one flow can
|
||||
#: validate the whole graph without re-reading every flow off the disk.
|
||||
self._flow_inputs: dict[str, dict[str, bool]] = {}
|
||||
@@ -642,15 +649,16 @@ class FlowController:
|
||||
entry.last_error_ts = None
|
||||
entry.failures_in_a_row = 0
|
||||
|
||||
async def set_enabled(self, flow: str, enabled: bool) -> None:
|
||||
async def set_enabled(self, flow_id: str, flow: str, enabled: bool) -> None:
|
||||
"""Stop or start one flow, without building anything.
|
||||
|
||||
Its nodes are built whether or not it runs — being stopped means
|
||||
having no subscriptions, schedules or webhooks, not being absent — so
|
||||
all this has to do is the lifecycle call and the gate that goes with
|
||||
it. Nothing is compiled and no other flow is touched.
|
||||
it. Nothing is compiled and no other flow is touched. *flow_id* is
|
||||
where that is stored, *flow* the key the gate goes by.
|
||||
"""
|
||||
await run_in_threadpool(self.store.write_enabled, flow, enabled)
|
||||
await run_in_threadpool(self.store.write_enabled, flow_id, enabled)
|
||||
if self.pipeline is None:
|
||||
# Nothing built yet, so there are no nodes to start or stop.
|
||||
await self.reload()
|
||||
@@ -726,10 +734,9 @@ class FlowController:
|
||||
self.supervisor = Supervisor(self.events)
|
||||
|
||||
published = self.store.read_all()
|
||||
self._flow_keys = {flow.id: flow.name for flow in published}
|
||||
self.disabled = {
|
||||
flow.name
|
||||
for flow in published
|
||||
if not self.store.read_enabled(flow.name)
|
||||
flow.name for flow in published if not self.store.read_enabled(flow.id)
|
||||
}
|
||||
# A batch flow is built — the canvas draws it and validation covers
|
||||
# it — but never activated: it runs when a run asks it to, and a
|
||||
@@ -795,7 +802,7 @@ class FlowController:
|
||||
self._rewired()
|
||||
|
||||
async def reload_flow(
|
||||
self, name: str, wait: float | None = FLOW_REBUILD_WAIT
|
||||
self, flow_id: str, wait: float | None = FLOW_REBUILD_WAIT
|
||||
) -> None:
|
||||
"""Rebuild one flow, leaving every other flow's nodes connected.
|
||||
|
||||
@@ -806,7 +813,8 @@ class FlowController:
|
||||
on a populated instance is every node opening its socket again,
|
||||
and only one flow's have changed.
|
||||
|
||||
A flow the store no longer has is taken out instead of replaced.
|
||||
A flow the store no longer has is taken out instead of replaced. The
|
||||
flow is named by its id; everything about running it goes by its key.
|
||||
|
||||
The failure shape is the whole-pipeline one narrowed to a flow, and
|
||||
deliberately no better: a build that raises changes nothing, and
|
||||
@@ -826,11 +834,14 @@ class FlowController:
|
||||
# flow is not running is its own teardown and nothing more. Off
|
||||
# the loop for the same reason the full build is: compiling a
|
||||
# python node waits for a worker slot.
|
||||
built = await run_in_threadpool(self._build_one, name)
|
||||
built = await run_in_threadpool(self._build_one, flow_id)
|
||||
# A flow never built has nothing to take out, so no key either.
|
||||
name = built[0].name if built else self._flow_keys.pop(flow_id, "")
|
||||
await self._teardown(name)
|
||||
|
||||
if built is None:
|
||||
pipeline.remove_flow(name)
|
||||
if name:
|
||||
pipeline.remove_flow(name)
|
||||
self.loaded = {
|
||||
node_id: entry
|
||||
for node_id, entry in self.loaded.items()
|
||||
@@ -843,6 +854,7 @@ class FlowController:
|
||||
self.batch = self.batch - {name}
|
||||
else:
|
||||
flow, nodes, loaded, initial_values, enabled = built
|
||||
self._flow_keys[flow_id] = name
|
||||
pipeline.replace_flow(name, nodes, initial_values)
|
||||
|
||||
# A rebuild is a fresh set of nodes, but not a fresh history:
|
||||
@@ -953,7 +965,7 @@ class FlowController:
|
||||
"""
|
||||
broken = sorted(
|
||||
{
|
||||
entry.flow
|
||||
entry.flow_id
|
||||
for entry in self.loaded.values()
|
||||
if entry.status is NodeStatus.ERROR
|
||||
}
|
||||
@@ -963,7 +975,10 @@ class FlowController:
|
||||
return broken
|
||||
|
||||
async def reload_lib_users(self, ref: str) -> list[str]:
|
||||
"""Rebuild the flows running a shared node whose source has changed."""
|
||||
"""Rebuild the flows running a shared node whose source has changed.
|
||||
|
||||
Returns their ids, which is what `usages` qualifies with.
|
||||
"""
|
||||
usages = self.store.usages(ref, draft=False)
|
||||
flows = sorted({usage.split(".", 1)[0] for usage in usages})
|
||||
for flow in flows:
|
||||
@@ -971,18 +986,18 @@ class FlowController:
|
||||
return flows
|
||||
|
||||
def _build_one(
|
||||
self, name: str
|
||||
self, flow_id: str
|
||||
) -> tuple[FlowDef, list[Node], dict[str, LoadedNode], dict[str, Any], bool] | None:
|
||||
"""Read one published flow and build its nodes. None when it is gone.
|
||||
|
||||
Blocking: this is the part of a rebuild that talks to the worker pool.
|
||||
"""
|
||||
try:
|
||||
flow = self.store.read_flow(name)
|
||||
flow = self.store.read_flow(flow_id)
|
||||
except FlowNotFound:
|
||||
return None
|
||||
nodes, loaded, initial_values, _ = self._build_flows([(flow, False)])
|
||||
return flow, nodes, loaded, initial_values, self.store.read_enabled(name)
|
||||
return flow, nodes, loaded, initial_values, self.store.read_enabled(flow_id)
|
||||
|
||||
def _all_flow_inputs(self) -> dict[str, bool]:
|
||||
"""Every flow's declared inputs, merged the way one build would see them.
|
||||
@@ -1145,7 +1160,7 @@ class FlowController:
|
||||
|
||||
for flow, draft in flows:
|
||||
for node_def in flow.nodes:
|
||||
entry = self._build_node(flow.name, node_def, draft=draft, run=run)
|
||||
entry = self._build_node(flow, node_def, draft=draft, run=run)
|
||||
loaded[entry.id] = entry
|
||||
if entry.node is not None:
|
||||
nodes.append(entry.node)
|
||||
@@ -1157,13 +1172,16 @@ class FlowController:
|
||||
|
||||
def _build_node(
|
||||
self,
|
||||
flow: str,
|
||||
flow_def: FlowDef,
|
||||
node_def: NodeDef,
|
||||
draft: bool = False,
|
||||
run: RunContext | None = None,
|
||||
) -> LoadedNode:
|
||||
"""One node of *flow_def*: its code read by the flow's id, and run,
|
||||
named and wired by its key."""
|
||||
flow, flow_id = flow_def.name, flow_def.id
|
||||
node_id = f"{flow}.{node_def.id}"
|
||||
entry = LoadedNode(id=node_id, flow=flow)
|
||||
entry = LoadedNode(id=node_id, flow=flow, flow_id=flow_id)
|
||||
try:
|
||||
node_type = NODE_TYPES.get(node_def.type)
|
||||
if node_type is None:
|
||||
@@ -1192,19 +1210,21 @@ class FlowController:
|
||||
code = self.store.read_lib_source(node_def.source_ref)
|
||||
else:
|
||||
owner, local = flow, node_def.id
|
||||
code = self.store.read_node_source(flow, node_def.id, draft=draft)
|
||||
code = self.store.read_node_source(
|
||||
flow_id, node_def.id, draft=draft
|
||||
)
|
||||
# The store answers the new-node template when nothing was
|
||||
# written, which is what an editor should open with and not
|
||||
# something to run. A draft legitimately has none yet.
|
||||
entry.missing_source = not draft and not self.store.has_node_source(
|
||||
flow, node_def.id
|
||||
flow_id, node_def.id
|
||||
)
|
||||
|
||||
# Only a body somebody wrote: the template a new node opens
|
||||
# with takes nothing, and a node declared before it is written
|
||||
# is reported as missing its source rather than as wrong.
|
||||
written = bool(node_def.source_ref) or self.store.has_node_source(
|
||||
flow, node_def.id, draft=draft
|
||||
flow_id, node_def.id, draft=draft
|
||||
)
|
||||
if written:
|
||||
gap = _signature_gap(
|
||||
@@ -1376,8 +1396,10 @@ class FlowController:
|
||||
node._on_health = self._health_changed
|
||||
|
||||
if _is_hook(node):
|
||||
# Hooks live under their flow, away from the JSON API.
|
||||
node.url = f"{HOOK_PREFIX}/{flow}/{node.url.lstrip('/')}"
|
||||
# Hooks live under their flow, away from the JSON API: under
|
||||
# its id, so a URL handed out keeps working when the key
|
||||
# changes.
|
||||
node.url = f"{HOOK_PREFIX}/{flow_id}/{node.url.lstrip('/')}"
|
||||
|
||||
entry.node = node
|
||||
except (SecretNotFound, LibNotFound) as exc:
|
||||
@@ -1493,7 +1515,7 @@ class FlowController:
|
||||
issue for issue in self.issues if not issue.flow or issue.flow == flow
|
||||
] + self._health_issues(flow)
|
||||
|
||||
def preview(self, name: str) -> Preview:
|
||||
def preview(self, flow_id: str) -> Preview:
|
||||
"""Build a flow's unpublished draft without deploying it.
|
||||
|
||||
The draft stands in for its published self among the other flows, so
|
||||
@@ -1501,13 +1523,15 @@ class FlowController:
|
||||
graph is thrown away afterwards and seeded into a throwaway state, so
|
||||
nothing here touches what is running.
|
||||
"""
|
||||
flows: list[tuple[FlowDef, bool]] = [
|
||||
(flow, False) for flow in self.store.read_all() if flow.name != name
|
||||
]
|
||||
try:
|
||||
flows.append((self.store.read_flow(name, draft=True), True))
|
||||
draft = self.store.read_flow(flow_id, draft=True)
|
||||
except FlowNotFound:
|
||||
return Preview(nodes=[], issues=[])
|
||||
name = draft.name
|
||||
flows: list[tuple[FlowDef, bool]] = [
|
||||
(flow, False) for flow in self.store.read_all() if flow.id != flow_id
|
||||
]
|
||||
flows.append((draft, True))
|
||||
|
||||
nodes, loaded, initial_values, flow_inputs = self._build_flows(flows)
|
||||
pipeline = Pipeline(
|
||||
@@ -1856,21 +1880,35 @@ class FlowController:
|
||||
return self.execution.stats() if self.execution is not None else {}
|
||||
|
||||
def forget_flow(self, flow: str) -> None:
|
||||
"""Drop what a deleted flow left behind. Blocking."""
|
||||
"""Drop what a deleted flow left behind, by its key. Blocking.
|
||||
|
||||
After its nodes are torn down, or they put some of it straight back.
|
||||
"""
|
||||
if self.execution is not None:
|
||||
self.execution.queue.clear_flow(flow)
|
||||
prefix = f"{flow}."
|
||||
with self.state.lock():
|
||||
stale = [
|
||||
key
|
||||
for key in self.state.keys()
|
||||
# Both the messages themselves and the engine's own bookkeeping
|
||||
# about them, which is keyed by message name too.
|
||||
if key.startswith(prefix) or f":{prefix}" in key
|
||||
]
|
||||
for key in stale:
|
||||
for key in [k for k in self.state.keys() if _in_namespace(k, flow)]:
|
||||
self.state.delete(key)
|
||||
|
||||
def key_residue(self, flow: str) -> list[str]:
|
||||
"""What still lives under the flow key *flow*. Blocking.
|
||||
|
||||
State and live queue work in its namespace: left by a flow that held
|
||||
the key before, or put there by anything still writing to it. A new
|
||||
flow taking the key would inherit it, so creating one is refused while
|
||||
this is not empty, and so is renaming a flow onto it. Work that was
|
||||
acknowledged, dead-lettered or marked done is history and not counted.
|
||||
Names what it found, state keys first.
|
||||
"""
|
||||
found = [key for key in self.state.keys() if _in_namespace(key, flow)]
|
||||
if self.execution is not None:
|
||||
found += [
|
||||
f"queued work of '{item.node}'"
|
||||
for item in self.execution.queue.live_items()
|
||||
if _touches(item, flow)
|
||||
]
|
||||
return found
|
||||
|
||||
def run_flow(self, flow: str, inputs: dict[str, Any] | None = None) -> None:
|
||||
"""Run every node of one flow. Blocking — call from a worker thread."""
|
||||
if self.pipeline is None:
|
||||
@@ -1912,7 +1950,7 @@ class FlowController:
|
||||
pipeline.history_limits = self.history_limits
|
||||
return pipeline
|
||||
|
||||
def run_preview(self, flow_name: str, inputs: dict[str, Any] | None = None) -> None:
|
||||
def run_preview(self, flow_id: str, inputs: dict[str, Any] | None = None) -> None:
|
||||
"""Run a flow's unpublished draft once, as the editor shows it.
|
||||
|
||||
The nodes are built and run but never activated, so a draft never
|
||||
@@ -1920,7 +1958,7 @@ class FlowController:
|
||||
that. State and events are the live ones, so values land on the canvas
|
||||
exactly as they do for a published run.
|
||||
"""
|
||||
flow = self.store.read_flow(flow_name, draft=True)
|
||||
flow = self.store.read_flow(flow_id, draft=True)
|
||||
nodes, _, initial_values, _ = self._build_flows([(flow, True)])
|
||||
pipeline = Pipeline(
|
||||
nodes=nodes,
|
||||
@@ -1960,6 +1998,20 @@ class FlowController:
|
||||
self.events.publish(event)
|
||||
|
||||
|
||||
def _in_namespace(state_key: str, flow: str) -> bool:
|
||||
"""Whether a state key is about the flow key *flow*: one of its messages,
|
||||
or the engine's own bookkeeping on one of its messages or nodes, which is
|
||||
keyed by those names too."""
|
||||
prefix = f"{flow}."
|
||||
return state_key.startswith(prefix) or f":{prefix}" in state_key
|
||||
|
||||
|
||||
def _touches(item: WorkItem, flow: str) -> bool:
|
||||
"""Whether a queued item names anything of the flow key *flow*: as its
|
||||
flow, its node or one of the messages it carries."""
|
||||
return owned_by(item, flow) or any(flow_of(name) == flow for name in item.outputs)
|
||||
|
||||
|
||||
def _bound(specs: list[MessageSpec]) -> list[MessageSpec]:
|
||||
"""Ports without a message name are not wired into the graph."""
|
||||
return [spec for spec in specs if spec.name]
|
||||
|
||||
@@ -28,11 +28,15 @@ from fluksio.flow.messages import DType, qualify
|
||||
from fluksio.flow.schemas import FlowDef, _validate_id, _validate_name
|
||||
from fluksio.flow.store import (
|
||||
FlowStore,
|
||||
IdTaken,
|
||||
StaleVersion,
|
||||
_updated_at,
|
||||
_version,
|
||||
checked,
|
||||
claim,
|
||||
copy_name,
|
||||
named,
|
||||
new_id,
|
||||
read_keys,
|
||||
)
|
||||
|
||||
@@ -619,6 +623,27 @@ class DashboardDef(BaseModel):
|
||||
return shapes
|
||||
|
||||
|
||||
class DashboardCreate(BaseModel):
|
||||
"""A new dashboard: one empty grid under a key, filled in by saves after."""
|
||||
|
||||
#: Empty for an id the server issues, or one a superuser pins, as for a
|
||||
#: flow (`FlowCreate`). Dashboards keep no history, so it only has to be free.
|
||||
id: str = ""
|
||||
name: str
|
||||
#: Empty reads the key as words.
|
||||
title: str = ""
|
||||
|
||||
@field_validator("id")
|
||||
@classmethod
|
||||
def _check_id(cls, value: str) -> str:
|
||||
return _validate_id(value)
|
||||
|
||||
@field_validator("name")
|
||||
@classmethod
|
||||
def _check_name(cls, value: str) -> str:
|
||||
return _validate_name(value)
|
||||
|
||||
|
||||
class DashboardSummary(BaseModel):
|
||||
"""A dashboard in a list, without its contents."""
|
||||
|
||||
@@ -766,14 +791,17 @@ class DashboardStore:
|
||||
self._keys = (at, keys)
|
||||
return keys
|
||||
|
||||
def _claim(self, defn: DashboardDef) -> DashboardDef:
|
||||
def _claim(self, defn: DashboardDef, create: bool = False) -> DashboardDef:
|
||||
"""*defn* with its id, once it is certain to be a save the store allows.
|
||||
|
||||
Under the flow store's write lock; see `claim`. An id left out is the
|
||||
key, which is how every dashboard is created for now.
|
||||
Under the flow store's write lock; see `claim`. A save is to a
|
||||
dashboard that exists, named by its id or, from before ids, its key.
|
||||
Only *create* may make a new one.
|
||||
"""
|
||||
dash_id = checked(defn.id or defn.name)
|
||||
current = self.read(dash_id, draft=True) if self.exists(dash_id) else None
|
||||
if current is None and not create:
|
||||
raise DashboardNotFound(dash_id)
|
||||
claim(
|
||||
dash_id,
|
||||
defn.name,
|
||||
@@ -807,21 +835,50 @@ class DashboardStore:
|
||||
"""Publish a dashboard directly, skipping the draft.
|
||||
|
||||
The API never does: it creates a draft and promotes it. This is for a
|
||||
caller that already has the finished document — a test, or a seed.
|
||||
caller that already has the finished document, a test or a seed, and
|
||||
creates one too, under the id it carries or else its key.
|
||||
"""
|
||||
with self._lock, self.flows._write_lock:
|
||||
defn = self._claim(defn)
|
||||
defn = self._claim(defn, create=True)
|
||||
path = self._file(defn.id)
|
||||
current = 0
|
||||
if path.exists():
|
||||
current = DashboardDef.model_validate_json(path.read_text()).version
|
||||
current = _version(path)
|
||||
if base_version is not None and base_version != current:
|
||||
raise StaleVersion(defn.name, current)
|
||||
|
||||
saved = defn.model_copy(update={"version": current + 1, "has_draft": False})
|
||||
# Above the draft too, which an editor may be holding.
|
||||
top = max(current, _version(self._draft_file(defn.id)))
|
||||
saved = defn.model_copy(update={"version": top + 1, "has_draft": False})
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(self._dump(saved))
|
||||
self.flows._commit(f"Save dashboard '{defn.name}'")
|
||||
self.flows._commit(f"Save dashboard {named(defn.name, defn.id)}")
|
||||
return saved
|
||||
|
||||
def create(
|
||||
self, defn: DashboardDef, pin: str = "", published: bool = False
|
||||
) -> DashboardDef:
|
||||
"""A new dashboard, only a draft unless *published* says otherwise.
|
||||
|
||||
Its id is issued here unless *pin* names one, a superuser's to ask for
|
||||
as for a flow (`FlowStore.create`). Dashboards have no history to
|
||||
adopt, so a pin only has to be free.
|
||||
"""
|
||||
with self._lock, self.flows._write_lock:
|
||||
keys = self.key_index()
|
||||
if pin:
|
||||
dash_id = checked(pin)
|
||||
if (self.root / dash_id).exists():
|
||||
raise IdTaken(dash_id, "dashboard")
|
||||
else:
|
||||
dash_id = new_id("d", lambda c: (self.root / c).exists() or c in keys)
|
||||
defn = self._claim(defn.model_copy(update={"id": dash_id}), create=True)
|
||||
if keys.get(dash_id, dash_id) != dash_id:
|
||||
raise IdTaken(dash_id, "dashboard")
|
||||
|
||||
saved = defn.model_copy(update={"version": 1, "has_draft": not published})
|
||||
path = self._file(dash_id) if published else self._draft_file(dash_id)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(self._dump(saved))
|
||||
self.flows._commit(f"Create dashboard {named(defn.name, dash_id)}")
|
||||
return saved
|
||||
|
||||
def write_draft(
|
||||
@@ -829,23 +886,19 @@ class DashboardStore:
|
||||
) -> DashboardDef:
|
||||
"""Save unpublished changes, refusing to overwrite someone else's.
|
||||
|
||||
``base_version`` is the version the editor last saw — of the working
|
||||
copy, which is the draft once there is one, and 0 for a dashboard that
|
||||
does not exist yet: creating one is its first draft.
|
||||
``base_version`` is the version the editor last saw, of the working
|
||||
copy, which is the draft once there is one. The dashboard has to
|
||||
exist: creating one is `create`.
|
||||
"""
|
||||
with self._lock, self.flows._write_lock:
|
||||
defn = self._claim(defn)
|
||||
current = 0
|
||||
if self.exists(defn.id):
|
||||
current = self.read(defn.id, draft=True).version
|
||||
current = self.read(defn.id, draft=True).version
|
||||
if base_version is not None and base_version != current:
|
||||
raise StaleVersion(defn.name, current)
|
||||
|
||||
saved = defn.model_copy(update={"version": current + 1, "has_draft": True})
|
||||
path = self._draft_file(defn.id)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(self._dump(saved))
|
||||
self.flows._commit(f"Update draft of dashboard '{defn.name}'")
|
||||
self._draft_file(defn.id).write_text(self._dump(saved))
|
||||
self.flows._commit(f"Update draft of dashboard {named(defn.name, defn.id)}")
|
||||
return saved
|
||||
|
||||
def publish(self, name: str, base_version: int | None = None) -> DashboardDef:
|
||||
@@ -859,14 +912,24 @@ class DashboardStore:
|
||||
if draft.exists():
|
||||
self._file(name).write_text(self._dump(current))
|
||||
draft.unlink()
|
||||
self.flows._commit(f"Publish dashboard '{name}'")
|
||||
self.flows._commit(f"Publish dashboard {named(current.name, name)}")
|
||||
return current.model_copy(update={"has_draft": False})
|
||||
|
||||
def discard_draft(self, name: str) -> DashboardDef:
|
||||
"""Throw the unpublished changes away and go back to what is shown."""
|
||||
"""Throw the unpublished changes away and go back to what is shown.
|
||||
|
||||
What is shown comes back a version above the draft, as a flow's does
|
||||
(`FlowStore.discard_draft`).
|
||||
"""
|
||||
with self._lock, self.flows._write_lock:
|
||||
draft = self._draft_file(name)
|
||||
if draft.exists():
|
||||
shown = self.read(name)
|
||||
top = _version(draft)
|
||||
if shown.version <= top:
|
||||
self._file(name).write_text(
|
||||
self._dump(shown.model_copy(update={"version": top + 1}))
|
||||
)
|
||||
draft.unlink()
|
||||
self.flows._commit(f"Discard draft of dashboard '{name}'")
|
||||
return self.read(name)
|
||||
@@ -887,27 +950,32 @@ class DashboardStore:
|
||||
def duplicate(self, name: str) -> DashboardDef:
|
||||
"""Copy the working copy into a new dashboard, as a draft only.
|
||||
|
||||
Named ``{name}_copy``, then ``_copy_2`` and on. Unpublished like a new
|
||||
dashboard, so no panel shows it until someone publishes it.
|
||||
Under an id of its own and the key ``{name}_copy``, then ``_copy_2``
|
||||
and on. Unpublished like a new dashboard, so no panel shows it until
|
||||
someone publishes it.
|
||||
"""
|
||||
with self._lock, self.flows._write_lock:
|
||||
source = self.read(name, draft=True)
|
||||
# The copy's id is its key, so it has to be free as both.
|
||||
keys = self.key_index()
|
||||
new_name = copy_name(source.name, lambda n: self.exists(n) or n in keys)
|
||||
new_name = copy_name(source.name, lambda n: n in keys)
|
||||
copy_id = new_id("d", lambda c: (self.root / c).exists() or c in keys)
|
||||
copy = source.model_copy(
|
||||
update={
|
||||
"id": new_name,
|
||||
"id": copy_id,
|
||||
"name": new_name,
|
||||
"title": f"{source.title} (copy)" if source.title else "",
|
||||
"version": 1,
|
||||
"has_draft": True,
|
||||
}
|
||||
)
|
||||
path = self._draft_file(new_name)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
copy = self._claim(copy, create=True)
|
||||
path = self._draft_file(copy_id)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(self._dump(copy))
|
||||
self.flows._commit(f"Duplicate dashboard '{name}' as '{new_name}'")
|
||||
self.flows._commit(
|
||||
f"Duplicate dashboard {named(source.name, name)} as "
|
||||
f"{named(new_name, copy_id)}"
|
||||
)
|
||||
return copy
|
||||
|
||||
def bindings_for(self, flow: str) -> Bindings:
|
||||
@@ -984,11 +1052,11 @@ class DashboardStore:
|
||||
return limits
|
||||
|
||||
|
||||
def default_dashboard(name: str) -> DashboardDef:
|
||||
"""A new dashboard: one page, one section, nothing in it yet."""
|
||||
def default_dashboard(name: str, title: str = "") -> DashboardDef:
|
||||
"""A new dashboard: one empty grid, titled after its key unless told."""
|
||||
return DashboardDef(
|
||||
name=name,
|
||||
title=name.replace("_", " ").capitalize(),
|
||||
title=title or name.replace("_", " ").capitalize(),
|
||||
)
|
||||
|
||||
|
||||
@@ -1094,6 +1162,7 @@ __all__ = [
|
||||
"INPUT_WIDGETS",
|
||||
"SETTING_DTYPES",
|
||||
"WIDGET_DTYPES",
|
||||
"DashboardCreate",
|
||||
"DashboardDef",
|
||||
"DashboardExists",
|
||||
"DashboardNotFound",
|
||||
|
||||
@@ -445,7 +445,7 @@ class DeviceStore:
|
||||
if self._wiring_at != revision:
|
||||
found: dict[str, list[Binding]] = {}
|
||||
for flow in self.flows.read_all():
|
||||
if not self.flows.read_enabled(flow.name):
|
||||
if not self.flows.read_enabled(flow.id):
|
||||
continue
|
||||
for binding in bindings_of(flow):
|
||||
found.setdefault(binding.device, []).append(binding)
|
||||
|
||||
@@ -24,6 +24,7 @@ from sqlmodel import Session, col
|
||||
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.core.db import engine
|
||||
from fluksio.flow import tombstones
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.models import EngineEvent, FlowRun, MetricBucket
|
||||
|
||||
@@ -456,6 +457,9 @@ class MetricsCollector:
|
||||
session.execute(delete(MetricBucket).where(col(MetricBucket.bucket) < cutoff))
|
||||
session.execute(delete(EngineEvent).where(col(EngineEvent.ts) < cutoff))
|
||||
session.execute(delete(FlowRun).where(col(FlowRun.started_at) < cutoff))
|
||||
# After the rows above: a deleted flow's name is kept as long as any
|
||||
# of its history is.
|
||||
tombstones.prune(session)
|
||||
# A cascade still open long after it started did not finish; saying so
|
||||
# is more honest than leaving it running forever. A batch run is not
|
||||
# swept: it is allowed to take hours, and `Run` has its own lease sweep
|
||||
|
||||
@@ -161,7 +161,7 @@ class HttpNode(Node):
|
||||
|
||||
url: str = Field(
|
||||
description=(
|
||||
"A path such as /tick, served as a webhook at /hooks/<flow>/<path>, "
|
||||
"A path such as /tick, served as a webhook at /hooks/<flow id>/<path>, "
|
||||
"or the full URL a sender requests."
|
||||
)
|
||||
)
|
||||
@@ -197,7 +197,7 @@ class HttpNode(Node):
|
||||
default="",
|
||||
description=(
|
||||
"Shared secret for a webhook, appended to its URL: "
|
||||
"/hooks/<flow>/<url>/<secret>. Empty leaves the webhook open "
|
||||
"/hooks/<flow id>/<url>/<secret>. Empty leaves the webhook open "
|
||||
"to anyone."
|
||||
),
|
||||
json_schema_extra={"x-secret": True},
|
||||
|
||||
@@ -31,7 +31,7 @@ class WebSocketNode(Node):
|
||||
|
||||
The URL picks the role: ``ws://`` or ``wss://`` dials out and holds the
|
||||
connection, a path such as ``/ticker`` is served at
|
||||
``/hooks/<flow>/<path>[/<secret>]`` while the flow runs. The ports pick the
|
||||
``/hooks/<flow id>/<path>[/<secret>]`` while the flow runs. The ports pick the
|
||||
direction, as on the mqtt node: outputs only make a **listener**, inputs a
|
||||
**sender**.
|
||||
|
||||
@@ -79,7 +79,7 @@ class WebSocketNode(Node):
|
||||
url: str = Field(
|
||||
description=(
|
||||
"ws:// or wss:// to connect to a server, or a path such as "
|
||||
"/ticker to serve one at /hooks/<flow>/<path>."
|
||||
"/ticker to serve one at /hooks/<flow id>/<path>."
|
||||
)
|
||||
)
|
||||
headers: dict[str, str] = Field(
|
||||
@@ -100,7 +100,7 @@ class WebSocketNode(Node):
|
||||
default="",
|
||||
description=(
|
||||
"Shared secret for a served socket, appended to its URL: "
|
||||
"/hooks/<flow>/<url>/<secret>. Empty leaves it open to anyone."
|
||||
"/hooks/<flow id>/<url>/<secret>. Empty leaves it open to anyone."
|
||||
),
|
||||
json_schema_extra={"x-secret": True},
|
||||
)
|
||||
|
||||
@@ -151,6 +151,17 @@ class WorkItem:
|
||||
)
|
||||
|
||||
|
||||
def owned_by(item: WorkItem, flow: str) -> bool:
|
||||
"""Whether *item* is work of the flow with key *flow*, not one reaching it.
|
||||
|
||||
A run item names its run's flow by identity rather than by key, and a
|
||||
run is never removed with a flow's queued work, so it is nobody's here.
|
||||
"""
|
||||
return item.kind != "run" and (
|
||||
item.flow == flow or item.node.split(".", 1)[0] == flow
|
||||
)
|
||||
|
||||
|
||||
class WorkQueue(ABC):
|
||||
"""What the execution service pulls from."""
|
||||
|
||||
@@ -236,7 +247,21 @@ class WorkQueue(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def clear_flow(self, flow: str) -> None:
|
||||
"""Forget anything held for a flow that no longer exists."""
|
||||
"""Drop the work of a flow that no longer exists (`owned_by`).
|
||||
|
||||
What it parked, what waits on a timer, and what is queued or claimed
|
||||
and not yet acknowledged, so a new flow under the same key never runs
|
||||
it. A cascade already running finishes; its acknowledgement then has
|
||||
nothing left to remove.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def live_items(self) -> list[WorkItem]:
|
||||
"""Every item still to run: queued, claimed, delayed or parked.
|
||||
|
||||
Not what was acknowledged, dead-lettered or marked done, which is
|
||||
history. Reads the whole queue, so it is for rare questions.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def stats(self) -> dict[str, Any]:
|
||||
@@ -388,6 +413,23 @@ class MemoryWorkQueue(WorkQueue):
|
||||
def clear_flow(self, flow: str) -> None:
|
||||
with self._lock:
|
||||
self._parked.pop(flow, None)
|
||||
for lane in (self._items, self._due):
|
||||
kept = [item for item in lane if not owned_by(item, flow)]
|
||||
lane.clear()
|
||||
lane.extend(kept)
|
||||
self._delayed = [e for e in self._delayed if not owned_by(e[2], flow)]
|
||||
heapq.heapify(self._delayed)
|
||||
|
||||
def live_items(self) -> list[WorkItem]:
|
||||
"""What is waiting. A claimed item is gone from here once it is handed
|
||||
out, so unlike Redis this cannot say what is running."""
|
||||
with self._lock:
|
||||
return [
|
||||
*self._items,
|
||||
*self._due,
|
||||
*(entry[2] for entry in self._delayed),
|
||||
*(item for held in self._parked.values() for item in held),
|
||||
]
|
||||
|
||||
def backlog(self) -> int:
|
||||
with self._lock:
|
||||
@@ -683,8 +725,68 @@ class RedisWorkQueue(WorkQueue):
|
||||
pipe = self._redis.pipeline()
|
||||
pipe.delete(self._parked_key(flow))
|
||||
pipe.srem(self._parked_flows_key, flow)
|
||||
# Timers before the streams: one promoted in between is in the due
|
||||
# stream by the time that is read.
|
||||
for raw in cast(list[str], self._redis.zrange(self._delayed_key, 0, -1)):
|
||||
if owned_by(WorkItem.from_fields(orjson.loads(raw), ""), flow):
|
||||
pipe.zrem(self._delayed_key, raw)
|
||||
for stream in (self._stream, self._due_stream):
|
||||
ids = [
|
||||
entry_id
|
||||
for entry_id, fields in self._live_entries(stream)
|
||||
if owned_by(WorkItem.from_fields(fields, entry_id), flow)
|
||||
]
|
||||
if ids:
|
||||
pipe.xack(stream, GROUP, *ids)
|
||||
pipe.xdel(stream, *ids)
|
||||
pipe.execute()
|
||||
|
||||
def _live_entries(self, stream: str) -> list[tuple[str, dict[str, str]]]:
|
||||
"""The entries of *stream* still to run: claimed and unacknowledged,
|
||||
then never handed out. Acknowledged ones stay in a stream until it is
|
||||
trimmed, and are history."""
|
||||
group = self._group_info(stream)
|
||||
if not group:
|
||||
return []
|
||||
last = str(group.get("last-delivered-id") or "0-0")
|
||||
live: list[tuple[str, dict[str, str]]] = []
|
||||
pending = int(group.get("pending") or 0)
|
||||
if pending:
|
||||
rows = cast(
|
||||
list[dict[str, Any]],
|
||||
self._redis.xpending_range(
|
||||
stream, GROUP, min="-", max="+", count=pending
|
||||
),
|
||||
)
|
||||
ids = {row["message_id"] for row in rows}
|
||||
if ids:
|
||||
# In id order, so the first is the oldest still held.
|
||||
entries = self._redis.xrange(
|
||||
stream, min=rows[0]["message_id"], max=last
|
||||
)
|
||||
live += [(i, f) for i, f in cast(Any, entries) if i in ids]
|
||||
live += cast(Any, self._redis.xrange(stream, min=f"({last}", max="+"))
|
||||
return live
|
||||
|
||||
def live_items(self) -> list[WorkItem]:
|
||||
items = [
|
||||
WorkItem.from_fields(fields, prefix + entry_id)
|
||||
for stream, prefix in ((self._due_stream, DUE_PREFIX), (self._stream, ""))
|
||||
for entry_id, fields in self._live_entries(stream)
|
||||
]
|
||||
items += [
|
||||
WorkItem.from_fields(orjson.loads(raw), "")
|
||||
for raw in cast(list[str], self._redis.zrange(self._delayed_key, 0, -1))
|
||||
]
|
||||
for flow in cast(set[str], self._redis.smembers(self._parked_flows_key)):
|
||||
items += [
|
||||
WorkItem.from_fields(orjson.loads(raw), "")
|
||||
for raw in cast(
|
||||
list[str], self._redis.lrange(self._parked_key(flow), 0, -1)
|
||||
)
|
||||
]
|
||||
return items
|
||||
|
||||
def _group_info(self, stream: str = "") -> dict[str, Any]:
|
||||
"""This consumer group's row of ``XINFO GROUPS``, or an empty one."""
|
||||
try:
|
||||
|
||||
@@ -155,7 +155,7 @@ def batch_issues(
|
||||
"the value would be dropped. Remove the interval, or mark "
|
||||
"the port as streaming if it is a curve being thinned out."
|
||||
)
|
||||
if store is not None and _has_no_body(store, flow.name, node, draft):
|
||||
if store is not None and _has_no_body(store, flow.id, node, draft):
|
||||
issues.append(
|
||||
f"Node '{node.id}' has no stored code. It would run as an "
|
||||
"empty node and publish nothing, so the run is refused."
|
||||
|
||||
@@ -352,6 +352,36 @@ class FlowDef(BaseModel):
|
||||
return _validate_name(value)
|
||||
|
||||
|
||||
class FlowCreate(BaseModel):
|
||||
"""A new flow as a client hands it over: a document but for what the
|
||||
server owns. The fields mean what they do on `FlowDef`, but ``id``."""
|
||||
|
||||
#: Empty for an id the server issues. Anything else pins it, which only a
|
||||
#: superuser may: an id that is free, or whose history this flow resumes.
|
||||
id: str = ""
|
||||
name: str
|
||||
title: str = ""
|
||||
nodes: list[NodeDef] = Field(default_factory=list)
|
||||
inputs: list[FlowInput] = Field(default_factory=list)
|
||||
mode: Literal["live", "batch"] = "live"
|
||||
outputs: list[str] = Field(default_factory=list)
|
||||
origin: FlowOrigin | None = None
|
||||
|
||||
@field_validator("id")
|
||||
@classmethod
|
||||
def _check_id(cls, value: str) -> str:
|
||||
return _validate_id(value)
|
||||
|
||||
@field_validator("name")
|
||||
@classmethod
|
||||
def _check_name(cls, value: str) -> str:
|
||||
return _validate_name(value)
|
||||
|
||||
def document(self) -> FlowDef:
|
||||
"""The flow to store, before the store gives it an id."""
|
||||
return FlowDef.model_validate(self.model_dump(exclude={"id"}))
|
||||
|
||||
|
||||
class NodeSource(BaseModel):
|
||||
"""The Python source of a node."""
|
||||
|
||||
|
||||
+208
-68
@@ -14,8 +14,9 @@ unpublished — which is what every flow written before this existed looks like.
|
||||
A flow is two names. Its id is which flow it is: the directory, the argument
|
||||
every method here takes, and never changed once given. Its key, ``name``, is
|
||||
the namespace of its messages and unique among flows. A flow from before ids
|
||||
existed has its key as its id (`fluksio.flow.identity`), and so does every new
|
||||
one for now. Published and draft documents always share both.
|
||||
existed has its key as its id (`fluksio.flow.identity`). A new one is given an
|
||||
opaque id by `create`, or the one a superuser pinned. Published and draft
|
||||
documents always share both.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -31,10 +32,13 @@ import threading
|
||||
from collections.abc import Callable, Iterable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
from fluksio.flow.schemas import FlowDef, _validate_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fluksio.flow.tombstones import Tombstones
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: Where the sources of nodes shared between flows live.
|
||||
@@ -97,6 +101,37 @@ class IdentityChange(ValueError):
|
||||
"""
|
||||
|
||||
|
||||
class RecordedKey(IdentityChange):
|
||||
"""A pinned id restored under another key than its history is filed under.
|
||||
|
||||
The history names the old key's messages, so taking a new key with it is
|
||||
a rename, not a create.
|
||||
"""
|
||||
|
||||
def __init__(self, obj_id: str, key: str) -> None:
|
||||
super().__init__(obj_id)
|
||||
self.id = obj_id
|
||||
self.key = key
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (
|
||||
f"'{self.id}' has history filed under the key '{self.key}', so it "
|
||||
"can only be restored with that key"
|
||||
)
|
||||
|
||||
|
||||
class IdTaken(ValueError):
|
||||
"""A pinned id that is in use: an object's, a directory's or another key."""
|
||||
|
||||
def __init__(self, obj_id: str, kind: str = "flow") -> None:
|
||||
super().__init__(obj_id)
|
||||
self.id = obj_id
|
||||
self.kind = kind
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"The id '{self.id}' is already taken by a {self.kind}"
|
||||
|
||||
|
||||
def claim(
|
||||
obj_id: str,
|
||||
key: str,
|
||||
@@ -146,6 +181,11 @@ def read_keys(directories: Iterable[Path], document: str) -> dict[str, str]:
|
||||
return keys
|
||||
|
||||
|
||||
def named(key: str, obj_id: str) -> str:
|
||||
"""How a commit message names an object: by key, and by id when it differs."""
|
||||
return f"'{key}'" if key == obj_id else f"'{key}' ({obj_id})"
|
||||
|
||||
|
||||
def new_id(kind: Literal["f", "d"], taken: Callable[[str], bool]) -> str:
|
||||
"""A fresh id for a flow (``f``) or a dashboard (``d``), e.g. ``f_k3q7m2xa``.
|
||||
|
||||
@@ -222,6 +262,14 @@ def _updated_at(paths: Iterable[Path]) -> datetime | None:
|
||||
return datetime.fromtimestamp(max(stamps), UTC) if stamps else None
|
||||
|
||||
|
||||
def _version(path: Path) -> int:
|
||||
"""The version a stored document carries, 0 for none or an unreadable one."""
|
||||
try:
|
||||
return int(json.loads(path.read_text()).get("version", 1))
|
||||
except (OSError, ValueError, TypeError, AttributeError):
|
||||
return 0
|
||||
|
||||
|
||||
def _same_content(left: FlowDef, right: FlowDef) -> bool:
|
||||
"""Equal but for the version counter, which the server owns."""
|
||||
return left.model_copy(update={"version": 0}) == right.model_copy(
|
||||
@@ -252,8 +300,11 @@ class FlowStore:
|
||||
#: a store reached 1898 commits, 5863 loose objects and no pack at all.
|
||||
_GC_EVERY = 500
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
def __init__(self, root: Path, tombstones: Tombstones | None = None) -> None:
|
||||
self.root = root
|
||||
#: Where a deleted flow's key is kept, and what history an id has.
|
||||
#: None leaves both out: a store without a database, as in tests.
|
||||
self.tombstones = tombstones
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
# Draft writes are check-and-set, so two clients saving at once must not
|
||||
# interleave between reading the current version and writing the next.
|
||||
@@ -679,14 +730,16 @@ class FlowStore:
|
||||
self._keys = (at, keys)
|
||||
return keys
|
||||
|
||||
def _claim(self, flow: FlowDef) -> FlowDef:
|
||||
def _claim(self, flow: FlowDef, create: bool = False) -> FlowDef:
|
||||
"""*flow* with its id, once it is certain to be a save the store allows.
|
||||
|
||||
Under `_write_lock`. An id left out is the key, which is how every
|
||||
flow is created for now.
|
||||
Under `_write_lock`. A save is to a flow that exists, named by its id
|
||||
or, from before ids, by its key. Only *create* may make a new one.
|
||||
"""
|
||||
flow_id = checked(flow.id or flow.name)
|
||||
current = self.read_flow(flow_id, draft=True) if self.exists(flow_id) else None
|
||||
if current is None and not create:
|
||||
raise FlowNotFound(flow_id)
|
||||
claim(
|
||||
flow_id,
|
||||
flow.name,
|
||||
@@ -696,6 +749,22 @@ class FlowStore:
|
||||
)
|
||||
return flow.model_copy(update={"id": flow_id})
|
||||
|
||||
def _issue(self, keys: dict[str, str]) -> str:
|
||||
"""A fresh flow id. Under `_write_lock`, with *keys* the key index.
|
||||
|
||||
Free as a directory, a key and in history: an id a deleted flow's runs
|
||||
are still filed under would hand them to the new one.
|
||||
"""
|
||||
graves = self.tombstones
|
||||
return new_id(
|
||||
"f",
|
||||
lambda c: (
|
||||
self._flow_dir(c).exists()
|
||||
or c in keys
|
||||
or (graves is not None and graves.recorded(c) is not None)
|
||||
),
|
||||
)
|
||||
|
||||
def read_all(self) -> list[FlowDef]:
|
||||
"""Every published flow — what the engine runs.
|
||||
|
||||
@@ -718,9 +787,14 @@ class FlowStore:
|
||||
return list(self._read_all)
|
||||
|
||||
def write_flow(self, flow: FlowDef) -> bool:
|
||||
"""Publish a flow directly. Returns False when nothing actually changed."""
|
||||
"""Publish a flow directly. Returns False when nothing actually changed.
|
||||
|
||||
Creates one too, under the id it carries or else its key: what a test
|
||||
or a seed holding a finished document wants. The API creates only
|
||||
through `create`.
|
||||
"""
|
||||
with self._write_lock:
|
||||
flow = self._claim(flow)
|
||||
flow = self._claim(flow, create=True)
|
||||
path = self._flow_file(flow.id)
|
||||
content = flow.model_dump_json(indent=2) + "\n"
|
||||
if path.exists() and path.read_text() == content:
|
||||
@@ -728,122 +802,188 @@ class FlowStore:
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content)
|
||||
self._commit(f"Update flow '{flow.name}'")
|
||||
self._commit(f"Update flow {named(flow.name, flow.id)}")
|
||||
return True
|
||||
|
||||
def create(self, flow: FlowDef, pin: str = "") -> FlowDef:
|
||||
"""A new flow, only a draft until someone publishes it.
|
||||
|
||||
Its id is issued here unless *pin* names one, which is a superuser's
|
||||
to ask for: the Demo, a test, a restore. A pinned id whose history
|
||||
outlived its flow adopts that history, so it has to take the key the
|
||||
history is filed under, and its tombstone goes once the document is on
|
||||
disk.
|
||||
"""
|
||||
with self._write_lock:
|
||||
keys = self.key_index()
|
||||
if pin:
|
||||
flow_id = checked(pin)
|
||||
# The directory rather than `exists`: anything left in it would
|
||||
# become the new flow's.
|
||||
if self._flow_dir(flow_id).exists():
|
||||
raise IdTaken(flow_id)
|
||||
recorded = (
|
||||
self.tombstones.recorded(flow_id) if self.tombstones else None
|
||||
)
|
||||
if recorded is not None and recorded != flow.name:
|
||||
raise RecordedKey(flow_id, recorded)
|
||||
else:
|
||||
flow_id = self._issue(keys)
|
||||
flow = self._claim(flow.model_copy(update={"id": flow_id}), create=True)
|
||||
# Another flow's key as this one's id would make naming either
|
||||
# ambiguous. Checked after the key, which is the likelier mistake.
|
||||
if keys.get(flow_id, flow_id) != flow_id:
|
||||
raise IdTaken(flow_id)
|
||||
|
||||
stored = flow.model_copy(update={"version": 1})
|
||||
path = self._draft_file(flow_id)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(stored.model_dump_json(indent=2) + "\n")
|
||||
self._commit(f"Create flow {named(flow.name, flow_id)}")
|
||||
if pin and self.tombstones is not None:
|
||||
self.tombstones.lift(flow_id)
|
||||
return stored
|
||||
|
||||
def write_draft(self, flow: FlowDef, base_version: int) -> FlowDef:
|
||||
"""Save unpublished changes, refusing to overwrite someone else's.
|
||||
|
||||
``base_version`` is the version the editor last saw. It has to match
|
||||
what is stored, otherwise another client saved in between and this
|
||||
write would silently discard their work.
|
||||
write would silently discard their work. The flow has to exist:
|
||||
creating one is `create`.
|
||||
"""
|
||||
with self._write_lock:
|
||||
flow = self._claim(flow)
|
||||
current: FlowDef | None = None
|
||||
if self.exists(flow.id):
|
||||
current = self.read_flow(flow.id, draft=True)
|
||||
if base_version != current.version:
|
||||
raise StaleVersion(flow.name, current.version)
|
||||
if _same_content(current, flow):
|
||||
return current
|
||||
current = self.read_flow(flow.id, draft=True)
|
||||
if base_version != current.version:
|
||||
raise StaleVersion(flow.name, current.version)
|
||||
if _same_content(current, flow):
|
||||
return current
|
||||
|
||||
stored = flow.model_copy(
|
||||
update={"version": (current.version if current else 0) + 1}
|
||||
)
|
||||
stored = flow.model_copy(update={"version": current.version + 1})
|
||||
path = self._draft_file(flow.id)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(stored.model_dump_json(indent=2) + "\n")
|
||||
self._commit(f"Update draft of flow '{flow.name}'")
|
||||
self._commit(f"Update draft of flow {named(flow.name, flow.id)}")
|
||||
return stored
|
||||
|
||||
def publish_flow(self, name: str, base_version: int) -> FlowDef:
|
||||
def publish_flow(self, flow_id: str, base_version: int) -> FlowDef:
|
||||
"""Promote the working copy to what the engine runs."""
|
||||
with self._write_lock:
|
||||
current = self.read_flow(name, draft=True)
|
||||
current = self.read_flow(flow_id, draft=True)
|
||||
if base_version != current.version:
|
||||
raise StaleVersion(name, current.version)
|
||||
raise StaleVersion(current.name, current.version)
|
||||
|
||||
draft = self._draft_file(name)
|
||||
draft = self._draft_file(flow_id)
|
||||
if draft.exists():
|
||||
self._flow_file(name).write_text(
|
||||
self._flow_file(flow_id).write_text(
|
||||
current.model_dump_json(indent=2) + "\n"
|
||||
)
|
||||
draft.unlink()
|
||||
|
||||
drafts = self._draft_nodes_dir(name)
|
||||
drafts = self._draft_nodes_dir(flow_id)
|
||||
if drafts.exists():
|
||||
target = self._flow_dir(name) / "nodes"
|
||||
target = self._flow_dir(flow_id) / "nodes"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for source in sorted(drafts.glob("*.py")):
|
||||
shutil.copy2(source, target / source.name)
|
||||
shutil.rmtree(drafts)
|
||||
|
||||
self._commit(f"Publish flow '{name}'")
|
||||
self._commit(f"Publish flow {named(current.name, flow_id)}")
|
||||
return current
|
||||
|
||||
def discard_draft(self, name: str) -> FlowDef:
|
||||
"""Throw the unpublished changes away and go back to what is running."""
|
||||
with self._write_lock:
|
||||
draft = self._draft_file(name)
|
||||
if draft.exists():
|
||||
draft.unlink()
|
||||
drafts = self._draft_nodes_dir(name)
|
||||
if drafts.exists():
|
||||
shutil.rmtree(drafts)
|
||||
self._commit(f"Discard draft of flow '{name}'")
|
||||
return self.read_flow(name)
|
||||
def discard_draft(self, flow_id: str) -> FlowDef:
|
||||
"""Throw the unpublished changes away and go back to what is running.
|
||||
|
||||
def delete_flow(self, name: str) -> None:
|
||||
with self._write_lock:
|
||||
directory = self._flow_dir(name)
|
||||
if not directory.exists():
|
||||
raise FlowNotFound(name)
|
||||
shutil.rmtree(directory)
|
||||
self._commit(f"Delete flow '{name}'")
|
||||
|
||||
def duplicate_flow(self, name: str) -> FlowDef:
|
||||
"""Copy a flow's working copy into a new flow that is only a draft.
|
||||
|
||||
Like a new flow, the copy runs nothing until someone publishes it. It is
|
||||
named ``{name}_copy``, then ``_copy_2`` and on. ``origin`` is dropped,
|
||||
since the copy is a local flow ``fluksio sync`` must not claim, and
|
||||
``runtime.json`` stays behind with everything else about running.
|
||||
What is running comes back a version above the draft, so an editor
|
||||
still holding the draft is refused rather than matching the next save
|
||||
made from the published copy.
|
||||
"""
|
||||
with self._write_lock:
|
||||
source = self.read_flow(name, draft=True)
|
||||
# The directory rather than `exists`: anything left in it would
|
||||
# otherwise end up in the copy. The copy's id is its key, so that
|
||||
# has to be free among the keys too.
|
||||
draft = self._draft_file(flow_id)
|
||||
if draft.exists():
|
||||
running = self.read_flow(flow_id)
|
||||
top = _version(draft)
|
||||
if running.version <= top:
|
||||
running.version = top + 1
|
||||
self._flow_file(flow_id).write_text(
|
||||
running.model_dump_json(indent=2) + "\n"
|
||||
)
|
||||
draft.unlink()
|
||||
drafts = self._draft_nodes_dir(flow_id)
|
||||
if drafts.exists():
|
||||
shutil.rmtree(drafts)
|
||||
published = self.read_flow(flow_id)
|
||||
self._commit(f"Discard draft of flow {named(published.name, flow_id)}")
|
||||
return published
|
||||
|
||||
def delete_flow(self, flow_id: str) -> str:
|
||||
"""Remove a flow's files. Returns the key it had.
|
||||
|
||||
The key is buried first, so a delete that stops half-way never leaves
|
||||
history without a name, and the live document wins while it is there.
|
||||
"""
|
||||
with self._write_lock:
|
||||
directory = self._flow_dir(flow_id)
|
||||
if not directory.exists():
|
||||
raise FlowNotFound(flow_id)
|
||||
key = next(
|
||||
(k for k, i in self.key_index().items() if i == flow_id), flow_id
|
||||
)
|
||||
if self.tombstones is not None:
|
||||
self.tombstones.bury(flow_id, key)
|
||||
shutil.rmtree(directory)
|
||||
self._commit(f"Delete flow {named(key, flow_id)}")
|
||||
return key
|
||||
|
||||
def duplicate_flow(
|
||||
self, flow_id: str, busy: Callable[[str], bool] | None = None
|
||||
) -> FlowDef:
|
||||
"""Copy a flow's working copy into a new flow that is only a draft.
|
||||
|
||||
Like a new flow, the copy runs nothing until someone publishes it. It
|
||||
gets an id of its own and the key ``{name}_copy``, then ``_copy_2`` and
|
||||
on, past any key *busy* says something still holds. ``origin`` is
|
||||
dropped, since the copy is a local flow ``fluksio sync`` must not
|
||||
claim, and ``runtime.json`` stays behind with everything else about
|
||||
running.
|
||||
"""
|
||||
with self._write_lock:
|
||||
source = self.read_flow(flow_id, draft=True)
|
||||
keys = self.key_index()
|
||||
new_name = copy_name(
|
||||
source.name, lambda n: self._flow_dir(n).exists() or n in keys
|
||||
source.name, lambda n: n in keys or (busy is not None and busy(n))
|
||||
)
|
||||
|
||||
copy = source.model_copy(
|
||||
update={
|
||||
"id": new_name,
|
||||
"id": self._issue(keys),
|
||||
"name": new_name,
|
||||
"title": f"{source.title} (copy)" if source.title else "",
|
||||
"version": 1,
|
||||
"origin": None,
|
||||
}
|
||||
)
|
||||
copy = self._claim(copy, create=True)
|
||||
# Its own messages named with its own key are the copy's now.
|
||||
self._retarget(copy, f"{source.name}.", f"{new_name}.")
|
||||
path = self._draft_file(new_name)
|
||||
path = self._draft_file(copy.id)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(copy.model_dump_json(indent=2) + "\n")
|
||||
|
||||
# The bodies the source's editor shows: published, edits over them.
|
||||
# As drafts, so publishing the copy promotes them like a new flow's.
|
||||
target = self._draft_nodes_dir(new_name)
|
||||
for folder in (self._flow_dir(name) / "nodes", self._draft_nodes_dir(name)):
|
||||
target = self._draft_nodes_dir(copy.id)
|
||||
for folder in (
|
||||
self._flow_dir(flow_id) / "nodes",
|
||||
self._draft_nodes_dir(flow_id),
|
||||
):
|
||||
for body in sorted(folder.glob("*.py")):
|
||||
target.mkdir(exist_ok=True)
|
||||
shutil.copy2(body, target / body.name)
|
||||
|
||||
self._commit(f"Duplicate flow '{name}' as '{new_name}'")
|
||||
self._commit(
|
||||
f"Duplicate flow {named(source.name, flow_id)} as "
|
||||
f"{named(new_name, copy.id)}"
|
||||
)
|
||||
return copy
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""What is left of a deleted flow: its last key, while its history is kept.
|
||||
|
||||
A flow's runs, events and metrics are filed under its id and outlive it. A new
|
||||
flow's id is opaque, so a tombstone keeps the key it last had, and `names_of`
|
||||
reads that back for anything showing the history. A flow deleted before
|
||||
tombstones existed has its old key as its id, which already reads right.
|
||||
|
||||
The flow store is the one writer: it buries a flow before removing its files,
|
||||
asks before issuing an id, and lifts the tombstone of a flow restored under a
|
||||
pinned id once the restored document is on disk.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import delete, or_
|
||||
from sqlmodel import Session, col, select
|
||||
|
||||
from fluksio.core.db import engine
|
||||
from fluksio.models import EngineEvent, FlowRun, FlowTombstone, MetricBucket, Run
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fluksio.flow.store import FlowStore
|
||||
|
||||
|
||||
def referenced(session: Session, flow_id: str) -> bool:
|
||||
"""Whether any history row names *flow_id*, as its flow or a node's.
|
||||
|
||||
A training run's ``for_node`` names the node it trains, in a flow other
|
||||
than the run's own, so the four ``flow`` columns are not enough.
|
||||
"""
|
||||
nodes = f"{flow_id}."
|
||||
queries = [
|
||||
select(Run.id).where(
|
||||
or_(
|
||||
col(Run.flow) == flow_id,
|
||||
col(Run.for_node).startswith(nodes, autoescape=True),
|
||||
)
|
||||
),
|
||||
select(FlowRun.id).where(col(FlowRun.flow) == flow_id),
|
||||
select(MetricBucket.flow).where(
|
||||
or_(
|
||||
col(MetricBucket.flow) == flow_id,
|
||||
col(MetricBucket.node).startswith(nodes, autoescape=True),
|
||||
)
|
||||
),
|
||||
select(EngineEvent.id).where(
|
||||
or_(
|
||||
col(EngineEvent.flow) == flow_id,
|
||||
col(EngineEvent.node).startswith(nodes, autoescape=True),
|
||||
)
|
||||
),
|
||||
]
|
||||
return any(session.exec(query.limit(1)).first() is not None for query in queries)
|
||||
|
||||
|
||||
class Tombstones:
|
||||
"""Tombstones and the history behind an id, as the flow store asks."""
|
||||
|
||||
def bury(self, flow_id: str, name: str) -> None:
|
||||
"""Keep *name* for *flow_id*, before its files go.
|
||||
|
||||
Written again by a retried delete, which may find another name if the
|
||||
first attempt never got as far as removing anything.
|
||||
"""
|
||||
with Session(engine) as session:
|
||||
stone = session.get(FlowTombstone, flow_id) or FlowTombstone(
|
||||
id=flow_id, name=name
|
||||
)
|
||||
stone.name = name
|
||||
session.add(stone)
|
||||
session.commit()
|
||||
|
||||
def lift(self, flow_id: str) -> None:
|
||||
"""Forget the tombstone of a flow that is live again."""
|
||||
with Session(engine) as session:
|
||||
session.execute(
|
||||
delete(FlowTombstone).where(col(FlowTombstone.id) == flow_id)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
def recorded(self, flow_id: str) -> str | None:
|
||||
"""The key the history of a flow that is not live is filed under.
|
||||
|
||||
Its tombstone's, or the id itself for history from before tombstones,
|
||||
whose id was its key. None when nothing names the id at all.
|
||||
"""
|
||||
with Session(engine) as session:
|
||||
stone = session.get(FlowTombstone, flow_id)
|
||||
if stone is not None:
|
||||
return stone.name
|
||||
return flow_id if referenced(session, flow_id) else None
|
||||
|
||||
|
||||
def names_of(ids: Iterable[str], store: FlowStore) -> dict[str, str]:
|
||||
"""Each flow id's key: the live flow's, else its tombstone's, else the id.
|
||||
|
||||
One read of the key index and one query, however many ids a response
|
||||
carries.
|
||||
"""
|
||||
wanted = set(ids)
|
||||
live = {flow_id: key for key, flow_id in store.key_index().items()}
|
||||
names = {flow_id: live[flow_id] for flow_id in wanted if flow_id in live}
|
||||
gone = wanted - names.keys()
|
||||
if gone:
|
||||
with Session(engine) as session:
|
||||
for stone in session.exec(
|
||||
select(FlowTombstone).where(col(FlowTombstone.id).in_(gone))
|
||||
):
|
||||
names[stone.id] = stone.name
|
||||
return {flow_id: names.get(flow_id, flow_id) for flow_id in wanted}
|
||||
|
||||
|
||||
def name_of(flow_id: str, store: FlowStore) -> str:
|
||||
"""One flow id's key, as `names_of`."""
|
||||
return names_of([flow_id], store)[flow_id]
|
||||
|
||||
|
||||
def prune(session: Session) -> int:
|
||||
"""Drop the tombstones no history row names any more. Returns how many."""
|
||||
stale = [
|
||||
stone.id
|
||||
for stone in session.exec(select(FlowTombstone))
|
||||
if not referenced(session, stone.id)
|
||||
]
|
||||
if stale:
|
||||
session.execute(delete(FlowTombstone).where(col(FlowTombstone.id).in_(stale)))
|
||||
return len(stale)
|
||||
@@ -60,6 +60,7 @@ from fluksio.flow.runs import RUN_STATE_TTL, RunService, sweep_artifacts
|
||||
from fluksio.flow.secrets import init_secrets
|
||||
from fluksio.flow.state import MemoryState, RedisState, StateBackend
|
||||
from fluksio.flow.store import FlowStore, IdentityChange, InvalidName
|
||||
from fluksio.flow.tombstones import Tombstones
|
||||
from fluksio.flow.watchdog import LoopWatchdog
|
||||
from fluksio.flow.workers import PythonWorkerPool
|
||||
|
||||
@@ -69,7 +70,7 @@ logger = logging.getLogger(__name__)
|
||||
#: worker's `?token=`, the pairing poll's `?secret=`.
|
||||
_CREDENTIAL_IN_QUERY = re.compile(r"([?&](?:[\w-]*token|[\w-]*secret|ticket)=)[^&\s]+")
|
||||
|
||||
#: A hook's secret, the last segment of `/hooks/<flow>/<path>/<secret>`. A hook
|
||||
#: A hook's secret, the last segment of `/hooks/<flow id>/<path>/<secret>`. A hook
|
||||
#: without one loses its last path segment from the log too, since the log
|
||||
#: cannot tell which hooks carry a secret.
|
||||
_SECRET_IN_HOOK_PATH = re.compile(r"(/hooks/[^/\s?]+/(?:[^/\s?]*/)*)[^/\s?]+")
|
||||
@@ -306,7 +307,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
# store's own `_migrate`, the device migrations below and every cache
|
||||
# after them write documents back with whatever id they read.
|
||||
await run_in_threadpool(assign_ids, settings.FLOWS_DIR)
|
||||
store = FlowStore(settings.FLOWS_DIR)
|
||||
store = FlowStore(settings.FLOWS_DIR, tombstones=Tombstones())
|
||||
# The packages node code imports. In the background, because a first
|
||||
# install of anything substantial takes minutes and `uv` is given five
|
||||
# of them twice over — held here, the container never answers its
|
||||
|
||||
@@ -607,6 +607,25 @@ class AlertSent(SQLModel, table=True):
|
||||
ts: float = Field(index=True)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Flow tombstones
|
||||
#
|
||||
# A deleted flow's runs, events and metrics stay, filed under its id. New ids
|
||||
# are opaque, so without the key the flow last had they would name nothing
|
||||
# anyone recognises.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FlowTombstone(SQLModel, table=True):
|
||||
"""A deleted flow's last key, kept while history names its id."""
|
||||
|
||||
__tablename__ = "flow_tombstone"
|
||||
|
||||
id: str = Field(primary_key=True, max_length=255)
|
||||
name: str = Field(max_length=255)
|
||||
deleted_at: datetime = Field(default_factory=get_datetime_utc, sa_type=UTCDateTime)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Worker tokens
|
||||
#
|
||||
|
||||
@@ -238,7 +238,7 @@ class Client:
|
||||
# -- flows -------------------------------------------------------------
|
||||
|
||||
def get_flow(self, name: str) -> dict[str, Any] | None:
|
||||
"""The stored flow, draft included, or ``None`` if there is none."""
|
||||
"""The stored flow with the id *name*, draft included, or ``None``."""
|
||||
try:
|
||||
result: dict[str, Any] = self._call("GET", f"/flows/{seg(name)}")
|
||||
except ApiError as exc:
|
||||
@@ -247,9 +247,23 @@ class Client:
|
||||
raise
|
||||
return result
|
||||
|
||||
def find_flow(self, key: str) -> dict[str, Any] | None:
|
||||
"""The listing of the flow holding the key *key*, or ``None``."""
|
||||
result = self._call("GET", "/flows/", params={"name": key})
|
||||
found = list(result.get("data") or [])
|
||||
return found[0] if found else None
|
||||
|
||||
def create_flow(self, document: dict[str, Any]) -> dict[str, Any]:
|
||||
"""A new flow, under the id the engine issues for it."""
|
||||
result: dict[str, Any] = self._call("POST", "/flows/", json=document)
|
||||
return result
|
||||
|
||||
def put_flow(self, document: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Save over the flow *document* names by its id, or by its key for
|
||||
one whose id is its key."""
|
||||
flow_id = document.get("id") or document["name"]
|
||||
result: dict[str, Any] = self._call(
|
||||
"PUT", f"/flows/{seg(document['name'])}", json=document
|
||||
"PUT", f"/flows/{seg(flow_id)}", json=document
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -821,7 +835,11 @@ def _sync_one(
|
||||
force: bool,
|
||||
) -> SyncReport:
|
||||
report = SyncReport(target.name)
|
||||
stored = client.get_flow(target.name)
|
||||
# The declared key is what an instance shares; the id is this engine's,
|
||||
# and every request after the lookup names the flow by it.
|
||||
found = client.find_flow(target.name)
|
||||
flow_id = str(found["id"]) if found else ""
|
||||
stored = client.get_flow(flow_id) if flow_id else None
|
||||
version = 1
|
||||
# What the drift check already read, so the write loop below does not ask
|
||||
# the engine for every node's body a second time.
|
||||
@@ -836,7 +854,11 @@ def _sync_one(
|
||||
|
||||
document = target.document(origin)
|
||||
moved = _moved_code(stored, document)
|
||||
saved = client.put_flow(document | {"version": version})
|
||||
if stored is None:
|
||||
saved = client.create_flow(document)
|
||||
flow_id = str((saved.get("definition") or {}).get("id") or target.name)
|
||||
else:
|
||||
saved = client.put_flow(document | {"id": flow_id, "version": version})
|
||||
# An engine older than the per-node digest parses the document without
|
||||
# those fields and stores it without them, silently: no error, no 404, and
|
||||
# a cache still keyed on the whole repository.
|
||||
@@ -860,16 +882,16 @@ def _sync_one(
|
||||
if not report.created:
|
||||
stored_code = known.get(node_id)
|
||||
if stored_code is None:
|
||||
stored_code = client.get_source(target.name, node_id)
|
||||
stored_code = client.get_source(flow_id, node_id)
|
||||
if stored_code == code:
|
||||
continue
|
||||
client.put_source(target.name, node_id, code)
|
||||
client.put_source(flow_id, node_id, code)
|
||||
if node_id not in report.changed:
|
||||
report.changed.append(node_id)
|
||||
|
||||
has_draft = bool((client.get_flow(target.name) or {}).get("has_draft"))
|
||||
has_draft = bool((client.get_flow(flow_id) or {}).get("has_draft"))
|
||||
if publish and has_draft:
|
||||
client.publish(target.name, version)
|
||||
client.publish(flow_id, version)
|
||||
report.published = True
|
||||
else:
|
||||
# Only when one is actually there. "Nothing needed publishing" and
|
||||
@@ -934,7 +956,9 @@ def _refuse_on_drift(
|
||||
for stored_node in definition.get("nodes") or []:
|
||||
node_id = str(stored_node.get("id"))
|
||||
try:
|
||||
entry = client.get_source_entry(target.name, node_id)
|
||||
entry = client.get_source_entry(
|
||||
str(definition.get("id") or target.name), node_id
|
||||
)
|
||||
except ApiError:
|
||||
continue
|
||||
if entry.get("missing"):
|
||||
|
||||
@@ -89,9 +89,10 @@ def ensure_flow(api: Api) -> None:
|
||||
try:
|
||||
current = api.call("GET", f"/api/v1/flows/{BENCH_FLOW}")
|
||||
definition["version"] = current["definition"]["version"]
|
||||
api.call("PUT", f"/api/v1/flows/{BENCH_FLOW}", definition)
|
||||
except urllib.error.HTTPError:
|
||||
pass
|
||||
api.call("PUT", f"/api/v1/flows/{BENCH_FLOW}", definition)
|
||||
# Pinned to its key, so every path below can name it.
|
||||
api.call("POST", "/api/v1/flows/", {**definition, "id": BENCH_FLOW})
|
||||
api.call(
|
||||
"PUT", f"/api/v1/flows/{BENCH_FLOW}/nodes/noop/source", {"code": NODE_SOURCE}
|
||||
)
|
||||
|
||||
@@ -354,9 +354,13 @@ def create_flow(
|
||||
definition: dict[str, Any],
|
||||
sources: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
"""Draft, node source, publish — the sequence the editor uses."""
|
||||
"""Draft, node source, publish — the sequence the editor uses.
|
||||
|
||||
Pinned to its key, which the superuser running this may, so the hooks and
|
||||
paths below can name it.
|
||||
"""
|
||||
name = definition["name"]
|
||||
response = client.put(f"/api/v1/flows/{name}", json=definition)
|
||||
response = client.post("/api/v1/flows/", json={**definition, "id": name})
|
||||
response.raise_for_status()
|
||||
version = response.json()["definition"]["version"]
|
||||
|
||||
|
||||
@@ -36,10 +36,10 @@ def save(
|
||||
) -> Callable[..., Any]:
|
||||
"""Seed the flow as a superuser, then save a change as someone else."""
|
||||
client.delete(f"{PREFIX}/{FLOW}", headers=superuser_token_headers)
|
||||
seeded = client.put(
|
||||
f"{PREFIX}/{FLOW}", headers=superuser_token_headers, json=a_flow(0)
|
||||
seeded = client.post(
|
||||
f"{PREFIX}/", headers=superuser_token_headers, json={**a_flow(0), "id": FLOW}
|
||||
)
|
||||
assert seeded.status_code == 200, seeded.text
|
||||
assert seeded.status_code == 201, seeded.text
|
||||
version = seeded.json()["definition"]["version"]
|
||||
|
||||
def put(headers: dict[str, str], **changes: Any) -> Any:
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
"""Dashboards over HTTP: saving one, which is also how the first one is made."""
|
||||
"""Dashboards over HTTP: creating one under an issued id, and saving it."""
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.flow.store import ISSUED_ID
|
||||
from tests.utils.flows import put_dashboard
|
||||
|
||||
PREFIX = f"{settings.API_V1_STR}/dashboards"
|
||||
|
||||
@@ -26,37 +28,78 @@ def a_dashboard(name: str) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def test_a_save_creates_a_dashboard_that_does_not_exist_yet(
|
||||
def test_a_dashboard_is_created_with_an_id_and_saved_by_it(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
"""A first draft, not a 404: creating one *is* saving it at version 0."""
|
||||
body = a_dashboard("put_creates")
|
||||
|
||||
response = client.put(
|
||||
f"{PREFIX}/put_creates", headers=superuser_token_headers, json=body
|
||||
"""A first draft under an id the server issues, which every path then names."""
|
||||
created = client.post(
|
||||
f"{PREFIX}/", headers=superuser_token_headers, json={"name": "made_here"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
saved = response.json()
|
||||
assert saved["version"] == 1 and saved["has_draft"] is True
|
||||
assert created.status_code == 201, created.text
|
||||
made = created.json()
|
||||
assert ISSUED_ID.fullmatch(made["id"]) and made["name"] == "made_here"
|
||||
assert made["version"] == 1 and made["has_draft"] is True
|
||||
# A draft alone: nothing was published, so no panel can be shown it.
|
||||
assert (
|
||||
client.get(f"{PREFIX}/put_creates", headers=superuser_token_headers).status_code
|
||||
client.get(
|
||||
f"{PREFIX}/{made['id']}", headers=superuser_token_headers
|
||||
).status_code
|
||||
== 404
|
||||
)
|
||||
body = {**a_dashboard("made_here"), "version": made["version"]}
|
||||
saved = client.put(
|
||||
f"{PREFIX}/{made['id']}", headers=superuser_token_headers, json=body
|
||||
)
|
||||
assert saved.status_code == 200, saved.text
|
||||
draft = client.get(
|
||||
f"{PREFIX}/put_creates?draft=true", headers=superuser_token_headers
|
||||
f"{PREFIX}/{made['id']}?draft=true", headers=superuser_token_headers
|
||||
)
|
||||
assert draft.json()["widgets"] == body["widgets"]
|
||||
(listed,) = client.get(
|
||||
f"{PREFIX}/?name=made_here", headers=superuser_token_headers
|
||||
).json()["data"]
|
||||
assert listed["id"] == made["id"]
|
||||
|
||||
copy = client.post(
|
||||
f"{PREFIX}/{made['id']}/duplicate", headers=superuser_token_headers
|
||||
).json()
|
||||
assert copy["name"] == "made_here_copy" and copy["id"] != made["id"]
|
||||
for dashboard_id in (made["id"], copy["id"]):
|
||||
client.delete(f"{PREFIX}/{dashboard_id}", headers=superuser_token_headers)
|
||||
|
||||
|
||||
def test_a_save_neither_creates_nor_renames(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
missing = client.put(
|
||||
f"{PREFIX}/nowhere",
|
||||
headers=superuser_token_headers,
|
||||
json={**a_dashboard("nowhere"), "version": 0},
|
||||
)
|
||||
assert missing.status_code == 404
|
||||
saved = put_dashboard(
|
||||
client, superuser_token_headers, a_dashboard("put_fixed")
|
||||
).json()
|
||||
renamed = client.put(
|
||||
f"{PREFIX}/put_fixed",
|
||||
headers=superuser_token_headers,
|
||||
json={**saved, "name": "renamed"},
|
||||
)
|
||||
assert renamed.status_code == 409
|
||||
elsewhere = client.put(
|
||||
f"{PREFIX}/put_fixed",
|
||||
headers=superuser_token_headers,
|
||||
json={**saved, "id": "elsewhere"},
|
||||
)
|
||||
assert elsewhere.status_code == 422
|
||||
client.delete(f"{PREFIX}/put_fixed", headers=superuser_token_headers)
|
||||
|
||||
|
||||
def test_a_save_of_an_existing_dashboard_keeps_what_it_did_not_change(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
seeded = a_dashboard("put_updates")
|
||||
first = client.put(
|
||||
f"{PREFIX}/put_updates", headers=superuser_token_headers, json=seeded
|
||||
).json()
|
||||
first = put_dashboard(client, superuser_token_headers, seeded).json()
|
||||
|
||||
renamed = {**first, "widgets": [{**first["widgets"][0], "title": "Outside"}]}
|
||||
second = client.put(
|
||||
@@ -78,37 +121,38 @@ def test_a_save_based_on_a_version_someone_moved_past_is_refused(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
body = a_dashboard("put_conflicts")
|
||||
client.put(f"{PREFIX}/put_conflicts", headers=superuser_token_headers, json=body)
|
||||
put_dashboard(client, superuser_token_headers, body)
|
||||
|
||||
stale = client.put(
|
||||
f"{PREFIX}/put_conflicts", headers=superuser_token_headers, json=body
|
||||
f"{PREFIX}/put_conflicts",
|
||||
headers=superuser_token_headers,
|
||||
json={**body, "version": 1},
|
||||
)
|
||||
|
||||
assert stale.status_code == 409
|
||||
assert stale.json()["detail"]["current_version"] == 1
|
||||
assert stale.json()["detail"]["current_version"] == 2
|
||||
|
||||
|
||||
def test_a_dashboard_is_known_by_an_id_no_save_changes(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
def test_only_a_superuser_pins_a_dashboard_id(
|
||||
client: TestClient,
|
||||
superuser_token_headers: dict[str, str],
|
||||
normal_user_token_headers: dict[str, str],
|
||||
) -> None:
|
||||
saved = client.put(
|
||||
f"{PREFIX}/put_identified",
|
||||
pinned = {"id": "pinned_hall", "name": "pinned_hall"}
|
||||
refused = client.post(f"{PREFIX}/", headers=normal_user_token_headers, json=pinned)
|
||||
assert refused.status_code == 403
|
||||
made = client.post(f"{PREFIX}/", headers=superuser_token_headers, json=pinned)
|
||||
assert made.status_code == 201 and made.json()["id"] == "pinned_hall"
|
||||
again = client.post(
|
||||
f"{PREFIX}/",
|
||||
headers=superuser_token_headers,
|
||||
json=a_dashboard("put_identified"),
|
||||
).json()
|
||||
assert saved["id"] == "put_identified"
|
||||
|
||||
elsewhere = client.put(
|
||||
f"{PREFIX}/put_identified",
|
||||
headers=superuser_token_headers,
|
||||
json={**saved, "id": "elsewhere"},
|
||||
json={"id": "pinned_hall", "name": "other_hall"},
|
||||
)
|
||||
assert elsewhere.status_code == 422
|
||||
renamed = client.post(
|
||||
f"{PREFIX}/put_identified/rename",
|
||||
headers=superuser_token_headers,
|
||||
json={"name": "renamed"},
|
||||
assert again.status_code == 409
|
||||
# Anyone may make one under an id of the server's.
|
||||
issued = client.post(
|
||||
f"{PREFIX}/", headers=normal_user_token_headers, json={"name": "their_hall"}
|
||||
)
|
||||
assert renamed.status_code in (404, 405)
|
||||
|
||||
client.delete(f"{PREFIX}/put_identified", headers=superuser_token_headers)
|
||||
assert issued.status_code == 201
|
||||
for dashboard_id in ("pinned_hall", issued.json()["id"]):
|
||||
client.delete(f"{PREFIX}/{dashboard_id}", headers=superuser_token_headers)
|
||||
|
||||
@@ -12,6 +12,7 @@ from fluksio.cloud import config as cloud_config
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.flow.devices import DeviceState
|
||||
from fluksio.main import app
|
||||
from tests.utils.flows import put_flow
|
||||
|
||||
PREFIX = f"{settings.API_V1_STR}/devices"
|
||||
#: What the openpanel firmware calls, kept as aliases.
|
||||
@@ -54,8 +55,10 @@ def _dashboard(client: TestClient, headers: dict[str, str], name: str) -> dict:
|
||||
|
||||
A new one is a draft, and a panel is only ever shown what is published.
|
||||
"""
|
||||
created = client.post(f"{DASHBOARDS}/{name}", headers=headers)
|
||||
assert created.status_code == 200, created.text
|
||||
created = client.post(
|
||||
f"{DASHBOARDS}/", headers=headers, json={"id": name, "name": name}
|
||||
)
|
||||
assert created.status_code == 201, created.text
|
||||
published = client.post(
|
||||
f"{DASHBOARDS}/{name}/publish",
|
||||
headers=headers,
|
||||
@@ -95,7 +98,11 @@ def test_assign_and_read_back(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
for name in ("hall_a", "hall_b"):
|
||||
client.post(f"{DASHBOARDS}/{name}", headers=superuser_token_headers)
|
||||
client.post(
|
||||
f"{DASHBOARDS}/",
|
||||
headers=superuser_token_headers,
|
||||
json={"id": name, "name": name},
|
||||
)
|
||||
|
||||
_panels(
|
||||
client,
|
||||
@@ -132,7 +139,11 @@ def test_touch_belongs_to_the_device(
|
||||
The same dashboard may hang on a hallway tablet and on a desk browser, so
|
||||
the flag rides on the panel and each one answers for itself.
|
||||
"""
|
||||
client.post(f"{DASHBOARDS}/shared", headers=superuser_token_headers)
|
||||
client.post(
|
||||
f"{DASHBOARDS}/",
|
||||
headers=superuser_token_headers,
|
||||
json={"id": "shared", "name": "shared"},
|
||||
)
|
||||
_panels(
|
||||
client,
|
||||
superuser_token_headers,
|
||||
@@ -553,12 +564,8 @@ def _publish_flow(
|
||||
) -> dict:
|
||||
"""Write a flow and publish it, replacing whatever was there."""
|
||||
client.delete(f"{FLOWS}/{name}", headers=headers)
|
||||
saved = client.put(
|
||||
f"{FLOWS}/{name}",
|
||||
headers=headers,
|
||||
json={"name": name, "nodes": nodes, "version": 0, **more},
|
||||
)
|
||||
assert saved.status_code == 200, saved.text
|
||||
saved = put_flow(client, headers, {"name": name, "nodes": nodes, **more}, name)
|
||||
assert saved.is_success, saved.text
|
||||
published = client.post(
|
||||
f"{FLOWS}/{name}/publish",
|
||||
headers=headers,
|
||||
@@ -590,10 +597,10 @@ def test_a_devices_io_comes_from_its_device_nodes(
|
||||
_panels(client, superuser_token_headers, {"panels": [{"id": "porch_io"}]})
|
||||
_publish_flow(client, superuser_token_headers, "porch", [PORCH_NODE])
|
||||
# A draft wires nothing until it is published.
|
||||
client.put(
|
||||
f"{FLOWS}/porch_draft",
|
||||
headers=superuser_token_headers,
|
||||
json={
|
||||
put_flow(
|
||||
client,
|
||||
superuser_token_headers,
|
||||
{
|
||||
"name": "porch_draft",
|
||||
"nodes": [
|
||||
_device_node(
|
||||
@@ -603,6 +610,7 @@ def test_a_devices_io_comes_from_its_device_nodes(
|
||||
],
|
||||
"version": 0,
|
||||
},
|
||||
"porch_draft",
|
||||
)
|
||||
|
||||
first = client.get(f"{PREFIX}/porch_io", headers=superuser_token_headers)
|
||||
@@ -792,10 +800,10 @@ def test_a_port_wired_twice_or_an_unknown_device_is_an_issue(
|
||||
"ports": [{"port": "touch", "direction": "in", "dtype": "bool"}],
|
||||
}
|
||||
store._write_states(states)
|
||||
client.put(
|
||||
f"{FLOWS}/porch_too",
|
||||
headers=superuser_token_headers,
|
||||
json={
|
||||
put_flow(
|
||||
client,
|
||||
superuser_token_headers,
|
||||
{
|
||||
"name": "porch_too",
|
||||
"version": client.get(
|
||||
f"{FLOWS}/porch_too", headers=superuser_token_headers
|
||||
@@ -816,6 +824,7 @@ def test_a_port_wired_twice_or_an_unknown_device_is_an_issue(
|
||||
)
|
||||
],
|
||||
},
|
||||
"porch_too",
|
||||
)
|
||||
assert codes("porch_too") == {
|
||||
("device_port", "relay", True),
|
||||
@@ -1440,7 +1449,11 @@ def test_a_panel_credential_cannot_move_artifacts(
|
||||
the scope check has to be handed the request there too — without it a panel
|
||||
would act as whoever approved it.
|
||||
"""
|
||||
client.post(f"{DASHBOARDS}/panel_art", headers=superuser_token_headers)
|
||||
client.post(
|
||||
f"{DASHBOARDS}/",
|
||||
headers=superuser_token_headers,
|
||||
json={"id": "panel_art", "name": "panel_art"},
|
||||
)
|
||||
_panels(
|
||||
client,
|
||||
superuser_token_headers,
|
||||
|
||||
@@ -7,6 +7,7 @@ from sqlmodel import Session, select
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.core.db import seed_flavors
|
||||
from fluksio.models import Flavor
|
||||
from tests.utils.flows import put_flow
|
||||
|
||||
|
||||
def test_an_instance_starts_with_sizes_to_pick_from(
|
||||
@@ -97,12 +98,11 @@ def test_a_flow_cannot_name_a_size_that_does_not_exist(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
"""Caught at the save, which covers the canvas and `fluksio sync` at once."""
|
||||
response = client.put(
|
||||
f"{settings.API_V1_STR}/flows/flavor_check",
|
||||
response = client.post(
|
||||
f"{settings.API_V1_STR}/flows/",
|
||||
headers=superuser_token_headers,
|
||||
json={
|
||||
"name": "flavor_check",
|
||||
"version": 1,
|
||||
"nodes": [
|
||||
{"id": "fit", "type": "python", "resources": {"flavor": "imaginary"}}
|
||||
],
|
||||
@@ -120,12 +120,11 @@ def test_a_size_a_node_still_asks_for_is_not_deleted(
|
||||
headers=superuser_token_headers,
|
||||
json={"name": "in-use", "cpus": 2},
|
||||
)
|
||||
client.put(
|
||||
f"{settings.API_V1_STR}/flows/flavor_user",
|
||||
headers=superuser_token_headers,
|
||||
json={
|
||||
put_flow(
|
||||
client,
|
||||
superuser_token_headers,
|
||||
{
|
||||
"name": "flavor_user",
|
||||
"version": 1,
|
||||
"nodes": [
|
||||
{"id": "fit", "type": "python", "resources": {"flavor": "in-use"}}
|
||||
],
|
||||
|
||||
@@ -0,0 +1,341 @@
|
||||
"""Flows over HTTP by id, with keys apart from them.
|
||||
|
||||
A flow made through the API gets an opaque id unless a superuser pins one, so
|
||||
most tests here run on both: a pinned ``x_<key>`` and an issued id. A path
|
||||
that quietly read the key where it meant the id passes for a flow whose id is
|
||||
its key, and nowhere else.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import Session, delete, select
|
||||
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.flow.store import ISSUED_ID
|
||||
from fluksio.flow.tombstones import names_of
|
||||
from fluksio.models import FlowTombstone, Run
|
||||
|
||||
FLOWS = f"{settings.API_V1_STR}/flows"
|
||||
DASHBOARDS = f"{settings.API_V1_STR}/dashboards"
|
||||
|
||||
SENSOR = "def process():\n return {'reading': 21.5}\n"
|
||||
DOUBLE = "def process(reading):\n return {'doubled': reading * 2}\n"
|
||||
|
||||
|
||||
def a_flow(key: str, mode: str = "live") -> dict[str, Any]:
|
||||
return {
|
||||
"name": key,
|
||||
"mode": mode,
|
||||
"nodes": [
|
||||
{
|
||||
"id": "sensor",
|
||||
"type": "python",
|
||||
"provides": [{"name": "reading", "dtype": "float"}],
|
||||
},
|
||||
{
|
||||
"id": "double",
|
||||
"type": "python",
|
||||
"requires": [{"name": "reading", "dtype": "float"}],
|
||||
"provides": [{"name": "doubled", "dtype": "float"}],
|
||||
},
|
||||
{
|
||||
"id": "hook",
|
||||
"type": "http",
|
||||
"params": {"url": "/tick", "method": "POST"},
|
||||
"provides": [{"name": "tick", "dtype": "float"}],
|
||||
},
|
||||
],
|
||||
"outputs": ["doubled"] if mode == "batch" else [],
|
||||
}
|
||||
|
||||
|
||||
def a_key() -> str:
|
||||
return f"ids_{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
Make = Callable[..., dict[str, Any]]
|
||||
|
||||
|
||||
@pytest.fixture(params=["pinned", "issued"])
|
||||
def make(
|
||||
request: pytest.FixtureRequest,
|
||||
client: TestClient,
|
||||
superuser_token_headers: dict[str, str],
|
||||
) -> Make:
|
||||
"""Create a flow, pinned to ``x_<key>`` or under an issued id; its detail."""
|
||||
made: list[str] = []
|
||||
|
||||
def create(key: str, **document: Any) -> dict[str, Any]:
|
||||
body = {**a_flow(key), **document}
|
||||
if request.param == "pinned":
|
||||
body["id"] = f"x_{key}"
|
||||
response = client.post(f"{FLOWS}/", headers=superuser_token_headers, json=body)
|
||||
assert response.status_code == 201, response.text
|
||||
detail: dict[str, Any] = response.json()
|
||||
made.append(detail["definition"]["id"])
|
||||
return detail
|
||||
|
||||
yield create # type: ignore[misc]
|
||||
for flow_id in made:
|
||||
client.delete(f"{FLOWS}/{flow_id}", headers=superuser_token_headers)
|
||||
|
||||
|
||||
def test_a_flow_is_made_found_and_saved_by_its_id(
|
||||
make: Make, client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
key = a_key()
|
||||
definition = make(key)["definition"]
|
||||
flow_id = definition["id"]
|
||||
assert flow_id != key and definition["name"] == key
|
||||
assert flow_id == f"x_{key}" or ISSUED_ID.fullmatch(flow_id)
|
||||
|
||||
read = client.get(f"{FLOWS}/{flow_id}", headers=superuser_token_headers)
|
||||
assert read.json()["definition"]["name"] == key
|
||||
# By its key it is not there: only a client resolves a key.
|
||||
assert client.get(
|
||||
f"{FLOWS}/{key}", headers=superuser_token_headers
|
||||
).status_code == (404)
|
||||
(listed,) = client.get(
|
||||
f"{FLOWS}/?name={key}", headers=superuser_token_headers
|
||||
).json()["data"]
|
||||
assert (listed["id"], listed["name"]) == (flow_id, key)
|
||||
assert (
|
||||
client.get(f"{FLOWS}/?name={key}_no", headers=superuser_token_headers).json()[
|
||||
"data"
|
||||
]
|
||||
== []
|
||||
)
|
||||
|
||||
saved = client.put(
|
||||
f"{FLOWS}/{flow_id}",
|
||||
headers=superuser_token_headers,
|
||||
json={**definition, "title": "Retitled"},
|
||||
)
|
||||
assert saved.status_code == 200, saved.text
|
||||
assert saved.json()["definition"]["title"] == "Retitled"
|
||||
renamed = client.put(
|
||||
f"{FLOWS}/{flow_id}",
|
||||
headers=superuser_token_headers,
|
||||
json={**saved.json()["definition"], "name": f"{key}_b"},
|
||||
)
|
||||
assert renamed.status_code == 409
|
||||
|
||||
copy = client.post(
|
||||
f"{FLOWS}/{flow_id}/duplicate", headers=superuser_token_headers
|
||||
).json()["definition"]
|
||||
assert copy["name"] == f"{key}_copy" and ISSUED_ID.fullmatch(copy["id"])
|
||||
client.delete(f"{FLOWS}/{copy['id']}", headers=superuser_token_headers)
|
||||
|
||||
|
||||
def test_a_flow_runs_under_its_key_and_is_reached_by_its_id(
|
||||
make: Make, client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
"""Publish, a message, the nodes running, the state under the key."""
|
||||
key = a_key()
|
||||
flow_id = make(key)["definition"]["id"]
|
||||
for node, code in (("sensor", SENSOR), ("double", DOUBLE)):
|
||||
saved = client.put(
|
||||
f"{FLOWS}/{flow_id}/nodes/{node}/source",
|
||||
headers=superuser_token_headers,
|
||||
json={"code": code},
|
||||
)
|
||||
assert saved.json()["id"] == f"{key}.{node}"
|
||||
version = client.get(f"{FLOWS}/{flow_id}", headers=superuser_token_headers).json()[
|
||||
"definition"
|
||||
]["version"]
|
||||
published = client.post(
|
||||
f"{FLOWS}/{flow_id}/publish",
|
||||
headers=superuser_token_headers,
|
||||
json={"version": version},
|
||||
)
|
||||
assert published.status_code == 200, published.text
|
||||
assert {n["id"]: n["status"] for n in published.json()["nodes"]} == {
|
||||
f"{key}.sensor": "active",
|
||||
f"{key}.double": "active",
|
||||
f"{key}.hook": "active",
|
||||
}
|
||||
|
||||
ran = client.post(
|
||||
f"{FLOWS}/{flow_id}/run", headers=superuser_token_headers, json={"inputs": {}}
|
||||
)
|
||||
assert ran.json()["values"][f"{key}.doubled"]["value"] == 43.0
|
||||
triggered = client.post(
|
||||
f"{FLOWS}/{flow_id}/nodes/double/trigger",
|
||||
headers=superuser_token_headers,
|
||||
json={"values": {f"{key}.reading": 1.0}},
|
||||
)
|
||||
assert triggered.json()["values"][f"{key}.doubled"]["value"] == 2.0
|
||||
history = client.get(
|
||||
f"{FLOWS}/{flow_id}/history/doubled", headers=superuser_token_headers
|
||||
).json()
|
||||
assert history["message"] == f"{key}.doubled" and history["points"]
|
||||
|
||||
# The webhook is under the id, which a later key change leaves alone.
|
||||
hooked = client.post(f"/hooks/{flow_id}/tick", json={"tick": 7.0})
|
||||
assert hooked.status_code == 200, hooked.text
|
||||
assert client.post(f"/hooks/{key}/tick", json={"tick": 8.0}).status_code == 404
|
||||
state = client.get(f"{FLOWS}/{flow_id}/state", headers=superuser_token_headers)
|
||||
assert state.json()["values"][f"{key}.tick"]["value"] == 7.0
|
||||
|
||||
paused = client.post(f"{FLOWS}/{flow_id}/pause", headers=superuser_token_headers)
|
||||
assert paused.status_code == 200
|
||||
assert client.get(f"{FLOWS}/{flow_id}", headers=superuser_token_headers).json()[
|
||||
"paused"
|
||||
]
|
||||
client.post(f"{FLOWS}/{flow_id}/resume", headers=superuser_token_headers)
|
||||
stopped = client.post(f"{FLOWS}/{flow_id}/stop", headers=superuser_token_headers)
|
||||
assert stopped.json()["enabled"] is False
|
||||
assert (
|
||||
client.post(
|
||||
f"{FLOWS}/{flow_id}/run",
|
||||
headers=superuser_token_headers,
|
||||
json={"inputs": {}},
|
||||
).status_code
|
||||
== 409
|
||||
)
|
||||
started = client.post(f"{FLOWS}/{flow_id}/start", headers=superuser_token_headers)
|
||||
assert started.json()["enabled"] is True
|
||||
acknowledged = client.post(
|
||||
f"{FLOWS}/{flow_id}/nodes/double/acknowledge", headers=superuser_token_headers
|
||||
)
|
||||
assert acknowledged.status_code == 200
|
||||
|
||||
|
||||
def test_a_results_dashboard_is_made_from_a_flow_id_under_the_flows_key(
|
||||
make: Make, client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
key = a_key()
|
||||
flow_id = make(key, mode="batch")["definition"]["id"]
|
||||
client.post(
|
||||
f"{FLOWS}/{flow_id}/publish",
|
||||
headers=superuser_token_headers,
|
||||
json={"version": 1},
|
||||
)
|
||||
|
||||
made = client.post(
|
||||
f"{DASHBOARDS}/from-flow/{flow_id}", headers=superuser_token_headers
|
||||
)
|
||||
assert made.status_code == 201, made.text
|
||||
board = made.json()
|
||||
assert board["name"] == f"{key}_results" and ISSUED_ID.fullmatch(board["id"])
|
||||
assert board["has_draft"] is False
|
||||
again = client.post(
|
||||
f"{DASHBOARDS}/from-flow/{flow_id}", headers=superuser_token_headers
|
||||
)
|
||||
assert again.status_code == 409
|
||||
by_key = client.post(
|
||||
f"{DASHBOARDS}/from-flow/{key}", headers=superuser_token_headers
|
||||
)
|
||||
assert by_key.status_code == 404
|
||||
client.delete(f"{DASHBOARDS}/{board['id']}", headers=superuser_token_headers)
|
||||
|
||||
|
||||
def test_who_may_create_what(
|
||||
client: TestClient,
|
||||
superuser_token_headers: dict[str, str],
|
||||
normal_user_token_headers: dict[str, str],
|
||||
) -> None:
|
||||
key = a_key()
|
||||
plain = {"name": key, "nodes": [{"id": "hold", "type": "delay"}]}
|
||||
|
||||
# Anyone makes a flow under an issued id, as anyone could save one before.
|
||||
made = client.post(f"{FLOWS}/", headers=normal_user_token_headers, json=plain)
|
||||
assert made.status_code == 201, made.text
|
||||
flow_id = made.json()["definition"]["id"]
|
||||
assert ISSUED_ID.fullmatch(flow_id)
|
||||
# Code stays a superuser's to put there, and so does choosing an id.
|
||||
coded = client.post(
|
||||
f"{FLOWS}/",
|
||||
headers=normal_user_token_headers,
|
||||
json={**plain, "name": f"{key}_code", "nodes": [{"id": "fn"}]},
|
||||
)
|
||||
assert coded.status_code == 403
|
||||
pinned = client.post(
|
||||
f"{FLOWS}/",
|
||||
headers=normal_user_token_headers,
|
||||
json={**plain, "name": f"{key}_pin", "id": f"x_{key}"},
|
||||
)
|
||||
assert pinned.status_code == 403
|
||||
|
||||
def pin(flow: str, name: str) -> int:
|
||||
return client.post(
|
||||
f"{FLOWS}/",
|
||||
headers=superuser_token_headers,
|
||||
json={**plain, "name": name, "id": flow},
|
||||
).status_code
|
||||
|
||||
assert pin("Not-An-Id", f"{key}_a") == 422
|
||||
assert pin("graph", f"{key}_b") == 422 # behind `GET /flows/graph`
|
||||
assert pin(flow_id, f"{key}_c") == 409 # a live flow's
|
||||
assert pin(f"x_{key}", key) == 409 # the key is taken
|
||||
assert pin(f"x_{key}", "f_k3q7m2xa") == 400 # shaped like an issued id
|
||||
|
||||
client.delete(f"{FLOWS}/{flow_id}", headers=superuser_token_headers)
|
||||
|
||||
|
||||
def test_a_deleted_flow_keeps_its_name_and_frees_its_key(
|
||||
make: Make,
|
||||
client: TestClient,
|
||||
superuser_token_headers: dict[str, str],
|
||||
db: Session,
|
||||
) -> None:
|
||||
key = a_key()
|
||||
old = make(key)["definition"]["id"]
|
||||
db.add(Run(id=f"run-{key}", flow=old, status="ok", created_at=datetime.now(UTC)))
|
||||
db.commit()
|
||||
controller = client.app.state.flow_controller # type: ignore[attr-defined]
|
||||
controller.state.set(f"{key}.doubled", 1.0)
|
||||
|
||||
deleted = client.delete(f"{FLOWS}/{old}", headers=superuser_token_headers)
|
||||
assert deleted.status_code == 200
|
||||
assert db.get(FlowTombstone, old).name == key
|
||||
assert names_of([old], controller.store) == {old: key}
|
||||
# What the old flow left under its key went with it.
|
||||
assert controller.key_residue(key) == []
|
||||
|
||||
# Something still writing there keeps the key from a new flow.
|
||||
controller.state.set(f"{key}.stray", 1.0)
|
||||
refused = client.post(
|
||||
f"{FLOWS}/", headers=superuser_token_headers, json=a_flow(key)
|
||||
)
|
||||
assert refused.status_code == 409 and f"{key}.stray" in refused.json()["detail"]
|
||||
controller.forget_flow(key)
|
||||
|
||||
# Under an id of its own, not the one the runs are filed under.
|
||||
made = client.post(f"{FLOWS}/", headers=superuser_token_headers, json=a_flow(key))
|
||||
assert made.status_code == 201, made.text
|
||||
new = made.json()["definition"]["id"]
|
||||
assert new != old
|
||||
assert db.exec(select(Run).where(Run.flow == new)).first() is None
|
||||
client.delete(f"{FLOWS}/{new}", headers=superuser_token_headers)
|
||||
db.exec(delete(Run).where(Run.id == f"run-{key}")) # type: ignore[call-overload]
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_a_pinned_restore_takes_its_history_back(
|
||||
client: TestClient, superuser_token_headers: dict[str, str], db: Session
|
||||
) -> None:
|
||||
key = a_key()
|
||||
pin = f"x_{key}"
|
||||
plain = {"name": key, "nodes": []}
|
||||
client.post(f"{FLOWS}/", headers=superuser_token_headers, json={**plain, "id": pin})
|
||||
client.delete(f"{FLOWS}/{pin}", headers=superuser_token_headers)
|
||||
|
||||
other = client.post(
|
||||
f"{FLOWS}/",
|
||||
headers=superuser_token_headers,
|
||||
json={**plain, "name": f"{key}_b", "id": pin},
|
||||
)
|
||||
assert other.status_code == 409
|
||||
restored = client.post(
|
||||
f"{FLOWS}/", headers=superuser_token_headers, json={**plain, "id": pin}
|
||||
)
|
||||
assert restored.status_code == 201, restored.text
|
||||
db.expire_all()
|
||||
assert db.get(FlowTombstone, pin) is None
|
||||
client.delete(f"{FLOWS}/{pin}", headers=superuser_token_headers)
|
||||
@@ -11,6 +11,7 @@ from fluksio.api.routes import flows as flows_route
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.flow.events import event_bus
|
||||
from fluksio.models import Run, RunArtifact, RunMetric, RunNode
|
||||
from tests.utils.flows import put_flow
|
||||
|
||||
PREFIX = f"{settings.API_V1_STR}/flows"
|
||||
|
||||
@@ -51,10 +52,8 @@ def test_flows_require_authentication(client: TestClient) -> None:
|
||||
def test_save_then_read_flow(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
response = client.put(
|
||||
f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow()
|
||||
)
|
||||
assert response.status_code == 200
|
||||
response = put_flow(client, superuser_token_headers, a_flow(), "demo")
|
||||
assert response.status_code == 201
|
||||
|
||||
response = client.get(f"{PREFIX}/demo", headers=superuser_token_headers)
|
||||
assert response.status_code == 200
|
||||
@@ -68,58 +67,35 @@ def test_save_then_read_flow(
|
||||
assert datetime.fromisoformat(summary["updated_at"]).tzinfo is not None
|
||||
|
||||
|
||||
def test_name_mismatch_is_rejected(
|
||||
def test_a_save_neither_creates_nor_renames(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
response = client.put(
|
||||
f"{PREFIX}/other", headers=superuser_token_headers, json=a_flow("demo")
|
||||
put_flow(client, superuser_token_headers, a_flow(), "demo")
|
||||
current = client.get(f"{PREFIX}/demo", headers=superuser_token_headers).json()
|
||||
document = current["definition"]
|
||||
|
||||
missing = client.put(
|
||||
f"{PREFIX}/nowhere", headers=superuser_token_headers, json=a_flow("nowhere")
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def test_a_flow_is_known_by_an_id_no_save_changes(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
"""For now the id of a new flow is its key, and the URL names both."""
|
||||
saved = client.put(
|
||||
f"{PREFIX}/identified",
|
||||
assert missing.status_code == 404
|
||||
renamed = client.put(
|
||||
f"{PREFIX}/demo",
|
||||
headers=superuser_token_headers,
|
||||
json=a_flow("identified"),
|
||||
).json()["definition"]
|
||||
assert saved["id"] == "identified"
|
||||
listed = client.get(f"{PREFIX}/", headers=superuser_token_headers).json()["data"]
|
||||
assert {"id": "identified", "name": "identified"}.items() <= next(
|
||||
f for f in listed if f["name"] == "identified"
|
||||
).items()
|
||||
|
||||
json={**document, "name": "other"},
|
||||
)
|
||||
assert renamed.status_code == 409
|
||||
elsewhere = client.put(
|
||||
f"{PREFIX}/identified",
|
||||
f"{PREFIX}/demo",
|
||||
headers=superuser_token_headers,
|
||||
json={**saved, "id": "elsewhere"},
|
||||
json={**document, "id": "elsewhere"},
|
||||
)
|
||||
assert elsewhere.status_code == 400
|
||||
# A key changes by a rename, and that route is not there yet.
|
||||
renamed = client.post(
|
||||
f"{PREFIX}/identified/rename",
|
||||
headers=superuser_token_headers,
|
||||
json={"new_name": "renamed"},
|
||||
)
|
||||
assert renamed.status_code in (404, 405)
|
||||
# Shaped like an id the server issues, which a new key may not be.
|
||||
issued = client.put(
|
||||
f"{PREFIX}/f_k3q7m2xa",
|
||||
headers=superuser_token_headers,
|
||||
json=a_flow("f_k3q7m2xa"),
|
||||
)
|
||||
assert issued.status_code == 400
|
||||
|
||||
client.delete(f"{PREFIX}/identified", headers=superuser_token_headers)
|
||||
|
||||
|
||||
def test_broken_node_is_reported_and_siblings_stay_active(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
|
||||
put_flow(client, superuser_token_headers, a_flow(), "demo")
|
||||
client.put(
|
||||
f"{PREFIX}/demo/nodes/sensor/source",
|
||||
headers=superuser_token_headers,
|
||||
@@ -143,7 +119,7 @@ def test_broken_node_is_reported_and_siblings_stay_active(
|
||||
def test_running_a_flow_produces_values(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
|
||||
put_flow(client, superuser_token_headers, a_flow(), "demo")
|
||||
client.put(
|
||||
f"{PREFIX}/demo/nodes/sensor/source",
|
||||
headers=superuser_token_headers,
|
||||
@@ -165,9 +141,7 @@ def test_running_a_flow_produces_values(
|
||||
def test_editing_does_not_deploy_until_published(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
saved = client.put(
|
||||
f"{PREFIX}/staged", headers=superuser_token_headers, json=a_flow("staged")
|
||||
).json()
|
||||
saved = put_flow(client, superuser_token_headers, a_flow("staged"), "staged").json()
|
||||
assert saved["has_draft"] is True
|
||||
# Nothing is running yet, so the engine knows no nodes of this flow.
|
||||
assert (
|
||||
@@ -197,8 +171,8 @@ def test_editing_does_not_deploy_until_published(
|
||||
def test_a_stale_save_is_refused(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
saved = client.put(
|
||||
f"{PREFIX}/contested", headers=superuser_token_headers, json=a_flow("contested")
|
||||
saved = put_flow(
|
||||
client, superuser_token_headers, a_flow("contested"), "contested"
|
||||
).json()
|
||||
stale = saved["definition"]
|
||||
|
||||
@@ -222,8 +196,8 @@ def test_a_stale_save_is_refused(
|
||||
def test_discarding_a_draft_restores_what_is_running(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
saved = client.put(
|
||||
f"{PREFIX}/reverted", headers=superuser_token_headers, json=a_flow("reverted")
|
||||
saved = put_flow(
|
||||
client, superuser_token_headers, a_flow("reverted"), "reverted"
|
||||
).json()
|
||||
client.post(
|
||||
f"{PREFIX}/reverted/publish",
|
||||
@@ -232,10 +206,11 @@ def test_discarding_a_draft_restores_what_is_running(
|
||||
)
|
||||
published = client.get(f"{PREFIX}/reverted", headers=superuser_token_headers).json()
|
||||
|
||||
client.put(
|
||||
f"{PREFIX}/reverted",
|
||||
headers=superuser_token_headers,
|
||||
json={**published["definition"], "title": "Scratch that"},
|
||||
put_flow(
|
||||
client,
|
||||
superuser_token_headers,
|
||||
{**published["definition"], "title": "Scratch that"},
|
||||
"reverted",
|
||||
)
|
||||
|
||||
response = client.post(
|
||||
@@ -251,9 +226,7 @@ def test_discarding_a_draft_restores_what_is_running(
|
||||
def test_stopping_a_flow_takes_it_off_the_engine(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
saved = client.put(
|
||||
f"{PREFIX}/halted", headers=superuser_token_headers, json=a_flow("halted")
|
||||
).json()
|
||||
saved = put_flow(client, superuser_token_headers, a_flow("halted"), "halted").json()
|
||||
client.post(
|
||||
f"{PREFIX}/halted/publish",
|
||||
headers=superuser_token_headers,
|
||||
@@ -286,9 +259,7 @@ def test_stopping_a_flow_takes_it_off_the_engine(
|
||||
def test_pausing_is_reported_back(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
saved = client.put(
|
||||
f"{PREFIX}/held", headers=superuser_token_headers, json=a_flow("held")
|
||||
).json()
|
||||
saved = put_flow(client, superuser_token_headers, a_flow("held"), "held").json()
|
||||
client.post(
|
||||
f"{PREFIX}/held/publish",
|
||||
headers=superuser_token_headers,
|
||||
@@ -318,7 +289,7 @@ def test_unconnected_input_is_surfaced(
|
||||
) -> None:
|
||||
flow = a_flow()
|
||||
flow["nodes"][0]["provides"] = [] # nothing produces "reading" any more
|
||||
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=flow)
|
||||
put_flow(client, superuser_token_headers, flow, "demo")
|
||||
|
||||
issues = client.post(
|
||||
f"{PREFIX}/demo/validate", headers=superuser_token_headers
|
||||
@@ -329,7 +300,7 @@ def test_unconnected_input_is_surfaced(
|
||||
def test_delete_flow(
|
||||
client: TestClient, superuser_token_headers: dict[str, str], db: Session
|
||||
) -> None:
|
||||
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
|
||||
put_flow(client, superuser_token_headers, a_flow(), "demo")
|
||||
# Finished: a flow with a live run refuses to be deleted, which is its own
|
||||
# test below.
|
||||
db.add(Run(id="run-1", flow="demo", status="ok", created_at=datetime.now(UTC)))
|
||||
@@ -362,7 +333,7 @@ def test_delete_flow(
|
||||
def test_delete_flow_is_refused_while_a_run_is_live(
|
||||
client: TestClient, superuser_token_headers: dict[str, str], db: Session
|
||||
) -> None:
|
||||
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
|
||||
put_flow(client, superuser_token_headers, a_flow(), "demo")
|
||||
db.add(
|
||||
Run(id="run-live", flow="demo", status="running", created_at=datetime.now(UTC))
|
||||
)
|
||||
@@ -390,8 +361,8 @@ def test_a_node_that_raises_answers_with_its_error(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
"""A manual run fails the way a scheduled one does, not as a 500."""
|
||||
saved = client.put(
|
||||
f"{PREFIX}/failing", headers=superuser_token_headers, json=a_flow("failing")
|
||||
saved = put_flow(
|
||||
client, superuser_token_headers, a_flow("failing"), "failing"
|
||||
).json()
|
||||
client.put(
|
||||
f"{PREFIX}/failing/nodes/sensor/source",
|
||||
@@ -422,9 +393,7 @@ def test_acknowledging_a_failure_is_announced_on_the_bus(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Another browser holds the same marker; the event is what clears it."""
|
||||
saved = client.put(
|
||||
f"{PREFIX}/acked", headers=superuser_token_headers, json=a_flow("acked")
|
||||
).json()
|
||||
saved = put_flow(client, superuser_token_headers, a_flow("acked"), "acked").json()
|
||||
client.post(
|
||||
f"{PREFIX}/acked/publish",
|
||||
headers=superuser_token_headers,
|
||||
@@ -505,9 +474,7 @@ async def test_the_live_socket_ends_when_its_client_goes(
|
||||
def test_saving_a_header_moves_the_node_ports_in_the_draft(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
saved = client.put(
|
||||
f"{PREFIX}/header", headers=superuser_token_headers, json=a_flow("header")
|
||||
).json()
|
||||
saved = put_flow(client, superuser_token_headers, a_flow("header"), "header").json()
|
||||
|
||||
response = client.put(
|
||||
f"{PREFIX}/header/nodes/logger/source",
|
||||
@@ -539,9 +506,7 @@ def test_saving_a_header_moves_the_node_ports_in_the_draft(
|
||||
def test_a_shared_node_in_use_is_retired_only_with_detach(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
client.put(
|
||||
f"{PREFIX}/shared", headers=superuser_token_headers, json=a_flow("shared")
|
||||
)
|
||||
put_flow(client, superuser_token_headers, a_flow("shared"), "shared")
|
||||
client.put(
|
||||
f"{PREFIX}/shared/nodes/sensor/source",
|
||||
headers=superuser_token_headers,
|
||||
@@ -571,9 +536,7 @@ def test_a_shared_node_only_the_published_flow_uses_is_kept(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
"""An unshare in the draft leaves the running flow on the source."""
|
||||
client.put(
|
||||
f"{PREFIX}/unshared", headers=superuser_token_headers, json=a_flow("unshared")
|
||||
)
|
||||
put_flow(client, superuser_token_headers, a_flow("unshared"), "unshared")
|
||||
shared = client.post(
|
||||
f"{PREFIX}/unshared/nodes/sensor/share",
|
||||
headers=superuser_token_headers,
|
||||
@@ -603,9 +566,7 @@ def test_a_shared_source_write_is_audited(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""It goes live at once, so it is recorded like a publish."""
|
||||
client.put(
|
||||
f"{PREFIX}/audited", headers=superuser_token_headers, json=a_flow("audited")
|
||||
)
|
||||
put_flow(client, superuser_token_headers, a_flow("audited"), "audited")
|
||||
client.post(
|
||||
f"{PREFIX}/audited/nodes/sensor/share",
|
||||
headers=superuser_token_headers,
|
||||
|
||||
@@ -96,10 +96,10 @@ def test_a_flow_that_cannot_run_makes_the_summary_degraded(
|
||||
|
||||
controller = client.app.state.flow_controller
|
||||
before = controller.issues
|
||||
before_flows = controller.store.list_flows
|
||||
before_flows = controller.store.key_index
|
||||
# The looping flow has to be one the store lists, otherwise "running"
|
||||
# counts nothing either way and the assertion below proves nothing.
|
||||
controller.store.list_flows = lambda: ["looping"]
|
||||
controller.store.key_index = lambda: {"looping": "f_looping"}
|
||||
controller.issues = [
|
||||
ValidationIssue(
|
||||
code="cycle",
|
||||
@@ -119,7 +119,7 @@ def test_a_flow_that_cannot_run_makes_the_summary_degraded(
|
||||
body = client.get(f"{PREFIX}/summary", headers=superuser_token_headers).json()
|
||||
finally:
|
||||
controller.issues = before
|
||||
controller.store.list_flows = before_flows
|
||||
controller.store.key_index = before_flows
|
||||
|
||||
assert body["status"] == "degraded"
|
||||
assert body["flows"]["invalid"] == 1
|
||||
|
||||
@@ -7,6 +7,7 @@ from fastapi.testclient import TestClient
|
||||
|
||||
from fluksio.api.routes.search import MASK, setting_text
|
||||
from fluksio.core.config import settings
|
||||
from tests.utils.flows import put_dashboard, put_flow
|
||||
|
||||
PREFIX = f"{settings.API_V1_STR}/search"
|
||||
FLOWS = f"{settings.API_V1_STR}/flows"
|
||||
@@ -22,19 +23,20 @@ def test_index_reaches_inside_flows_and_dashboards(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
"""A node and a widget are the point: neither is on any list endpoint."""
|
||||
client.put(
|
||||
f"{FLOWS}/searchable",
|
||||
headers=superuser_token_headers,
|
||||
json={
|
||||
put_flow(
|
||||
client,
|
||||
superuser_token_headers,
|
||||
{
|
||||
"name": "searchable",
|
||||
"title": "Searchable",
|
||||
"nodes": [{"id": "sensor", "type": "python", "title": "Hall sensor"}],
|
||||
},
|
||||
"searchable",
|
||||
)
|
||||
client.put(
|
||||
f"{DASHBOARDS}/hall",
|
||||
headers=superuser_token_headers,
|
||||
json={
|
||||
put_dashboard(
|
||||
client,
|
||||
superuser_token_headers,
|
||||
{
|
||||
"name": "hall",
|
||||
"title": "Hall",
|
||||
"widgets": [
|
||||
@@ -48,6 +50,7 @@ def test_index_reaches_inside_flows_and_dashboards(
|
||||
],
|
||||
"version": 0,
|
||||
},
|
||||
"hall",
|
||||
)
|
||||
|
||||
entries = client.get(f"{PREFIX}/", headers=superuser_token_headers).json()
|
||||
@@ -69,15 +72,16 @@ def test_index_lists_a_flows_inputs_and_outputs(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
"""Named by the qualified message, which is the label the canvas draws."""
|
||||
client.put(
|
||||
f"{FLOWS}/fit",
|
||||
headers=superuser_token_headers,
|
||||
json={
|
||||
put_flow(
|
||||
client,
|
||||
superuser_token_headers,
|
||||
{
|
||||
"name": "fit",
|
||||
"mode": "batch",
|
||||
"inputs": [{"spec": {"name": "epochs", "dtype": "int"}}],
|
||||
"outputs": ["accuracy"],
|
||||
},
|
||||
"fit",
|
||||
)
|
||||
|
||||
entries = client.get(f"{PREFIX}/", headers=superuser_token_headers).json()
|
||||
@@ -151,10 +155,10 @@ def test_index_lists_a_nodes_settings(
|
||||
normal_user_token_headers: dict[str, str],
|
||||
) -> None:
|
||||
"""A setting is found by its node and name, with its value beside it."""
|
||||
client.put(
|
||||
f"{FLOWS}/tuned",
|
||||
headers=superuser_token_headers,
|
||||
json={
|
||||
put_flow(
|
||||
client,
|
||||
superuser_token_headers,
|
||||
{
|
||||
"name": "tuned",
|
||||
"nodes": [
|
||||
{"id": "gain", "type": "python", "params": {"factor": 2.5}},
|
||||
@@ -169,6 +173,7 @@ def test_index_lists_a_nodes_settings(
|
||||
},
|
||||
],
|
||||
},
|
||||
"tuned",
|
||||
)
|
||||
|
||||
def settings_of(headers: dict[str, str]) -> tuple[dict[str, dict], str]:
|
||||
|
||||
@@ -38,6 +38,13 @@ def api(client: TestClient, superuser_token_headers: dict[str, str]) -> Client:
|
||||
return Client(http=client, token=superuser_token_headers["Authorization"][7:])
|
||||
|
||||
|
||||
def flow_id(api: Client, key: str) -> str:
|
||||
"""The id sync stored the flow with key *key* under."""
|
||||
found = api.find_flow(key)
|
||||
assert found is not None, key
|
||||
return str(found["id"])
|
||||
|
||||
|
||||
def commits() -> int:
|
||||
"""How many commits the flow store has made."""
|
||||
from fluksio.flow.store import FlowStore
|
||||
@@ -62,14 +69,14 @@ def test_sync_stores_a_runnable_flow_with_its_origin(api, flows):
|
||||
assert [r.flow for r in reports] == ["train"]
|
||||
assert reports[0].created and reports[0].published
|
||||
|
||||
stored = api.get_flow("train")
|
||||
stored = api.get_flow(flow_id(api, "train"))
|
||||
definition = stored["definition"]
|
||||
assert definition["origin"]["commit"] == "a1b2c3d4"
|
||||
assert [n["id"] for n in definition["nodes"]] == ["prepare", "fit", "evaluate"]
|
||||
assert not stored["has_draft"]
|
||||
# The store holds a complete definition, and the bodies say where the real
|
||||
# code is rather than being a copy of it.
|
||||
source = api.get_source("train", "fit")
|
||||
source = api.get_source(definition["id"], "fit")
|
||||
assert source.startswith(MARKER)
|
||||
assert "from myresearch.train import fit" in source
|
||||
|
||||
@@ -79,7 +86,7 @@ def test_a_flow_read_by_another_process_syncs_as_the_flow_does(api, flows):
|
||||
carried = Declared(flows["train"].declaration())
|
||||
|
||||
sync([carried], api, origin=ORIGIN)
|
||||
stored = api.get_flow("train")["definition"]
|
||||
stored = api.get_flow(flow_id(api, "train"))["definition"]
|
||||
assert stored["origin"]["declared_in"] == "myresearch/pipeline.py"
|
||||
|
||||
# Nothing differs from what the flow itself would have stored.
|
||||
@@ -90,7 +97,8 @@ def test_a_node_carries_the_modules_its_code_reaches(api, flows):
|
||||
"""The shim names one function; the cache has to key on what it calls."""
|
||||
sync([flows["train"]], api, origin=ORIGIN)
|
||||
|
||||
nodes = {n["id"]: n for n in api.get_flow("train")["definition"]["nodes"]}
|
||||
definition = api.get_flow(flow_id(api, "train"))["definition"]
|
||||
nodes = {n["id"]: n for n in definition["nodes"]}
|
||||
files = nodes["fit"]["code_files"]
|
||||
|
||||
# The module the function is in, and the package whose `__init__` runs on
|
||||
@@ -156,14 +164,15 @@ def test_a_second_sync_changes_nothing_and_commits_nothing(api, flows):
|
||||
|
||||
def test_sync_refuses_to_overwrite_a_canvas_edit(api, flows):
|
||||
sync([flows["train"]], api, origin=ORIGIN)
|
||||
api.put_source("train", "evaluate", "def process(weights):\n return {}\n")
|
||||
train = flow_id(api, "train")
|
||||
api.put_source(train, "evaluate", "def process(weights):\n return {}\n")
|
||||
|
||||
with pytest.raises(SyncError, match="edited on the canvas"):
|
||||
sync([flows["train"]], api, origin=ORIGIN)
|
||||
|
||||
reports = sync([flows["train"]], api, origin=ORIGIN, force=True)
|
||||
assert "evaluate" in reports[0].changed
|
||||
assert api.get_source("train", "evaluate").startswith(MARKER)
|
||||
assert api.get_source(train, "evaluate").startswith(MARKER)
|
||||
|
||||
|
||||
def test_an_interrupted_sync_finishes_on_the_next_one(api, flows):
|
||||
@@ -177,23 +186,23 @@ def test_an_interrupted_sync_finishes_on_the_next_one(api, flows):
|
||||
target = flows["train"]
|
||||
target.name = "resumed"
|
||||
try:
|
||||
# Exactly what the document PUT leaves behind before any body is written.
|
||||
api.put_flow(target.document(ORIGIN) | {"version": 1})
|
||||
assert api.get_source_entry("resumed", "fit")["missing"]
|
||||
# Exactly what creating the document leaves before any body is written.
|
||||
resumed = api.create_flow(target.document(ORIGIN))["definition"]["id"]
|
||||
assert api.get_source_entry(resumed, "fit")["missing"]
|
||||
|
||||
reports = sync([target], api, origin=ORIGIN)
|
||||
|
||||
assert "fit" in reports[0].changed
|
||||
assert api.get_source("resumed", "fit").startswith(MARKER)
|
||||
assert not api.get_source_entry("resumed", "fit")["missing"]
|
||||
assert not api.get_flow("resumed")["has_draft"]
|
||||
assert api.get_source(resumed, "fit").startswith(MARKER)
|
||||
assert not api.get_source_entry(resumed, "fit")["missing"]
|
||||
assert not api.get_flow(resumed)["has_draft"]
|
||||
finally:
|
||||
target.name = "train"
|
||||
|
||||
|
||||
def test_sync_refuses_a_flow_it_did_not_create(api, flows):
|
||||
api.put_flow({"name": "drawn", "version": 1, "nodes": [], "mode": "batch"})
|
||||
api.publish("drawn", 1)
|
||||
drawn = api.create_flow({"name": "drawn", "nodes": [], "mode": "batch"})
|
||||
api.publish(drawn["definition"]["id"], 1)
|
||||
theirs = flows["train"]
|
||||
theirs.name = "drawn"
|
||||
|
||||
@@ -207,7 +216,8 @@ def test_sync_refuses_a_flow_it_did_not_create(api, flows):
|
||||
def test_use_stores_the_rewiring_it_was_given(api, flows):
|
||||
sync([flows["finetune"]], api, origin=ORIGIN)
|
||||
|
||||
nodes = {n["id"]: n for n in api.get_flow("finetune")["definition"]["nodes"]}
|
||||
definition = api.get_flow(flow_id(api, "finetune"))["definition"]
|
||||
nodes = {n["id"]: n for n in definition["nodes"]}
|
||||
assert [p["name"] for p in nodes["fit"]["requires"]] == ["augmented", "lr"]
|
||||
assert nodes["fit"]["params"] == {"epochs": 3}
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from fluksio.core.config import settings
|
||||
from fluksio.core.db import engine as db_engine
|
||||
from fluksio.models import Run, RunArtifact
|
||||
from tests.flow.conftest import FakeWren
|
||||
from tests.utils.flows import put_flow
|
||||
|
||||
FLOW = "wren_house"
|
||||
PREFIX = f"{settings.API_V1_STR}/flows/{FLOW}/nodes"
|
||||
@@ -70,10 +71,8 @@ def house(
|
||||
{"id": "sensor", "type": "python"},
|
||||
]
|
||||
url = f"{settings.API_V1_STR}/flows/{FLOW}"
|
||||
saved = client.put(
|
||||
url, headers=superuser_token_headers, json={"name": FLOW, "nodes": nodes}
|
||||
)
|
||||
assert saved.status_code == 200, saved.text
|
||||
saved = put_flow(client, superuser_token_headers, {"name": FLOW, "nodes": nodes})
|
||||
assert saved.is_success, saved.text
|
||||
yield
|
||||
client.delete(url, headers=superuser_token_headers)
|
||||
|
||||
|
||||
@@ -19,7 +19,8 @@ from fluksio.flow.nodes import Node
|
||||
from fluksio.flow.pipeline import Pipeline
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||
from fluksio.flow.state import MemoryState
|
||||
from fluksio.flow.store import FlowStore, StaleVersion
|
||||
from fluksio.flow.store import ISSUED_ID, FlowStore, StaleVersion
|
||||
from tests.utils.stores import created
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -93,25 +94,25 @@ def test_discarding_leaves_what_is_published(store: DashboardStore):
|
||||
|
||||
def test_a_new_dashboard_is_a_draft_until_it_is_published(store: DashboardStore):
|
||||
"""Creating one does not put it on a wall; publishing is what does."""
|
||||
created = store.write_draft(default_dashboard("house"), 0)
|
||||
made = created(store, default_dashboard("house"))
|
||||
|
||||
assert not store.is_published("house")
|
||||
with pytest.raises(DashboardNotFound):
|
||||
store.read("house")
|
||||
assert store.read("house", draft=True).title == created.title
|
||||
assert store.read("house", draft=True).title == made.title
|
||||
# Listed all the same, so the editor can find what it just made.
|
||||
assert [(d.name, d.has_draft, d.version) for d in store.list()] == [
|
||||
("house", True, created.version)
|
||||
("house", True, made.version)
|
||||
]
|
||||
|
||||
store.publish("house", created.version)
|
||||
store.publish("house", made.version)
|
||||
|
||||
assert store.is_published("house")
|
||||
assert store.read("house").title == created.title
|
||||
assert store.read("house").title == made.title
|
||||
|
||||
|
||||
def test_an_unpublished_dashboard_can_be_deleted(store: DashboardStore):
|
||||
store.write_draft(default_dashboard("house"), 0)
|
||||
created(store, default_dashboard("house"))
|
||||
|
||||
store.delete("house")
|
||||
assert not store.exists("house")
|
||||
@@ -128,7 +129,8 @@ def test_a_duplicate_is_an_unpublished_copy_of_the_working_copy(
|
||||
copy = store.duplicate("house")
|
||||
|
||||
assert (copy.name, copy.title, copy.version) == ("house_copy", "Edited (copy)", 1)
|
||||
assert store.has_draft("house_copy") and not store.is_published("house_copy")
|
||||
assert ISSUED_ID.fullmatch(copy.id)
|
||||
assert store.has_draft(copy.id) and not store.is_published(copy.id)
|
||||
assert store.read("house", draft=True).version == edited.version
|
||||
assert store.duplicate("house").name == "house_copy_2"
|
||||
|
||||
|
||||
@@ -8,7 +8,8 @@ import pytest
|
||||
from fluksio.flow.controller import FlowController
|
||||
from fluksio.flow.messages import MessageSpec
|
||||
from fluksio.flow.schemas import FlowDef, FlowOrigin, NodeDef
|
||||
from fluksio.flow.store import FlowStore, StaleVersion
|
||||
from fluksio.flow.store import ISSUED_ID, FlowStore, StaleVersion
|
||||
from tests.utils.stores import created
|
||||
|
||||
SOURCE = "def process():\n return {}\n"
|
||||
EDITED = "def process():\n return {'temp': 1}\n"
|
||||
@@ -28,7 +29,7 @@ def a_flow(title: str = "") -> FlowDef:
|
||||
|
||||
|
||||
def test_a_draft_is_invisible_to_the_engine(store: FlowStore):
|
||||
store.write_draft(a_flow(), 0)
|
||||
created(store, a_flow())
|
||||
|
||||
assert store.has_draft("heating")
|
||||
assert store.list_flows() == ["heating"] # the editor sees it
|
||||
@@ -36,7 +37,7 @@ def test_a_draft_is_invisible_to_the_engine(store: FlowStore):
|
||||
|
||||
|
||||
def test_the_version_counter_advances_per_save(store: FlowStore):
|
||||
first = store.write_draft(a_flow(), 0)
|
||||
first = created(store, a_flow())
|
||||
assert first.version == 1
|
||||
|
||||
second = store.write_draft(a_flow(title="Warm"), first.version)
|
||||
@@ -44,14 +45,14 @@ def test_the_version_counter_advances_per_save(store: FlowStore):
|
||||
|
||||
|
||||
def test_resaving_identical_content_neither_bumps_nor_commits(store: FlowStore):
|
||||
stored = store.write_draft(a_flow(), 0)
|
||||
stored = created(store, a_flow())
|
||||
|
||||
again = store.write_draft(a_flow(), stored.version)
|
||||
assert again.version == stored.version
|
||||
|
||||
|
||||
def test_a_stale_write_is_refused(store: FlowStore):
|
||||
store.write_draft(a_flow(), 0)
|
||||
created(store, a_flow())
|
||||
store.write_draft(a_flow(title="Mine"), 1)
|
||||
|
||||
# A second client still holding version 1 must not overwrite version 2.
|
||||
@@ -61,7 +62,7 @@ def test_a_stale_write_is_refused(store: FlowStore):
|
||||
|
||||
|
||||
def test_publishing_promotes_the_draft_and_its_sources(store: FlowStore):
|
||||
stored = store.write_draft(a_flow(title="Warm"), 0)
|
||||
stored = created(store, a_flow(title="Warm"))
|
||||
store.write_node_source("heating", "sensor", EDITED, draft=True)
|
||||
|
||||
published = store.publish_flow("heating", stored.version)
|
||||
@@ -74,7 +75,7 @@ def test_publishing_promotes_the_draft_and_its_sources(store: FlowStore):
|
||||
|
||||
|
||||
def test_publishing_a_stale_draft_is_refused(store: FlowStore):
|
||||
store.write_draft(a_flow(), 0)
|
||||
created(store, a_flow())
|
||||
|
||||
with pytest.raises(StaleVersion):
|
||||
store.publish_flow("heating", 99)
|
||||
@@ -150,7 +151,7 @@ def test_a_worker_on_a_node_without_code_is_reported(store: FlowStore):
|
||||
|
||||
def test_a_node_being_written_in_the_editor_is_not_reported(store: FlowStore):
|
||||
"""A draft legitimately has no published body yet — that is what a draft is."""
|
||||
store.write_draft(a_flow(), 0)
|
||||
created(store, a_flow())
|
||||
controller = FlowController(store)
|
||||
|
||||
assert controller.preview("heating").issues == []
|
||||
@@ -175,12 +176,14 @@ def test_a_duplicate_is_a_local_draft_of_what_the_editor_shows(store: FlowStore)
|
||||
copy = store.duplicate_flow("heating")
|
||||
|
||||
assert (copy.name, copy.title, copy.version) == ("heating_copy", "Warm (copy)", 1)
|
||||
# An id of its own, which its key is not.
|
||||
assert ISSUED_ID.fullmatch(copy.id)
|
||||
assert copy.origin is None
|
||||
assert copy.nodes[1].requires[0].name == "heating_copy.temp"
|
||||
assert store.has_draft("heating_copy") and not store.is_published("heating_copy")
|
||||
assert store.read_node_source("heating_copy", "sensor", draft=True) == EDITED
|
||||
assert not (store.root / "heating_copy" / "runtime.json").exists()
|
||||
assert store.has_draft(copy.id) and not store.is_published(copy.id)
|
||||
assert store.read_node_source(copy.id, "sensor", draft=True) == EDITED
|
||||
assert not (store.root / copy.id / "runtime.json").exists()
|
||||
assert store.duplicate_flow("heating").name == "heating_copy_2"
|
||||
|
||||
store.publish_flow("heating_copy", copy.version)
|
||||
assert store.read_node_source("heating_copy", "sensor") == EDITED
|
||||
store.publish_flow(copy.id, copy.version)
|
||||
assert store.read_node_source(copy.id, "sensor") == EDITED
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
"""The engine runs a flow by its key and finds it by its id.
|
||||
|
||||
Every test runs twice, once with the id equal to the key the way a flow from
|
||||
before ids has it, and once with them apart. A mix-up of the two is invisible
|
||||
in the first run, which is why the second one exists.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from fluksio.flow.controller import HOOK_PREFIX, FlowController
|
||||
from fluksio.flow.messages import DType, MessageSpec
|
||||
from fluksio.flow.queue import MemoryWorkQueue, WorkItem
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||
from fluksio.flow.store import FlowStore
|
||||
|
||||
SENSOR = "def process():\n return {'reading': 21.5}\n"
|
||||
DOUBLE = "def process(reading):\n return {'doubled': reading * 2}\n"
|
||||
|
||||
|
||||
@pytest.fixture(params=["house", "x_house"], ids=["id=key", "id!=key"])
|
||||
def flow_id(request: pytest.FixtureRequest) -> str:
|
||||
return str(request.param)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path, flow_id: str) -> FlowStore:
|
||||
store = FlowStore(tmp_path / "flows")
|
||||
store.write_flow(
|
||||
FlowDef(
|
||||
id=flow_id,
|
||||
name="house",
|
||||
nodes=[
|
||||
NodeDef(
|
||||
id="sensor",
|
||||
provides=[MessageSpec(name="reading", dtype=DType.FLOAT)],
|
||||
),
|
||||
NodeDef(
|
||||
id="double",
|
||||
requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
|
||||
provides=[MessageSpec(name="doubled", dtype=DType.FLOAT)],
|
||||
),
|
||||
NodeDef(
|
||||
id="hook",
|
||||
type="http",
|
||||
params={"url": "/tick", "method": "POST"},
|
||||
provides=[MessageSpec(name="tick", dtype=DType.FLOAT)],
|
||||
),
|
||||
],
|
||||
)
|
||||
)
|
||||
store.write_node_source(flow_id, "sensor", SENSOR)
|
||||
store.write_node_source(flow_id, "double", DOUBLE)
|
||||
return store
|
||||
|
||||
|
||||
def test_a_flow_runs_under_its_key_from_the_code_its_id_holds(
|
||||
store: FlowStore, flow_id: str
|
||||
):
|
||||
engine = FlowController(store)
|
||||
asyncio.run(engine.reload())
|
||||
|
||||
entry = engine.loaded["house.double"]
|
||||
assert (entry.flow, entry.flow_id) == ("house", flow_id)
|
||||
# Read by the id: by the key it would be the template, and missing.
|
||||
assert not entry.missing_source and entry.status.value == "active"
|
||||
assert engine.loaded["house.hook"].node.url == f"{HOOK_PREFIX}/{flow_id}/tick"
|
||||
|
||||
engine.run_flow("house")
|
||||
|
||||
assert engine.values()["house.doubled"]["value"] == 43.0
|
||||
assert {s.id for s in engine.node_statuses("house")} >= {"house.double"}
|
||||
|
||||
|
||||
def test_stopping_writes_under_the_id_and_gates_the_key(store: FlowStore, flow_id: str):
|
||||
engine = FlowController(store)
|
||||
|
||||
async def scenario() -> None:
|
||||
await engine.reload()
|
||||
await engine.set_enabled(flow_id, "house", False)
|
||||
assert engine.disabled == {"house"}
|
||||
assert not store.read_enabled(flow_id)
|
||||
# Read back by id when everything is built again.
|
||||
await engine.reload()
|
||||
assert engine.disabled == {"house"}
|
||||
|
||||
asyncio.run(scenario())
|
||||
assert (store.root / flow_id / "runtime.json").exists()
|
||||
|
||||
|
||||
def test_a_flow_rebuilt_by_id_is_taken_out_by_its_key_once_gone(
|
||||
store: FlowStore, flow_id: str
|
||||
):
|
||||
engine = FlowController(store)
|
||||
|
||||
async def scenario() -> None:
|
||||
await engine.reload()
|
||||
assert "house.sensor" in engine.loaded
|
||||
await engine.reload_flow(flow_id)
|
||||
assert "house.sensor" in engine.loaded
|
||||
|
||||
store.delete_flow(flow_id)
|
||||
await engine.reload_flow(flow_id)
|
||||
|
||||
asyncio.run(scenario())
|
||||
assert not [node for node in engine.loaded if node.startswith("house.")]
|
||||
assert engine.pipeline is not None and not engine.pipeline.flow_nodes("house")
|
||||
|
||||
|
||||
def test_a_draft_previews_by_id_and_reports_by_key(store: FlowStore, flow_id: str):
|
||||
draft = store.read_flow(flow_id).model_copy(update={"title": "Edited"})
|
||||
store.write_draft(draft, draft.version)
|
||||
engine = FlowController(store)
|
||||
|
||||
preview = engine.preview(flow_id)
|
||||
|
||||
assert {node.id for node in preview.nodes} == {
|
||||
"house.sensor",
|
||||
"house.double",
|
||||
"house.hook",
|
||||
}
|
||||
assert all(node.status == "active" for node in preview.nodes)
|
||||
engine.run_preview(flow_id)
|
||||
assert engine.state.get("house.doubled") == 43.0
|
||||
|
||||
|
||||
def test_what_a_key_leaves_behind_is_found_and_forgotten(store: FlowStore):
|
||||
engine = FlowController(store)
|
||||
queue = MemoryWorkQueue()
|
||||
engine.execution = SimpleNamespace(queue=queue) # type: ignore[assignment]
|
||||
engine.state.set("house.reading", 1.0)
|
||||
engine.state.set("__node__:house.double:last", 2)
|
||||
engine.state.set("housework.reading", 3.0)
|
||||
queue.add_delayed(WorkItem(kind="flush", node="house.double", flow="house"), 9e9)
|
||||
queue.park("house", WorkItem(kind="cascade", node="house.sensor", flow="house"))
|
||||
# Another flow's work, carrying one of this key's messages.
|
||||
queue.add(
|
||||
WorkItem(
|
||||
kind="cascade", node="garden.pump", flow="garden", outputs={"house.x": 1}
|
||||
)
|
||||
)
|
||||
|
||||
found = engine.key_residue("house")
|
||||
|
||||
assert "house.reading" in found and "__node__:house.double:last" in found
|
||||
assert "housework.reading" not in found
|
||||
assert sum("queued work" in what for what in found) == 3
|
||||
assert engine.key_residue("garden") == ["queued work of 'garden.pump'"]
|
||||
|
||||
engine.forget_flow("house")
|
||||
|
||||
# What the other flow queued is its own, and runs as it would have.
|
||||
assert engine.key_residue("house") == ["queued work of 'garden.pump'"]
|
||||
assert engine.state.get("housework.reading") == 3.0
|
||||
@@ -8,6 +8,7 @@ is why the migration is driven through crashes, refusals and stores without git.
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -18,17 +19,23 @@ from fluksio.flow.dashboards import (
|
||||
DASHBOARD_DIR,
|
||||
DashboardDef,
|
||||
DashboardExists,
|
||||
DashboardNotFound,
|
||||
DashboardStore,
|
||||
default_dashboard,
|
||||
results_dashboard,
|
||||
)
|
||||
from fluksio.flow.identity import COMMIT_MESSAGE, IdsRefused, assign_ids
|
||||
from fluksio.flow.schemas import FlowDef
|
||||
from fluksio.flow.schemas import FlowCreate, FlowDef
|
||||
from fluksio.flow.store import (
|
||||
ISSUED_ID,
|
||||
FlowExists,
|
||||
FlowNotFound,
|
||||
FlowStore,
|
||||
IdentityChange,
|
||||
IdTaken,
|
||||
RecordedKey,
|
||||
ReservedKey,
|
||||
StaleVersion,
|
||||
new_id,
|
||||
)
|
||||
|
||||
@@ -37,23 +44,51 @@ from fluksio.flow.store import (
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Graves:
|
||||
"""Tombstones and history, as `fluksio.flow.tombstones.Tombstones` keeps
|
||||
them in the database, which store tests have none of."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.stones: dict[str, str] = {}
|
||||
#: Ids that history rows name.
|
||||
self.history: set[str] = set()
|
||||
|
||||
def bury(self, flow_id: str, name: str) -> None:
|
||||
self.stones[flow_id] = name
|
||||
|
||||
def lift(self, flow_id: str) -> None:
|
||||
self.stones.pop(flow_id, None)
|
||||
|
||||
def recorded(self, flow_id: str) -> str | None:
|
||||
if flow_id in self.stones:
|
||||
return self.stones[flow_id]
|
||||
return flow_id if flow_id in self.history else None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> FlowStore:
|
||||
return FlowStore(tmp_path / "flows")
|
||||
def graves() -> Graves:
|
||||
return Graves()
|
||||
|
||||
|
||||
def test_a_flow_is_given_its_key_as_id_and_keeps_both(store: FlowStore):
|
||||
created = store.write_draft(FlowDef(name="heating"), 0)
|
||||
assert (created.id, created.name) == ("heating", "heating")
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path, graves: Graves) -> FlowStore:
|
||||
return FlowStore(tmp_path / "flows", tombstones=graves) # type: ignore[arg-type]
|
||||
|
||||
published = store.publish_flow("heating", created.version)
|
||||
|
||||
def test_a_new_flow_gets_an_issued_id_and_keeps_it(store: FlowStore):
|
||||
created = store.create(FlowDef(name="heating"))
|
||||
flow_id = created.id
|
||||
assert ISSUED_ID.fullmatch(flow_id) and flow_id.startswith("f_")
|
||||
assert created.name == "heating" and created.version == 1
|
||||
assert store.key_index() == {"heating": flow_id}
|
||||
|
||||
published = store.publish_flow(flow_id, created.version)
|
||||
store.write_draft(published.model_copy(update={"title": "Warm"}), 1)
|
||||
discarded = store.discard_draft("heating")
|
||||
discarded = store.discard_draft(flow_id)
|
||||
|
||||
assert json.loads((store.root / "heating" / "flow.json").read_text())["id"] == (
|
||||
"heating"
|
||||
)
|
||||
assert (discarded.id, discarded.name) == ("heating", "heating")
|
||||
on_disk = json.loads((store.root / flow_id / "flow.json").read_text())
|
||||
assert (on_disk["id"], on_disk["name"]) == (flow_id, "heating")
|
||||
assert (discarded.id, discarded.name) == (flow_id, "heating")
|
||||
|
||||
|
||||
def test_a_document_from_before_ids_reads_with_its_directory_as_id(store: FlowStore):
|
||||
@@ -70,9 +105,13 @@ def test_a_document_from_before_ids_reads_with_its_directory_as_id(store: FlowSt
|
||||
assert store.write_draft(FlowDef(name="heating", title="Warm"), 1).version == 2
|
||||
|
||||
|
||||
def test_a_save_cannot_change_an_id_or_a_key(store: FlowStore):
|
||||
store.write_flow(FlowDef(id="x_heating", name="heating"))
|
||||
def test_a_save_neither_creates_nor_changes_an_id_or_a_key(store: FlowStore):
|
||||
with pytest.raises(FlowNotFound):
|
||||
store.write_draft(FlowDef(name="nobody"), 0)
|
||||
with pytest.raises(DashboardNotFound):
|
||||
DashboardStore(store).write_draft(default_dashboard("nobody"), 0)
|
||||
|
||||
store.write_flow(FlowDef(id="x_heating", name="heating"))
|
||||
with pytest.raises(IdentityChange):
|
||||
store.write_draft(FlowDef(id="x_heating", name="warmth"), 1)
|
||||
with pytest.raises(IdentityChange):
|
||||
@@ -87,56 +126,218 @@ def test_a_save_cannot_change_an_id_or_a_key(store: FlowStore):
|
||||
|
||||
def test_a_key_belongs_to_one_flow_drafts_included(store: FlowStore):
|
||||
"""A draft nobody published holds its key, and a delete lets it go."""
|
||||
store.write_draft(FlowDef(id="x_heating", name="heating"), 0)
|
||||
assert store.key_index() == {"heating": "x_heating"}
|
||||
first = store.create(FlowDef(name="heating"))
|
||||
assert store.key_index() == {"heating": first.id}
|
||||
|
||||
with pytest.raises(FlowExists):
|
||||
store.write_draft(FlowDef(name="heating"), 0)
|
||||
store.create(FlowDef(name="heating"))
|
||||
with pytest.raises(FlowExists):
|
||||
store.create(FlowDef(name="heating"), "x_heating")
|
||||
with pytest.raises(FlowExists):
|
||||
store.write_flow(FlowDef(id="y_heating", name="heating"))
|
||||
|
||||
store.delete_flow("x_heating")
|
||||
assert store.write_draft(FlowDef(name="heating"), 0).id == "heating"
|
||||
assert store.key_index() == {"heating": "heating"}
|
||||
store.delete_flow(first.id)
|
||||
second = store.create(FlowDef(name="heating"))
|
||||
assert second.id != first.id
|
||||
assert store.key_index() == {"heating": second.id}
|
||||
|
||||
|
||||
def test_a_key_belongs_to_one_dashboard_drafts_included(store: FlowStore):
|
||||
dashboards = DashboardStore(store)
|
||||
dashboards.write_draft(DashboardDef(id="x_home", name="home"), 0)
|
||||
first = dashboards.create(default_dashboard("home"))
|
||||
assert ISSUED_ID.fullmatch(first.id) and first.id.startswith("d_")
|
||||
# Its own id space: a flow may share the key.
|
||||
store.write_draft(FlowDef(name="home"), 0)
|
||||
store.create(FlowDef(name="home"))
|
||||
|
||||
with pytest.raises(DashboardExists):
|
||||
dashboards.write_draft(default_dashboard("home"), 0)
|
||||
dashboards.create(default_dashboard("home"))
|
||||
with pytest.raises(DashboardExists):
|
||||
dashboards.write(DashboardDef(id="y_home", name="home"))
|
||||
|
||||
dashboards.delete("x_home")
|
||||
assert dashboards.write(default_dashboard("home")).id == "home"
|
||||
dashboards.delete(first.id)
|
||||
assert dashboards.create(default_dashboard("home"), "home").id == "home"
|
||||
|
||||
|
||||
def test_a_copy_takes_a_key_nobody_holds(store: FlowStore):
|
||||
def test_a_pin_is_an_id_nothing_holds(store: FlowStore):
|
||||
store.create(FlowDef(name="heating"), "x_heating")
|
||||
with pytest.raises(IdTaken):
|
||||
store.create(FlowDef(name="cooling"), "x_heating")
|
||||
# Another flow's key would make naming either ambiguous.
|
||||
with pytest.raises(IdTaken):
|
||||
store.create(FlowDef(name="cooling"), "heating")
|
||||
# And a directory left behind would hand the new flow what is in it.
|
||||
(store.root / "leftover" / "nodes").mkdir(parents=True)
|
||||
with pytest.raises(IdTaken):
|
||||
store.create(FlowDef(name="leftover"), "leftover")
|
||||
dashboards = DashboardStore(store)
|
||||
dashboards.create(default_dashboard("home"), "x_home")
|
||||
with pytest.raises(IdTaken):
|
||||
dashboards.create(default_dashboard("house"), "x_home")
|
||||
|
||||
|
||||
def test_an_issued_id_avoids_whatever_history_or_a_directory_holds(
|
||||
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
candidates = iter(["f_aaaaaaaa", "f_bbbbbbbb", "f_cccccccc", "f_dddddddd"])
|
||||
monkeypatch.setattr(
|
||||
"fluksio.flow.store.new_id",
|
||||
lambda kind, taken: next(c for c in candidates if not taken(c)),
|
||||
)
|
||||
graves.bury("f_aaaaaaaa", "gone")
|
||||
graves.history.add("f_bbbbbbbb")
|
||||
(store.root / "f_cccccccc").mkdir()
|
||||
|
||||
assert store.create(FlowDef(name="heating")).id == "f_dddddddd"
|
||||
|
||||
|
||||
def test_a_deleted_flow_is_buried_before_its_files_go(
|
||||
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
flow_id = store.create(FlowDef(name="heating")).id
|
||||
|
||||
def crash(path: Path) -> None:
|
||||
raise OSError("disk gone")
|
||||
|
||||
monkeypatch.setattr("fluksio.flow.store.shutil.rmtree", crash)
|
||||
with pytest.raises(OSError):
|
||||
store.delete_flow(flow_id)
|
||||
# The name is safe, and the live document still wins over it.
|
||||
assert graves.stones == {flow_id: "heating"}
|
||||
assert store.read_flow(flow_id, draft=True).name == "heating"
|
||||
|
||||
monkeypatch.undo()
|
||||
assert store.delete_flow(flow_id) == "heating"
|
||||
assert graves.stones == {flow_id: "heating"}
|
||||
assert not store.exists(flow_id)
|
||||
|
||||
|
||||
def test_a_delete_that_stops_after_the_files_has_kept_the_name(
|
||||
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
flow_id = store.create(FlowDef(name="heating")).id
|
||||
|
||||
def killed(message: str) -> None:
|
||||
raise OSError("killed before the commit")
|
||||
|
||||
monkeypatch.setattr(store, "_commit", killed)
|
||||
with pytest.raises(OSError):
|
||||
store.delete_flow(flow_id)
|
||||
monkeypatch.undo()
|
||||
|
||||
assert not store.exists(flow_id)
|
||||
assert graves.stones == {flow_id: "heating"}
|
||||
with pytest.raises(FlowNotFound):
|
||||
store.delete_flow(flow_id)
|
||||
|
||||
|
||||
def test_a_pinned_restore_takes_up_its_history_under_its_own_key(
|
||||
store: FlowStore, graves: Graves
|
||||
):
|
||||
flow_id = store.create(FlowDef(name="heating")).id
|
||||
store.delete_flow(flow_id)
|
||||
|
||||
with pytest.raises(RecordedKey):
|
||||
store.create(FlowDef(name="warmth"), flow_id)
|
||||
assert graves.stones == {flow_id: "heating"}
|
||||
|
||||
restored = store.create(FlowDef(name="heating"), flow_id)
|
||||
assert restored.id == flow_id and graves.stones == {}
|
||||
|
||||
# History from before tombstones is filed under an id that was its key.
|
||||
graves.history.add("legacy")
|
||||
with pytest.raises(RecordedKey):
|
||||
store.create(FlowDef(name="other"), "legacy")
|
||||
assert store.create(FlowDef(name="legacy"), "legacy").id == "legacy"
|
||||
|
||||
|
||||
def test_a_key_reused_after_a_delete_starts_a_new_history(
|
||||
store: FlowStore, graves: Graves
|
||||
):
|
||||
old = store.create(FlowDef(name="heating")).id
|
||||
store.delete_flow(old)
|
||||
|
||||
new = store.create(FlowDef(name="heating")).id
|
||||
|
||||
assert new != old
|
||||
assert graves.stones == {old: "heating"}
|
||||
|
||||
|
||||
def test_two_creates_of_one_key_make_one_flow(store: FlowStore):
|
||||
def create(_: int) -> str:
|
||||
try:
|
||||
return store.create(FlowDef(name="heating")).id
|
||||
except FlowExists:
|
||||
return ""
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
made = [flow_id for flow_id in pool.map(create, range(8)) if flow_id]
|
||||
|
||||
assert len(made) == 1
|
||||
assert store.list_flows() == made
|
||||
|
||||
|
||||
def test_two_results_dashboards_of_one_flow_make_one(store: FlowStore):
|
||||
dashboards = DashboardStore(store)
|
||||
flow = FlowDef(id="x_lab", name="lab", mode="batch")
|
||||
|
||||
def generate(_: int) -> str:
|
||||
try:
|
||||
return dashboards.create(results_dashboard(flow), "", True).id
|
||||
except DashboardExists:
|
||||
return ""
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
made = [board for board in pool.map(generate, range(4)) if board]
|
||||
|
||||
assert len(made) == 1
|
||||
assert dashboards.read(made[0]).name == "lab_results"
|
||||
|
||||
|
||||
def test_a_discard_leaves_no_old_draft_version_to_match(store: FlowStore):
|
||||
flow_id = store.create(FlowDef(name="heating")).id
|
||||
published = store.publish_flow(flow_id, 1)
|
||||
held = store.write_draft(published.model_copy(update={"title": "A"}), 1)
|
||||
store.write_draft(held.model_copy(update={"title": "B"}), held.version)
|
||||
|
||||
restored = store.discard_draft(flow_id)
|
||||
|
||||
# Above every draft version handed out, so an editor holding one is stale.
|
||||
assert restored.version == 4 and restored.name == "heating"
|
||||
with pytest.raises(StaleVersion):
|
||||
store.write_draft(held.model_copy(update={"title": "late"}), held.version)
|
||||
assert store.write_draft(restored.model_copy(update={"title": "C"}), 4).version == 5
|
||||
|
||||
dashboards = DashboardStore(store)
|
||||
board = dashboards.create(default_dashboard("home"))
|
||||
dashboards.publish(board.id, 1)
|
||||
dashboards.write_draft(dashboards.read(board.id).model_copy(), 1)
|
||||
assert dashboards.discard_draft(board.id).version == 3
|
||||
# A direct write goes above a draft too.
|
||||
dashboards.write_draft(dashboards.read(board.id).model_copy(), 3)
|
||||
assert dashboards.write(dashboards.read(board.id)).version == 5
|
||||
|
||||
|
||||
def test_a_copy_gets_an_id_of_its_own_and_a_key_nobody_holds(store: FlowStore):
|
||||
store.write_flow(FlowDef(name="heating"))
|
||||
store.write_draft(FlowDef(id="x_other", name="heating_copy"), 0)
|
||||
store.create(FlowDef(name="heating_copy"))
|
||||
dashboards = DashboardStore(store)
|
||||
dashboards.write(default_dashboard("home"))
|
||||
dashboards.write_draft(DashboardDef(id="x_other", name="home_copy"), 0)
|
||||
dashboards.create(default_dashboard("home_copy"))
|
||||
|
||||
copy = store.duplicate_flow("heating")
|
||||
copy = store.duplicate_flow("heating", busy=lambda key: key == "heating_copy_2")
|
||||
board = dashboards.duplicate("home")
|
||||
|
||||
assert (copy.id, copy.name) == ("heating_copy_2", "heating_copy_2")
|
||||
assert (board.id, board.name) == ("home_copy_2", "home_copy_2")
|
||||
# `_copy_2` still had work behind it, so the copy went past it.
|
||||
assert copy.name == "heating_copy_3" and ISSUED_ID.fullmatch(copy.id)
|
||||
assert board.name == "home_copy_2" and ISSUED_ID.fullmatch(board.id)
|
||||
|
||||
|
||||
def test_a_new_key_shaped_like_an_issued_id_is_refused(store: FlowStore):
|
||||
with pytest.raises(ReservedKey):
|
||||
store.write_draft(FlowDef(name="f_k3q7m2xa"), 0)
|
||||
store.create(FlowDef(name="f_k3q7m2xa"))
|
||||
with pytest.raises(ReservedKey):
|
||||
DashboardStore(store).write_draft(default_dashboard("d_k3q7m2xa"), 0)
|
||||
assert store.write_draft(FlowDef(name="f_k3q7m2xa_lab"), 0).name == (
|
||||
"f_k3q7m2xa_lab"
|
||||
)
|
||||
DashboardStore(store).create(default_dashboard("d_k3q7m2xa"))
|
||||
assert store.create(FlowDef(name="f_k3q7m2xa_lab")).name == "f_k3q7m2xa_lab"
|
||||
|
||||
|
||||
def test_an_existing_key_shaped_like_an_issued_id_stays_saveable(store: FlowStore):
|
||||
@@ -515,3 +716,10 @@ def test_ls_lists_ids_keys_and_kinds(tmp_path: Path, capsys):
|
||||
assert rows[0] == ["ID", "KEY", "KIND"]
|
||||
assert ["home", "home", "dashboard"] in rows and ["home", "home", "flow"] in rows
|
||||
assert ["warmth", "radiators", "flow"] in rows
|
||||
|
||||
|
||||
def test_what_a_create_takes_is_a_document_but_for_its_version():
|
||||
"""`FlowCreate` lists the fields again; one added to `FlowDef` belongs there."""
|
||||
assert set(FlowCreate.model_fields) == set(FlowDef.model_fields) - {"version"}
|
||||
with pytest.raises(ValueError):
|
||||
FlowCreate(name="Not A Key")
|
||||
|
||||
@@ -9,6 +9,7 @@ from fluksio.flow.controller import FlowController
|
||||
from fluksio.flow.messages import MessageSpec
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||
from fluksio.flow.store import FlowStore, LibExists, LibNotFound
|
||||
from tests.utils.stores import created
|
||||
|
||||
SOURCE = "def process():\n return {'temp': 1}\n"
|
||||
|
||||
@@ -79,7 +80,7 @@ def test_a_shared_source_change_rebuilds_only_what_is_published(store: FlowStore
|
||||
# Shared in a draft that was never published.
|
||||
cooling = a_flow("cooling")
|
||||
cooling.nodes[0].source_ref = "read_temp"
|
||||
store.write_draft(cooling, 0)
|
||||
created(store, cooling)
|
||||
|
||||
controller = FlowController(store)
|
||||
rebuilt: list[str] = []
|
||||
@@ -131,7 +132,7 @@ def test_retiring_a_shared_source_leaves_each_user_a_copy(store: FlowStore):
|
||||
# Shared in a draft that was never published.
|
||||
cooling = a_flow("cooling")
|
||||
cooling.nodes[0].source_ref = "read_temp"
|
||||
store.write_draft(cooling, 0)
|
||||
created(store, cooling)
|
||||
|
||||
store.detach_lib("read_temp")
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ from fluksio.flow.pipeline import Pipeline
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||
from fluksio.flow.state import MemoryState
|
||||
from fluksio.flow.store import FlowStore
|
||||
from tests.utils.stores import created
|
||||
|
||||
SOURCE = "def process(reading, factor):\n return {'scaled': reading * factor}\n"
|
||||
|
||||
@@ -53,7 +54,7 @@ def test_a_setting_reaches_the_function_as_a_keyword_argument(store: FlowStore):
|
||||
|
||||
|
||||
def test_a_setting_named_after_a_port_is_refused(store: FlowStore):
|
||||
store.write_draft(a_flow(reading=3), 0)
|
||||
created(store, a_flow(reading=3))
|
||||
store.write_node_source("house", "scale", SOURCE, draft=True)
|
||||
controller = FlowController(store)
|
||||
|
||||
@@ -81,7 +82,7 @@ READING = [MessageSpec(name="reading", dtype=DType.FLOAT)]
|
||||
|
||||
|
||||
def _preview(store: FlowStore, flow: FlowDef, source: str | None):
|
||||
store.write_draft(flow, 0)
|
||||
created(store, flow)
|
||||
if source is not None:
|
||||
store.write_node_source("house", "scale", source, draft=True)
|
||||
return FlowController(store).preview("house").nodes[0]
|
||||
|
||||
@@ -680,3 +680,28 @@ def test_done_markers_do_not_grow_without_bound():
|
||||
# The newest are what redelivery would ask about.
|
||||
assert queue.was_done(f"entry-{queue_module.DONE_MARKERS + 499}", "f.node")
|
||||
assert not queue.was_done("entry-0", "f.node")
|
||||
|
||||
|
||||
def test_clearing_a_flow_drops_its_work_and_nothing_else():
|
||||
"""A new flow under the same key must never run what the old one queued."""
|
||||
queue = MemoryWorkQueue()
|
||||
queue.add(WorkItem(kind="cascade", node="gone.a", flow="gone"))
|
||||
# Another flow's work carrying one of its messages, and a run, which is
|
||||
# filed under a flow's id rather than its key: neither is the flow's.
|
||||
queue.add(
|
||||
WorkItem(kind="cascade", node="kept.a", flow="kept", outputs={"gone.x": 1})
|
||||
)
|
||||
queue.add(WorkItem(kind="run", node="", flow="gone", run_id="r1"))
|
||||
queue.add_delayed(WorkItem(kind="flush", node="gone.b", flow="gone"), 9e9)
|
||||
queue.add_delayed(WorkItem(kind="flush", node="kept.b", flow="kept"), 9e9)
|
||||
queue.park("gone", WorkItem(kind="cascade", node="gone.c", flow="gone"))
|
||||
assert len(queue.live_items()) == 6
|
||||
|
||||
queue.clear_flow("gone")
|
||||
|
||||
assert sorted((i.kind, i.node) for i in queue.live_items()) == [
|
||||
("cascade", "kept.a"),
|
||||
("flush", "kept.b"),
|
||||
("run", ""),
|
||||
]
|
||||
assert queue.next_due() == 9e9
|
||||
|
||||
@@ -66,3 +66,30 @@ def test_a_refused_stream_write_keeps_the_timer_for_retry(queue):
|
||||
assert queue.move_due(200.0) == 1
|
||||
assert queue.move_due(200.0) == 0
|
||||
assert [item.node for item in queue.claim(10, 10, due_only=True)] == ["f.stop"]
|
||||
|
||||
|
||||
def test_clearing_a_flow_drops_its_queued_claimed_and_held_work(queue):
|
||||
queue.add(WorkItem(kind="cascade", node="gone.p", flow="gone"))
|
||||
(claimed,) = queue.claim(1, 10)
|
||||
assert claimed.node == "gone.p"
|
||||
queue.add(WorkItem(kind="cascade", node="gone.a", flow="gone"))
|
||||
queue.add(
|
||||
WorkItem(kind="cascade", node="kept.a", flow="kept", outputs={"gone.x": 1})
|
||||
)
|
||||
queue.add(WorkItem(kind="run", node="", flow="gone", run_id="r1"))
|
||||
queue.add_delayed(WorkItem(kind="flush", node="gone.b", flow="gone"), 9e9)
|
||||
queue.add_delayed(WorkItem(kind="flush", node="kept.b", flow="kept"), 9e9)
|
||||
queue.park("gone", WorkItem(kind="cascade", node="gone.c", flow="gone"))
|
||||
assert len(queue.live_items()) == 7
|
||||
|
||||
queue.clear_flow("gone")
|
||||
|
||||
assert sorted((i.kind, i.node) for i in queue.live_items()) == [
|
||||
("cascade", "kept.a"),
|
||||
("flush", "kept.b"),
|
||||
("run", ""),
|
||||
]
|
||||
# Nothing comes back to be run again: not redelivered, not reclaimed.
|
||||
assert queue.reclaim_stale(0) == []
|
||||
queue.ack(claimed)
|
||||
assert [i.node for i in queue.claim(10, 10)] == ["kept.a", ""]
|
||||
|
||||
@@ -318,7 +318,7 @@ def test_stopping_one_flow_does_not_touch_the_others(
|
||||
before = {name: engine.loaded[name].node for name in ("a.io", "b.io")}
|
||||
lifecycle.clear()
|
||||
|
||||
await engine.set_enabled("a", False)
|
||||
await engine.set_enabled("a", "a", False)
|
||||
|
||||
assert lifecycle == ["stop a.io"]
|
||||
# Still built, still the same objects: only the lifecycle changed.
|
||||
@@ -326,14 +326,14 @@ def test_stopping_one_flow_does_not_touch_the_others(
|
||||
assert engine.disabled == {"a"}
|
||||
|
||||
lifecycle.clear()
|
||||
await engine.set_enabled("a", True)
|
||||
await engine.set_enabled("a", "a", True)
|
||||
|
||||
assert lifecycle == ["start a.io"]
|
||||
assert {name: engine.loaded[name].node for name in ("a.io", "b.io")} == before
|
||||
|
||||
# Started while running is a restart: the cancelled loops come back.
|
||||
lifecycle.clear()
|
||||
await engine.set_enabled("a", True)
|
||||
await engine.set_enabled("a", "a", True)
|
||||
|
||||
assert lifecycle == ["stop a.io", "start a.io"]
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ from fluksio.flow.schemas import FlowDef, FlowInput, NodeDef
|
||||
from fluksio.flow.state import MemoryState
|
||||
from fluksio.flow.store import FlowStore
|
||||
from fluksio.models import Run, RunArtifact
|
||||
from tests.utils.stores import created
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -126,9 +127,7 @@ def test_a_sweep_keeps_what_a_flow_input_starts_with(store, tmp_path):
|
||||
flows.write_flow(
|
||||
FlowDef(name="f", inputs=[FlowInput(spec=clip, initial=published)])
|
||||
)
|
||||
flows.write_draft(
|
||||
FlowDef(name="g", inputs=[FlowInput(spec=clip, initial=drafted)]), 0
|
||||
)
|
||||
created(flows, FlowDef(name="g", inputs=[FlowInput(spec=clip, initial=drafted)]))
|
||||
|
||||
assert sweep_artifacts(store, MemoryState(), grace_s=0, flows=flows) == 1
|
||||
assert store.path(orphan["digest"]) is None
|
||||
|
||||
@@ -275,7 +275,9 @@ def test_a_panel_scoped_portal_token_reaches_only_its_panel(
|
||||
assert saved.status_code == 200, saved.text
|
||||
for name in ("kitchen", "bench"):
|
||||
created = client.post(
|
||||
f"{settings.API_V1_STR}/dashboards/{name}", headers=superuser_token_headers
|
||||
f"{settings.API_V1_STR}/dashboards/",
|
||||
headers=superuser_token_headers,
|
||||
json={"id": name, "name": name},
|
||||
)
|
||||
assert created.status_code in (200, 201, 409), created.text
|
||||
# A new dashboard is a draft, and a panel only reaches what is
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""A deleted flow's last key, kept for as long as history names its id."""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from sqlmodel import Session, delete, select
|
||||
|
||||
from fluksio.flow import tombstones
|
||||
from fluksio.flow.schemas import FlowDef
|
||||
from fluksio.flow.store import FlowStore
|
||||
from fluksio.flow.tombstones import Tombstones, names_of
|
||||
from fluksio.models import EngineEvent, FlowTombstone, MetricBucket, Run
|
||||
|
||||
|
||||
def test_a_tombstone_stays_while_history_names_its_id(db: Session) -> None:
|
||||
graves = Tombstones()
|
||||
for flow_id in ("f_gonegone", "f_trainee", "f_measured", "f_forgotten"):
|
||||
graves.bury(flow_id, flow_id.removeprefix("f_"))
|
||||
now = datetime.now(UTC)
|
||||
# A training run is filed under the training flow, and names the node it
|
||||
# trains in another, by that flow's id.
|
||||
db.add(
|
||||
Run(
|
||||
id="tomb-run",
|
||||
flow="wren_train",
|
||||
for_node="f_trainee.lights",
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
db.add(Run(id="tomb-own", flow="f_gonegone", created_at=now))
|
||||
db.add(MetricBucket(flow="other", node="f_measured.sensor", bucket=now))
|
||||
db.add(EngineEvent(ts=now, type="audit", flow="f_forgotten_too"))
|
||||
db.commit()
|
||||
|
||||
assert graves.recorded("f_trainee") == "trainee"
|
||||
assert graves.recorded("wren_train") == "wren_train" # history, no tombstone
|
||||
assert graves.recorded("f_nothing") is None
|
||||
|
||||
assert tombstones.prune(db) == 1
|
||||
db.commit()
|
||||
kept = {stone.id for stone in db.exec(select(FlowTombstone))}
|
||||
assert kept == {"f_gonegone", "f_trainee", "f_measured"}
|
||||
|
||||
db.exec(delete(Run).where(Run.id.in_(["tomb-run", "tomb-own"]))) # type: ignore[attr-defined,call-overload]
|
||||
db.exec(delete(MetricBucket)) # type: ignore[call-overload]
|
||||
db.exec(delete(EngineEvent)) # type: ignore[call-overload]
|
||||
db.commit()
|
||||
assert tombstones.prune(db) == 3
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_a_name_comes_from_the_live_flow_then_its_tombstone_then_the_id(
|
||||
db: Session, tmp_path: Path
|
||||
) -> None:
|
||||
store = FlowStore(tmp_path / "flows")
|
||||
store.write_flow(FlowDef(id="f_livelive", name="heating"))
|
||||
Tombstones().bury("f_deadbeef", "cooling")
|
||||
|
||||
assert names_of(["f_livelive", "f_deadbeef", "old_key"], store) == {
|
||||
"f_livelive": "heating",
|
||||
"f_deadbeef": "cooling",
|
||||
"old_key": "old_key",
|
||||
}
|
||||
Tombstones().lift("f_deadbeef")
|
||||
assert db.get(FlowTombstone, "f_deadbeef") is None
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Flows and dashboards over HTTP, the way most tests want them there."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from fluksio.core.config import settings
|
||||
|
||||
FLOWS = f"{settings.API_V1_STR}/flows"
|
||||
DASHBOARDS = f"{settings.API_V1_STR}/dashboards"
|
||||
|
||||
|
||||
def put_flow(
|
||||
client: TestClient,
|
||||
headers: dict[str, str],
|
||||
definition: dict[str, Any],
|
||||
flow_id: str = "",
|
||||
) -> httpx.Response:
|
||||
"""Create the flow under *flow_id*, its key unless given, or save over it.
|
||||
|
||||
Creating pins the id, so the test can address the flow by what it knows,
|
||||
which needs a superuser's *headers*. Saving goes from whatever version is
|
||||
stored. What a test meant by ``PUT`` before ``PUT`` stopped creating.
|
||||
"""
|
||||
flow_id = flow_id or definition["name"]
|
||||
current = client.get(f"{FLOWS}/{flow_id}", headers=headers)
|
||||
if current.status_code == 404:
|
||||
return client.post(
|
||||
f"{FLOWS}/", headers=headers, json={**definition, "id": flow_id}
|
||||
)
|
||||
version = current.json()["definition"]["version"]
|
||||
return client.put(
|
||||
f"{FLOWS}/{flow_id}",
|
||||
headers=headers,
|
||||
json={**definition, "id": flow_id, "version": version},
|
||||
)
|
||||
|
||||
|
||||
def put_dashboard(
|
||||
client: TestClient,
|
||||
headers: dict[str, str],
|
||||
document: dict[str, Any],
|
||||
dashboard_id: str = "",
|
||||
) -> httpx.Response:
|
||||
"""Create the dashboard under *dashboard_id*, its key unless given, and
|
||||
save *document* over it, as `put_flow` does for a flow."""
|
||||
dashboard_id = dashboard_id or document["name"]
|
||||
current = client.get(f"{DASHBOARDS}/{dashboard_id}?draft=true", headers=headers)
|
||||
if current.status_code == 404:
|
||||
created = client.post(
|
||||
f"{DASHBOARDS}/",
|
||||
headers=headers,
|
||||
json={"id": dashboard_id, "name": document["name"]},
|
||||
)
|
||||
assert created.status_code == 201, created.text
|
||||
current = created
|
||||
return client.put(
|
||||
f"{DASHBOARDS}/{dashboard_id}",
|
||||
headers=headers,
|
||||
json={
|
||||
**document,
|
||||
"id": dashboard_id,
|
||||
"version": current.json()["version"],
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Store helpers for tests that work below the API."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def created(store: Any, document: Any) -> Any:
|
||||
"""*document* created as a draft, under its key as its id.
|
||||
|
||||
How a store test made one before ids were issued: the id it addresses the
|
||||
object by is the name it gave. Works for flows and dashboards alike.
|
||||
"""
|
||||
return store.create(document, document.name)
|
||||
Reference in New Issue
Block a user