From 08a6376836e223f5c6dde82b86a52088612b8316 Mon Sep 17 00:00:00 2001 From: stroblme Date: Mon, 2 Nov 2026 17:41:23 +0100 Subject: [PATCH] Flows and dashboards are created by POST under an issued or superuser-pinned id; PUT only updates; deleted flows leave a tombstone with their key; the controller stores by id and runs by key, hooks live under /hooks/; a key with state or queued work behind it is refused until cleared --- .../versions/c4f1a8e2d7b3_flow_tombstone.py | 35 ++ backend/fluksio/api/deps.py | 32 +- backend/fluksio/api/routes/dashboards.py | 197 ++++---- backend/fluksio/api/routes/flows.py | 424 +++++++++++------- backend/fluksio/api/routes/observability.py | 3 +- backend/fluksio/flow/controller.py | 132 ++++-- backend/fluksio/flow/dashboards.py | 135 ++++-- backend/fluksio/flow/devices.py | 2 +- backend/fluksio/flow/metrics.py | 4 + backend/fluksio/flow/nodes/http.py | 4 +- backend/fluksio/flow/nodes/websocket.py | 6 +- backend/fluksio/flow/queue.py | 104 ++++- backend/fluksio/flow/runs.py | 2 +- backend/fluksio/flow/schemas.py | 30 ++ backend/fluksio/flow/store.py | 276 +++++++++--- backend/fluksio/flow/tombstones.py | 130 ++++++ backend/fluksio/main.py | 5 +- backend/fluksio/models.py | 19 + backend/fluksio/sdk/client.py | 42 +- backend/scripts/bench_startup.py | 5 +- backend/scripts/soak.py | 8 +- backend/tests/api/routes/test_code_rights.py | 6 +- backend/tests/api/routes/test_dashboards.py | 122 +++-- backend/tests/api/routes/test_devices.py | 51 ++- backend/tests/api/routes/test_flavors.py | 15 +- backend/tests/api/routes/test_flow_ids.py | 341 ++++++++++++++ backend/tests/api/routes/test_flows.py | 123 ++--- .../tests/api/routes/test_observability.py | 6 +- backend/tests/api/routes/test_search.py | 37 +- backend/tests/api/routes/test_sync.py | 40 +- backend/tests/api/routes/test_wren_train.py | 7 +- backend/tests/flow/test_dashboards.py | 18 +- backend/tests/flow/test_drafts.py | 29 +- backend/tests/flow/test_flow_ids.py | 157 +++++++ backend/tests/flow/test_identity.py | 280 ++++++++++-- backend/tests/flow/test_library.py | 5 +- backend/tests/flow/test_node_settings.py | 5 +- backend/tests/flow/test_queue.py | 25 ++ backend/tests/flow/test_redis_queue.py | 27 ++ backend/tests/flow/test_runtime_control.py | 6 +- backend/tests/test_artifacts.py | 5 +- backend/tests/test_cloud.py | 4 +- backend/tests/test_tombstones.py | 65 +++ backend/tests/utils/flows.py | 66 +++ backend/tests/utils/stores.py | 12 + docs/code/api.md | 86 ++-- docs/interface/portal.md | 2 +- docs/reference/node-types.md | 7 +- frontend/src/client/schemas.gen.ts | 84 ++++ frontend/src/client/sdk.gen.ts | 338 ++++++++------ frontend/src/client/types.gen.ts | 119 +++-- .../components/Dashboard/DashboardEditor.tsx | 3 +- frontend/src/components/Dashboard/queries.ts | 9 +- .../src/components/Devices/DeviceDetail.tsx | 4 +- .../src/components/Flow/EdgeInspector.tsx | 2 +- frontend/src/components/Flow/FlowEditor.tsx | 18 +- frontend/src/components/Flow/FlowNode.tsx | 2 +- frontend/src/components/Flow/NodePanel.tsx | 9 +- frontend/src/components/Flow/queries.ts | 14 +- frontend/src/components/Health/FlowTable.tsx | 4 +- .../src/components/Runs/OpenInDashboard.tsx | 2 +- .../src/routes/_layout/dashboards/index.tsx | 10 +- frontend/src/routes/_layout/flows/index.tsx | 8 +- 63 files changed, 2832 insertions(+), 936 deletions(-) create mode 100644 backend/fluksio/alembic/versions/c4f1a8e2d7b3_flow_tombstone.py create mode 100644 backend/fluksio/flow/tombstones.py create mode 100644 backend/tests/api/routes/test_flow_ids.py create mode 100644 backend/tests/flow/test_flow_ids.py create mode 100644 backend/tests/test_tombstones.py create mode 100644 backend/tests/utils/flows.py create mode 100644 backend/tests/utils/stores.py diff --git a/backend/fluksio/alembic/versions/c4f1a8e2d7b3_flow_tombstone.py b/backend/fluksio/alembic/versions/c4f1a8e2d7b3_flow_tombstone.py new file mode 100644 index 0000000..ed75d39 --- /dev/null +++ b/backend/fluksio/alembic/versions/c4f1a8e2d7b3_flow_tombstone.py @@ -0,0 +1,35 @@ +"""flow_tombstone + +The last key of a deleted flow, so its history still reads as something. + +Revision ID: c4f1a8e2d7b3 +Revises: b9e4d2a7c1f6 +Create Date: 2026-11-02 + +""" + +import sqlalchemy as sa +import sqlmodel.sql.sqltypes +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c4f1a8e2d7b3" +down_revision = "b9e4d2a7c1f6" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "flow_tombstone", + sa.Column("id", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False), + sa.Column( + "name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False + ), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade(): + op.drop_table("flow_tombstone") diff --git a/backend/fluksio/api/deps.py b/backend/fluksio/api/deps.py index 69a1f67..6110016 100644 --- a/backend/fluksio/api/deps.py +++ b/backend/fluksio/api/deps.py @@ -4,7 +4,7 @@ from typing import Annotated, Any from urllib.parse import unquote import jwt -from fastapi import Depends, HTTPException, Request, status +from fastapi import APIRouter, Depends, HTTPException, Request, status from fastapi.security import OAuth2PasswordBearer from jwt.exceptions import InvalidTokenError from pydantic import ValidationError @@ -615,3 +615,33 @@ def get_code_author(current_user: CurrentUser) -> User: if not current_user.is_superuser: raise HTTPException(status_code=403, detail=CODE_IS_SUPERUSERS) return current_user + + +def check_pin(pin: str, user: User, *routers: APIRouter) -> None: + """Refuse a pinned id from anyone but a superuser, or one a route shadows. + + Creating a flow or a dashboard gives it an opaque id. A pinned one is how + the Demo, a test or a restore keeps the id it already knows. *routers* are + the ones serving that kind's paths: a pin equal to one of their literal + segments, such as ``graph``, would be unreachable behind that route. + """ + if not pin: + return + if not user.is_superuser: + raise HTTPException(status_code=403, detail="Only a superuser can choose an id") + shadowed = { + segment + for router in routers + for route in router.routes + if "{" + not in ( + segment := getattr(route, "path", "")[len(router.prefix) :] + .strip("/") + .split("/")[0] + ) + } + if pin in shadowed: + raise HTTPException( + status_code=422, + detail=f"'{pin}' is a path of the API, so it cannot be an id", + ) diff --git a/backend/fluksio/api/routes/dashboards.py b/backend/fluksio/api/routes/dashboards.py index cc2fe54..2c1a184 100644 --- a/backend/fluksio/api/routes/dashboards.py +++ b/backend/fluksio/api/routes/dashboards.py @@ -1,4 +1,8 @@ -"""Dashboards: documents of widgets bound to message names.""" +"""Dashboards: documents of widgets bound to message names. + +A dashboard is addressed by its id in every path. Its key, ``name``, is a +label unique among dashboards and what ``GET /dashboards?name=`` looks up. +""" import time from typing import Any @@ -7,18 +11,24 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.concurrency import run_in_threadpool from pydantic import BaseModel -from fluksio.api.deps import DashboardStoreDep, FlowControllerDep, get_current_user +from fluksio.api.deps import ( + CurrentUser, + DashboardStoreDep, + FlowControllerDep, + check_pin, + get_current_user, +) from fluksio.flow.dashboards import ( + DashboardCreate, DashboardDef, DashboardExists, DashboardNotFound, DashboardsPublic, default_dashboard, results_dashboard, - results_name, ) from fluksio.flow.events import event_bus -from fluksio.flow.store import FlowNotFound, StaleVersion +from fluksio.flow.store import FlowNotFound, IdTaken, StaleVersion from fluksio.models import Message router = APIRouter( @@ -35,97 +45,118 @@ def _apply_history_limits(store: Any, controller: Any) -> None: controller.set_history_limits(store.history_requirements()) +def _missing(dashboard_id: str) -> HTTPException: + return HTTPException( + status_code=404, detail=f"No dashboard with the id '{dashboard_id}'" + ) + + @router.get("/", response_model=DashboardsPublic) -async def read_dashboards(store: DashboardStoreDep) -> Any: - """Every dashboard, without its contents.""" +async def read_dashboards(store: DashboardStoreDep, name: str | None = None) -> Any: + """Every dashboard, without its contents, or with ``name`` the one with + that key.""" summaries = await run_in_threadpool(store.list) + if name is not None: + summaries = [summary for summary in summaries if summary.name == name] return DashboardsPublic(data=summaries, count=len(summaries)) -@router.get("/{name}", response_model=DashboardDef) +@router.post("/", response_model=DashboardDef, status_code=201) +async def create_dashboard( + body: DashboardCreate, store: DashboardStoreDep, user: CurrentUser +) -> Any: + """Start a dashboard: one empty grid under a key of its own. + + The server gives it an opaque id; a superuser may pin one with ``id``. A + draft, like every edit that follows it — a dashboard reaches a panel only + once someone publishes it, so an empty one never does. + """ + check_pin(body.id, user, router) + try: + # No history limits to apply: they are read from the published + # documents, and this one is not one of them yet. + return await run_in_threadpool( + store.create, default_dashboard(body.name, body.title), body.id + ) + except (DashboardExists, IdTaken) as exc: + raise HTTPException(status_code=409, detail=str(exc)) + + +@router.get("/{dashboard_id}", response_model=DashboardDef) async def read_dashboard( - name: str, store: DashboardStoreDep, draft: bool = False + dashboard_id: str, store: DashboardStoreDep, draft: bool = False ) -> Any: """What a panel shows, or with ``draft`` the copy the editor is on.""" try: - return await run_in_threadpool(store.read, name, draft) + return await run_in_threadpool(store.read, dashboard_id, draft) except DashboardNotFound: - raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'") + raise _missing(dashboard_id) -@router.post("/{name}", response_model=DashboardDef) -async def create_dashboard(name: str, store: DashboardStoreDep) -> Any: - """Start a dashboard: one page, one section, nothing on it yet. - - A draft, like every edit that follows it — a dashboard reaches a panel - only once someone publishes it, so an empty one never does. - """ - if await run_in_threadpool(store.exists, name): - raise HTTPException(status_code=409, detail=f"'{name}' already exists") - try: - defn = default_dashboard(name) - except ValueError as exc: - raise HTTPException(status_code=422, detail=str(exc)) - # No history limits to apply: they are read from the published documents, - # and this one is not one of them yet. - return await run_in_threadpool(store.write_draft, defn, 0) - - -@router.post("/from-flow/{flow}", response_model=DashboardDef) +@router.post("/from-flow/{flow_id}", response_model=DashboardDef, status_code=201) async def generate_results_dashboard( - flow: str, + flow_id: str, store: DashboardStoreDep, controller: FlowControllerDep, ) -> Any: """Draw a batch flow's results as a dashboard, from the ports it declares. - Published straight away rather than left as a draft: there is nothing to - review that the flow did not already say, and what makes it useful is - being able to open it against a run immediately. It is an ordinary - dashboard afterwards — editing it is how it stops being generic. + Keyed after the flow's key, ``_results``, which a second one refuses + (409) rather than numbering. Published straight away rather than left as a + draft: there is nothing to review that the flow did not already say, and + what makes it useful is being able to open it against a run immediately. + It is an ordinary dashboard afterwards — editing it is how it stops being + generic. """ try: - definition = await run_in_threadpool(controller.store.read_flow, flow) + definition = await run_in_threadpool(controller.store.read_flow, flow_id) except FlowNotFound: - raise HTTPException(status_code=404, detail=f"No flow named '{flow}'") + raise HTTPException(status_code=404, detail=f"No flow with the id '{flow_id}'") if definition.mode != "batch": raise HTTPException( status_code=422, - detail=f"'{flow}' is a live flow; a results dashboard is a run's view", + detail=( + f"'{definition.name}' is a live flow; a results dashboard is a " + "run's view" + ), ) - name = results_name(flow) - if await run_in_threadpool(store.exists, name): - raise HTTPException(status_code=409, detail=f"'{name}' already exists") - try: - written = await run_in_threadpool(store.write, results_dashboard(definition)) + written = await run_in_threadpool( + store.create, results_dashboard(definition), "", True + ) except DashboardExists as exc: raise HTTPException(status_code=409, detail=str(exc)) await run_in_threadpool(_apply_history_limits, store, controller) event_bus.publish( - {"type": "dashboard_changed", "dashboard": name, "ts": time.time()} + {"type": "dashboard_changed", "dashboard": written.id, "ts": time.time()} ) return written -@router.put("/{name}", response_model=DashboardDef) +@router.put("/{dashboard_id}", response_model=DashboardDef) async def save_dashboard( - name: str, + dashboard_id: str, body: DashboardDef, store: DashboardStoreDep, ) -> Any: """Save unpublished changes, refusing a save someone else has moved past. - This writes a draft: panels keep showing the published document until - someone publishes, so nothing here can change what a wall is displaying — - which is also why the engine's history limits are left alone. + Only to a dashboard that exists: creating one is ``POST /dashboards``, and + neither the id nor the key changes here. This writes a draft: panels keep + showing the published document until someone publishes, so nothing here + can change what a wall is displaying — which is also why the engine's + history limits are left alone. """ - if body.name != name: - raise HTTPException(status_code=422, detail="The name in the body must match") - if body.id and body.id != name: + if body.id and body.id != dashboard_id: raise HTTPException(status_code=422, detail="The id in the body must match") try: - saved = await run_in_threadpool(store.write_draft, body, body.version) + return await run_in_threadpool( + store.write_draft, + body.model_copy(update={"id": dashboard_id}), + body.version, + ) + except DashboardNotFound: + raise _missing(dashboard_id) except StaleVersion as exc: raise HTTPException( status_code=409, @@ -134,27 +165,25 @@ async def save_dashboard( "current_version": exc.current, }, ) - except DashboardExists as exc: - raise HTTPException(status_code=409, detail=str(exc)) - return saved -@router.post("/{name}/publish", response_model=DashboardDef) +@router.post("/{dashboard_id}/publish", response_model=DashboardDef) async def publish_dashboard( - name: str, + dashboard_id: str, body: PublishRequest, store: DashboardStoreDep, controller: FlowControllerDep, ) -> Any: """Put the unpublished changes on the panels.""" - if not await run_in_threadpool(store.exists, name): - raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'") - if not await run_in_threadpool(store.has_draft, name): + if not await run_in_threadpool(store.exists, dashboard_id): + raise _missing(dashboard_id) + if not await run_in_threadpool(store.has_draft, dashboard_id): raise HTTPException( - status_code=400, detail=f"Dashboard '{name}' has no unpublished changes" + status_code=400, + detail=f"Dashboard '{dashboard_id}' has no unpublished changes", ) try: - published = await run_in_threadpool(store.publish, name, body.version) + published = await run_in_threadpool(store.publish, dashboard_id, body.version) except StaleVersion as exc: raise HTTPException( status_code=409, @@ -167,50 +196,52 @@ async def publish_dashboard( # What a panel is showing has changed. Panels watch the flow socket, and # the tile values alone cannot tell them the document itself moved. event_bus.publish( - {"type": "dashboard_changed", "dashboard": name, "ts": time.time()} + {"type": "dashboard_changed", "dashboard": dashboard_id, "ts": time.time()} ) return published -@router.post("/{name}/discard", response_model=DashboardDef) -async def discard_dashboard_draft(name: str, store: DashboardStoreDep) -> Any: +@router.post("/{dashboard_id}/discard", response_model=DashboardDef) +async def discard_dashboard_draft(dashboard_id: str, store: DashboardStoreDep) -> Any: """Throw the unpublished changes away and go back to what is shown.""" - if not await run_in_threadpool(store.exists, name): - raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'") - if not await run_in_threadpool(store.has_draft, name): + if not await run_in_threadpool(store.exists, dashboard_id): + raise _missing(dashboard_id) + if not await run_in_threadpool(store.has_draft, dashboard_id): raise HTTPException( - status_code=400, detail=f"Dashboard '{name}' has no unpublished changes" + status_code=400, + detail=f"Dashboard '{dashboard_id}' has no unpublished changes", ) - if not await run_in_threadpool(store.is_published, name): + if not await run_in_threadpool(store.is_published, dashboard_id): raise HTTPException( status_code=400, detail=( - f"Dashboard '{name}' has never been published. Delete it instead " - "of discarding it" + f"Dashboard '{dashboard_id}' has never been published. Delete it " + "instead of discarding it" ), ) - return await run_in_threadpool(store.discard_draft, name) + return await run_in_threadpool(store.discard_draft, dashboard_id) -@router.delete("/{name}", response_model=Message) +@router.delete("/{dashboard_id}", response_model=Message) async def delete_dashboard( - name: str, store: DashboardStoreDep, controller: FlowControllerDep + dashboard_id: str, store: DashboardStoreDep, controller: FlowControllerDep ) -> Any: try: - await run_in_threadpool(store.delete, name) + await run_in_threadpool(store.delete, dashboard_id) except DashboardNotFound: - raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'") + raise _missing(dashboard_id) await run_in_threadpool(_apply_history_limits, store, controller) - return Message(message=f"Deleted dashboard '{name}'") + return Message(message=f"Deleted dashboard '{dashboard_id}'") -@router.post("/{name}/duplicate", response_model=DashboardDef) -async def duplicate_dashboard(name: str, store: DashboardStoreDep) -> Any: +@router.post("/{dashboard_id}/duplicate", response_model=DashboardDef) +async def duplicate_dashboard(dashboard_id: str, store: DashboardStoreDep) -> Any: """Copy a dashboard into a new one that is only a draft, named after it. - No history limits to apply, as for a new dashboard: nothing published moved. + The copy gets an id of its own. No history limits to apply, as for a new + dashboard: nothing published moved. """ try: - return await run_in_threadpool(store.duplicate, name) + return await run_in_threadpool(store.duplicate, dashboard_id) except DashboardNotFound: - raise HTTPException(status_code=404, detail=f"No dashboard named '{name}'") + raise _missing(dashboard_id) diff --git a/backend/fluksio/api/routes/flows.py b/backend/fluksio/api/routes/flows.py index de1c831..fd74bea 100644 --- a/backend/fluksio/api/routes/flows.py +++ b/backend/fluksio/api/routes/flows.py @@ -1,4 +1,10 @@ -"""The flow API. Everything the editor can do is available here first.""" +"""The flow API. Everything the editor can do is available here first. + +A flow is addressed by its id in every path, and never by its key: the key is +the namespace of its messages, which is what the engine goes by, so each route +reads the flow and runs, pauses or reads values under the key it holds. Node +ids in paths are the flow's own, unqualified. +""" import asyncio import logging @@ -30,6 +36,7 @@ from fluksio.api.deps import ( HiddenDep, SessionDep, TokenDep, + check_pin, decode_token, get_code_author, get_current_user, @@ -63,6 +70,7 @@ from fluksio.flow.runs import RunRejected from fluksio.flow.schemas import ( NAME_PATTERN, BrainGraph, + FlowCreate, FlowDef, FlowsPublic, FlowStatePublic, @@ -81,6 +89,7 @@ from fluksio.flow.state import as_number from fluksio.flow.store import ( FlowExists, FlowNotFound, + IdTaken, LibExists, LibNotFound, StaleVersion, @@ -247,10 +256,10 @@ def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail: "paused": controller.is_paused(name), } endpoints = _endpoints(controller, definition) - if controller.store.has_draft(name): + if controller.store.has_draft(definition.id): # Report the draft the editor is showing, not the version running # underneath it — otherwise a node the author just broke looks fine. - preview = controller.preview(name) + preview = controller.preview(definition.id) return FlowDetail( definition=definition, nodes=preview.nodes, @@ -268,16 +277,21 @@ def _detail(controller: FlowController, definition: FlowDef) -> FlowDetail: ) -def _read_flow(controller: FlowController, name: str) -> FlowDef: +def _missing(flow_id: str) -> HTTPException: + return HTTPException(status_code=404, detail=f"No flow with the id '{flow_id}'") + + +def _read_flow(controller: FlowController, flow_id: str) -> FlowDef: """The working copy: the draft when there is one, else what is published.""" try: - return controller.store.read_flow(name, draft=True) + return controller.store.read_flow(flow_id, draft=True) except FlowNotFound: - raise HTTPException(status_code=404, detail=f"No flow named '{name}'") + raise _missing(flow_id) def _audit(action: str, flow: str, user: CurrentUser) -> None: - """Record who changed what. The collector writes it down; the bus carries it.""" + """Record who changed what, by the flow's key. The collector writes it + down; the bus carries it.""" event_bus.publish( { "type": "audit", @@ -296,6 +310,7 @@ def _source_ref(definition: FlowDef, node_id: str) -> str | None: def _require_enabled(controller: FlowController, name: str) -> None: + """Refuse running a stopped flow, *name* being its key.""" if not controller.is_enabled(name): raise HTTPException( status_code=409, @@ -306,7 +321,8 @@ def _require_enabled(controller: FlowController, name: str) -> None: def _flow_state( controller: FlowController, name: str, hidden: frozenset[str] ) -> FlowStatePublic: - """The flow's values, but those of a camera the caller may not see.""" + """The values of the flow with key *name*, but a camera's the caller may + not see.""" return FlowStatePublic( values={ key: MessageValue(**value) @@ -323,29 +339,34 @@ def _flow_state( @router.get("/", response_model=FlowsPublic) -def read_flows(controller: FlowControllerDep) -> Any: - """List every flow.""" +def read_flows(controller: FlowControllerDep, name: str | None = None) -> Any: + """List every flow, or with ``name`` the one holding that key.""" + ids = controller.store.list_flows() + if name is not None: + found = controller.store.key_index().get(name) + ids = [found] if found else [] summaries = [] - for name in controller.store.list_flows(): + for flow_id in ids: try: - definition = controller.store.read_flow(name, draft=True) + definition = controller.store.read_flow(flow_id, draft=True) except FlowNotFound: continue - statuses = controller.node_statuses(name) + key = definition.name + statuses = controller.node_statuses(key) summaries.append( FlowSummary( id=definition.id, - name=definition.name, + name=key, title=definition.title, mode=definition.mode, node_count=len(definition.nodes), error_count=sum(1 for s in statuses if s.status == "error"), - has_draft=controller.store.has_draft(name), - enabled=controller.is_enabled(name), - paused=controller.is_paused(name), - quarantined=controller.is_quarantined(name), + has_draft=controller.store.has_draft(flow_id), + enabled=controller.is_enabled(key), + paused=controller.is_paused(key), + quarantined=controller.is_quarantined(key), version=definition.version, - updated_at=controller.store.updated_at(name), + updated_at=controller.store.updated_at(flow_id), ) ) return FlowsPublic(data=summaries, count=len(summaries)) @@ -359,7 +380,7 @@ def read_node_types() -> Any: return node_type_info() -# Above "/{name}" for the same reason "library" is: a flow called "graph" would +# Above "/{flow_id}" for the same reason "library" is: a flow called "graph" would # otherwise be unreachable. @router.get("/graph", response_model=BrainGraph) def read_graph(controller: FlowControllerDep) -> Any: @@ -370,7 +391,7 @@ def read_graph(controller: FlowControllerDep) -> Any: # ----------------------------------------------------------------------------- # Shared nodes # -# Declared above the "/{name}" routes: "library" would otherwise be read as a +# Declared above the "/{flow_id}" routes: "library" would otherwise be read as a # flow name. # ----------------------------------------------------------------------------- @@ -399,8 +420,9 @@ async def delete_shared_node( await run_in_threadpool(controller.store.detach_lib, lib_name) except LibNotFound: raise HTTPException(status_code=404, detail=f"No shared node '{lib_name}'") - for flow in sorted({usage.split(".", 1)[0] for usage in running}): - await controller.reload_flow(flow) + # `usages` qualifies with the flow's id. + for flow_id in sorted({usage.split(".", 1)[0] for usage in running}): + await controller.reload_flow(flow_id) return Message(message=f"Deleted shared node '{lib_name}'") # The published flows too: a draft that unshared a node still runs it. used_by = sorted( @@ -418,10 +440,10 @@ async def delete_shared_node( return Message(message=f"Deleted shared node '{lib_name}'") -@router.get("/{name}", response_model=FlowDetail) -def read_flow(name: str, controller: FlowControllerDep) -> Any: +@router.get("/{flow_id}", response_model=FlowDetail) +def read_flow(flow_id: str, controller: FlowControllerDep) -> Any: """Read one flow, with the state of its nodes.""" - return _detail(controller, _read_flow(controller, name)) + return _detail(controller, _read_flow(controller, flow_id)) def _check_flavors(definition: FlowDef) -> None: @@ -518,147 +540,192 @@ def _refuse_code_changes(definition: FlowDef, before: FlowDef | None) -> None: ) -@router.put("/{name}", response_model=FlowDetail) +def _check_document(definition: FlowDef) -> None: + """What a create and a save both refuse in a document.""" + if len({n.id for n in definition.nodes}) != len(definition.nodes): + raise HTTPException(status_code=400, detail="Node names must be unique") + _check_flavors(definition) + _check_origin(definition) + + +@router.post("/", response_model=FlowDetail, status_code=201) +async def create_flow( + body: FlowCreate, controller: FlowControllerDep, user: CurrentUser +) -> Any: + """Create a flow. It is only a draft until someone publishes it. + + The server gives it an opaque id. A superuser may pin one with ``id``: an + id no flow holds, or one whose history this flow takes up again, under the + key that history was kept under. A key with state or queued work still + behind it is refused until that is gone, so a new flow never inherits what + another left. + """ + check_pin(body.id, user, router, ws_router) + definition = body.document() + _check_document(definition) + if not user.is_superuser: + _refuse_code_changes(definition, None) + residue = await run_in_threadpool(controller.key_residue, definition.name) + if residue: + raise HTTPException( + status_code=409, + detail=( + f"The key '{definition.name}' still has values or queued work " + f"behind it ({', '.join(residue[:3])}). Wait for the work to " + "finish, or stop what still writes to it." + ), + ) + try: + stored = await run_in_threadpool(controller.store.create, definition, body.id) + except (FlowExists, IdTaken) as exc: + raise HTTPException(status_code=409, detail=str(exc)) + return await run_in_threadpool(_detail, controller, stored) + + +@router.put("/{flow_id}", response_model=FlowDetail) async def save_flow( - name: str, + flow_id: str, definition: FlowDef, controller: FlowControllerDep, user: CurrentUser, ) -> Any: """Save unpublished changes to a flow. - This writes a draft: the running pipeline keeps the published version until - someone publishes. ``version`` is the one the editor last saw — a mismatch - means another client saved in between and answers 409 rather than throwing - their work away. + Only to one that exists: creating one is ``POST /flows``, and neither the + id nor the key changes here. This writes a draft: the running pipeline + keeps the published version until someone publishes. ``version`` is the + one the editor last saw — a mismatch means another client saved in + between and answers 409 rather than throwing their work away. """ - if definition.name != name: - raise HTTPException( - status_code=400, detail="The flow name in the body must match the URL" - ) - if definition.id and definition.id != name: + if definition.id and definition.id != flow_id: raise HTTPException( status_code=400, detail="The flow id in the body must match the URL" ) - - duplicates = {n.id for n in definition.nodes} - if len(duplicates) != len(definition.nodes): - raise HTTPException(status_code=400, detail="Node names must be unique") - - _check_flavors(definition) - _check_origin(definition) + definition = definition.model_copy(update={"id": flow_id}) + _check_document(definition) + before = _read_flow(controller, flow_id) if not user.is_superuser: - before = _read_flow(controller, name) if controller.store.exists(name) else None _refuse_code_changes(definition, before) try: stored = await run_in_threadpool( controller.store.write_draft, definition, definition.version ) + except FlowNotFound: + raise _missing(flow_id) except StaleVersion as exc: raise HTTPException( status_code=409, detail={"message": str(exc), "current_version": exc.current}, ) - except FlowExists as exc: - raise HTTPException(status_code=409, detail=str(exc)) return await run_in_threadpool(_detail, controller, stored) -@router.post("/{name}/publish", response_model=FlowDetail) +@router.post("/{flow_id}/publish", response_model=FlowDetail) async def publish_flow( - name: str, + flow_id: str, body: PublishRequest, controller: FlowControllerDep, user: CurrentUser, ) -> Any: """Deploy the unpublished changes: the engine picks them up from here.""" - _read_flow(controller, name) - if not controller.store.has_draft(name): + definition = _read_flow(controller, flow_id) + if not controller.store.has_draft(flow_id): raise HTTPException( - status_code=400, detail=f"Flow '{name}' has no unpublished changes" + status_code=400, detail=f"Flow '{flow_id}' has no unpublished changes" ) try: published = await run_in_threadpool( - controller.store.publish_flow, name, body.version + controller.store.publish_flow, flow_id, body.version ) except StaleVersion as exc: raise HTTPException( status_code=409, detail={"message": str(exc), "current_version": exc.current}, ) - _audit("published", name, user) - await controller.reload_flow(name) + _audit("published", definition.name, user) + await controller.reload_flow(flow_id) return await run_in_threadpool(_detail, controller, published) -@router.post("/{name}/discard-draft", response_model=FlowDetail) -async def discard_draft(name: str, controller: FlowControllerDep) -> Any: +@router.post("/{flow_id}/discard-draft", response_model=FlowDetail) +async def discard_draft(flow_id: str, controller: FlowControllerDep) -> Any: """Throw the unpublished changes away and go back to what is running.""" - _read_flow(controller, name) - if not controller.store.has_draft(name): + _read_flow(controller, flow_id) + if not controller.store.has_draft(flow_id): raise HTTPException( - status_code=400, detail=f"Flow '{name}' has no unpublished changes" + status_code=400, detail=f"Flow '{flow_id}' has no unpublished changes" ) - if not controller.store.is_published(name): + if not controller.store.is_published(flow_id): raise HTTPException( status_code=400, detail=( - f"Flow '{name}' has never been published. Delete it instead of " + f"Flow '{flow_id}' has never been published. Delete it instead of " "discarding it" ), ) - published = await run_in_threadpool(controller.store.discard_draft, name) + published = await run_in_threadpool(controller.store.discard_draft, flow_id) return await run_in_threadpool(_detail, controller, published) -@router.delete("/{name}", response_model=Message) +@router.delete("/{flow_id}", response_model=Message) async def delete_flow( - name: str, controller: FlowControllerDep, user: CurrentUser, session: SessionDep + flow_id: str, controller: FlowControllerDep, user: CurrentUser, session: SessionDep ) -> Any: - """Delete a flow and everything in it, except the record of what it ran.""" + """Delete a flow and everything in it, except the record of what it ran. + + Its key is kept with its id (`fluksio.flow.tombstones`) for as long as + that record is, and what is left under the key, values and queued work, + goes with the flow so the key is free for a new one. + """ live = session.exec( select(Run.id).where( - col(Run.flow) == name, col(Run.status).in_(("running", "queued")) + col(Run.flow) == flow_id, col(Run.status).in_(("running", "queued")) ) ).first() if live is not None: raise HTTPException( status_code=409, detail=( - f"Flow '{name}' has a run in progress ({live}). Cancel it, or " + f"Flow '{flow_id}' has a run in progress ({live}). Cancel it, or " "wait for it to finish, before deleting the flow." ), ) try: - await run_in_threadpool(controller.store.delete_flow, name) + key = await run_in_threadpool(controller.store.delete_flow, flow_id) except FlowNotFound: - raise HTTPException(status_code=404, detail=f"No flow named '{name}'") - _audit("deleted", name, user) - # Its files are gone; its values and queued work would otherwise linger. + raise _missing(flow_id) + _audit("deleted", key, user) + # Its nodes first, or they would put values and work straight back. # Its runs stay: a run record is a record of what ran, and surviving the # flow it belonged to is the point of keeping one. `DELETE /runs?flow=` # is what clears them, and with `history=true` its cascades too. - await run_in_threadpool(controller.forget_flow, name) - await controller.reload_flow(name) - return Message(message=f"Deleted flow '{name}'") + await controller.reload_flow(flow_id) + await run_in_threadpool(controller.forget_flow, key) + return Message(message=f"Deleted flow '{key}'") -@router.post("/{name}/duplicate", response_model=FlowDetail) +@router.post("/{flow_id}/duplicate", response_model=FlowDetail) async def duplicate_flow( - name: str, controller: FlowControllerDep, user: CurrentUser + flow_id: str, controller: FlowControllerDep, user: CurrentUser ) -> Any: """Copy a flow into a new one that is only a draft, named after it. - Nothing to reload: a draft does not run. Its code nodes are new code in - the copy, so copying them is a superuser's, like writing them. + The copy gets an id of its own, and the first ``_copy`` key with nothing + left behind it. Nothing to reload: a draft does not run. Its code nodes + are new code in the copy, so copying them is a superuser's, like writing + them. """ if not user.is_superuser: - _refuse_code_changes(_read_flow(controller, name), None) + _refuse_code_changes(_read_flow(controller, flow_id), None) try: - copy = await run_in_threadpool(controller.store.duplicate_flow, name) + copy = await run_in_threadpool( + controller.store.duplicate_flow, + flow_id, + lambda key: bool(controller.key_residue(key)), + ) except FlowNotFound: - raise HTTPException(status_code=404, detail=f"No flow named '{name}'") + raise _missing(flow_id) return await run_in_threadpool(_detail, controller, copy) @@ -667,14 +734,14 @@ async def duplicate_flow( # ----------------------------------------------------------------------------- -@router.get("/{name}/nodes/{node_id}/source", response_model=NodeSource) +@router.get("/{flow_id}/nodes/{node_id}/source", response_model=NodeSource) def read_node_source( - name: str, + flow_id: str, node_id: str, controller: FlowControllerDep, ) -> Any: """Read a node's Python source, including unpublished edits.""" - definition = _read_flow(controller, name) + definition = _read_flow(controller, flow_id) ref = _source_ref(definition, node_id) if ref: try: @@ -682,13 +749,13 @@ def read_node_source( except LibNotFound: raise HTTPException(status_code=404, detail=f"No shared node '{ref}'") return NodeSource( - code=controller.store.read_node_source(name, node_id, draft=True), - missing=not controller.store.has_node_source(name, node_id, draft=True), + code=controller.store.read_node_source(flow_id, node_id, draft=True), + missing=not controller.store.has_node_source(flow_id, node_id, draft=True), ) def _follow_header( - controller: FlowController, name: str, node_id: str, code: str, drop: bool + controller: FlowController, flow_id: str, node_id: str, code: str, drop: bool ) -> FlowDef | None: """Bring a function node's ports and settings in line with its header. @@ -697,7 +764,7 @@ def _follow_header( and a synced flow's ports are declared in its repository. Returns the draft when it changed. """ - definition = controller.store.read_flow(name, draft=True) + definition = controller.store.read_flow(flow_id, draft=True) node = next((n for n in definition.nodes if n.id == node_id), None) spec = NODE_TYPES.get(node.type) if node is not None else None if ( @@ -720,12 +787,12 @@ def _follow_header( @router.put( - "/{name}/nodes/{node_id}/source", + "/{flow_id}/nodes/{node_id}/source", response_model=NodeSourceSaved, dependencies=[Depends(get_code_author)], ) async def save_node_source( - name: str, + flow_id: str, node_id: str, source: NodeSource, controller: FlowControllerDep, @@ -749,7 +816,8 @@ async def save_node_source( adds and renames from the header but keeps what it no longer names until the editor is left. """ - definition = _read_flow(controller, name) + definition = _read_flow(controller, flow_id) + name = definition.name ref = _source_ref(definition, node_id) if ref: changed = await run_in_threadpool( @@ -760,10 +828,10 @@ async def save_node_source( await controller.reload_lib_users(ref) else: await run_in_threadpool( - controller.store.write_node_source, name, node_id, source.code, True + controller.store.write_node_source, flow_id, node_id, source.code, True ) followed = await run_in_threadpool( - _follow_header, controller, name, node_id, source.code, drop + _follow_header, controller, flow_id, node_id, source.code, drop ) node_def = next((n for n in definition.nodes if n.id == node_id), None) worker = ( @@ -785,15 +853,15 @@ async def save_node_source( ) -@router.post("/{name}/nodes/{node_id}/share", response_model=FlowDetail) +@router.post("/{flow_id}/nodes/{node_id}/share", response_model=FlowDetail) async def share_node( - name: str, + flow_id: str, node_id: str, body: ShareRequest, controller: FlowControllerDep, ) -> Any: """Move this node's code into the library so other flows can run it too.""" - _read_flow(controller, name) + _read_flow(controller, flow_id) if not NAME_PATTERN.match(body.lib_name): raise HTTPException( status_code=400, @@ -803,26 +871,26 @@ async def share_node( ) try: await run_in_threadpool( - controller.store.share_node, name, node_id, body.lib_name + controller.store.share_node, flow_id, node_id, body.lib_name ) except LibExists as exc: raise HTTPException(status_code=409, detail=str(exc)) - return await run_in_threadpool(_detail, controller, _read_flow(controller, name)) + return await run_in_threadpool(_detail, controller, _read_flow(controller, flow_id)) -@router.post("/{name}/nodes/{node_id}/unshare", response_model=FlowDetail) +@router.post("/{flow_id}/nodes/{node_id}/unshare", response_model=FlowDetail) async def unshare_node( - name: str, + flow_id: str, node_id: str, controller: FlowControllerDep, ) -> Any: """Take a private copy of the shared code back into this flow.""" - _read_flow(controller, name) + _read_flow(controller, flow_id) try: - await run_in_threadpool(controller.store.unshare_node, name, node_id) + await run_in_threadpool(controller.store.unshare_node, flow_id, node_id) except LibNotFound as exc: raise HTTPException(status_code=404, detail=str(exc)) - return await run_in_threadpool(_detail, controller, _read_flow(controller, name)) + return await run_in_threadpool(_detail, controller, _read_flow(controller, flow_id)) # ----------------------------------------------------------------------------- @@ -830,49 +898,50 @@ async def unshare_node( # ----------------------------------------------------------------------------- -@router.post("/{name}/start", response_model=FlowDetail) +@router.post("/{flow_id}/start", response_model=FlowDetail) async def start_flow( - name: str, controller: FlowControllerDep, user: CurrentUser + flow_id: str, controller: FlowControllerDep, user: CurrentUser ) -> Any: """Let the engine run this flow again.""" - _read_flow(controller, name) - await controller.set_enabled(name, True) - _audit("started", name, user) - return await run_in_threadpool(_detail, controller, _read_flow(controller, name)) + definition = _read_flow(controller, flow_id) + await controller.set_enabled(flow_id, definition.name, True) + _audit("started", definition.name, user) + return await run_in_threadpool(_detail, controller, definition) -@router.post("/{name}/stop", response_model=FlowDetail) -async def stop_flow(name: str, controller: FlowControllerDep, user: CurrentUser) -> Any: +@router.post("/{flow_id}/stop", response_model=FlowDetail) +async def stop_flow( + flow_id: str, controller: FlowControllerDep, user: CurrentUser +) -> Any: """Take this flow off the engine: no subscriptions, schedules or webhooks.""" - _read_flow(controller, name) - await controller.set_enabled(name, False) - _audit("stopped", name, user) - return await run_in_threadpool(_detail, controller, _read_flow(controller, name)) + definition = _read_flow(controller, flow_id) + await controller.set_enabled(flow_id, definition.name, False) + _audit("stopped", definition.name, user) + return await run_in_threadpool(_detail, controller, definition) -@router.post("/{name}/pause", response_model=Message) -def pause_flow(name: str, controller: FlowControllerDep) -> Any: +@router.post("/{flow_id}/pause", response_model=Message) +def pause_flow(flow_id: str, controller: FlowControllerDep) -> Any: """Hold the flow's nodes so its messages can be stepped through.""" - _read_flow(controller, name) - controller.pause_flow(name) - return Message(message=f"Paused flow '{name}'") + controller.pause_flow(_read_flow(controller, flow_id).name) + return Message(message=f"Paused flow '{flow_id}'") -@router.post("/{name}/resume", response_model=Message) -async def resume_flow(name: str, controller: FlowControllerDep) -> Any: +@router.post("/{flow_id}/resume", response_model=Message) +async def resume_flow(flow_id: str, controller: FlowControllerDep) -> Any: """Let the flow carry on, running whatever was held back.""" - _read_flow(controller, name) + name = _read_flow(controller, flow_id).name await run_in_threadpool(controller.resume_flow, name) - return Message(message=f"Resumed flow '{name}'") + return Message(message=f"Resumed flow '{flow_id}'") -@router.post("/{name}/step", response_model=Message) -async def step_flow(name: str, controller: FlowControllerDep) -> Any: +@router.post("/{flow_id}/step", response_model=Message) +async def step_flow(flow_id: str, controller: FlowControllerDep) -> Any: """Run one message a pause is holding back, leaving the flow paused.""" - _read_flow(controller, name) + name = _read_flow(controller, flow_id).name node = await run_in_threadpool(controller.step_flow, name) if node is None: - return Message(message=f"Nothing held back in flow '{name}'") + return Message(message=f"Nothing held back in flow '{flow_id}'") return Message(message=f"Stepped '{node}'") @@ -881,18 +950,18 @@ async def step_flow(name: str, controller: FlowControllerDep) -> Any: # ----------------------------------------------------------------------------- -@router.post("/{name}/validate", response_model=ValidationResult) -def validate_flow(name: str, controller: FlowControllerDep) -> Any: +@router.post("/{flow_id}/validate", response_model=ValidationResult) +def validate_flow(flow_id: str, controller: FlowControllerDep) -> Any: """Report what would keep this flow from running.""" - _read_flow(controller, name) - if controller.store.has_draft(name): - return ValidationResult(issues=controller.preview(name).issues) - return ValidationResult(issues=controller.flow_issues(name)) + definition = _read_flow(controller, flow_id) + if controller.store.has_draft(flow_id): + return ValidationResult(issues=controller.preview(flow_id).issues) + return ValidationResult(issues=controller.flow_issues(definition.name)) -@router.post("/{name}/run", response_model=FlowStatePublic) +@router.post("/{flow_id}/run", response_model=FlowStatePublic) async def run_flow( - name: str, + flow_id: str, body: RunRequest, request: Request, controller: FlowControllerDep, @@ -908,7 +977,8 @@ async def run_flow( one means: it is what keeps the parameters, the series and the result, and a button that quietly did something else would be a trap. """ - definition = _read_flow(controller, name) + definition = _read_flow(controller, flow_id) + name = definition.name _require_enabled(controller, name) if definition.mode == "batch": service = getattr(request.app.state, "run_service", None) @@ -917,33 +987,34 @@ async def run_flow( try: await run_in_threadpool( service.submit, - name, + flow_id, params=body.inputs, cause="api", actor=user.email, - draft=controller.store.has_draft(name), + draft=controller.store.has_draft(flow_id), ) except RunRejected as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc return _flow_state(controller, name, hidden) inputs = {qualify(name, key): value for key, value in body.inputs.items()} - if controller.store.has_draft(name): - await run_in_threadpool(controller.run_preview, name, inputs) + if controller.store.has_draft(flow_id): + await run_in_threadpool(controller.run_preview, flow_id, inputs) else: await run_in_threadpool(controller.run_flow, name, inputs) return _flow_state(controller, name, hidden) -@router.post("/{name}/nodes/{node_id}/trigger", response_model=FlowStatePublic) +@router.post("/{flow_id}/nodes/{node_id}/trigger", response_model=FlowStatePublic) async def trigger_node( - name: str, + flow_id: str, node_id: str, body: TriggerRequest, controller: FlowControllerDep, hidden: HiddenDep, ) -> Any: """Feed values into a single node.""" + name = _read_flow(controller, flow_id).name _require_enabled(controller, name) try: error = await run_in_threadpool( @@ -951,7 +1022,7 @@ async def trigger_node( ) except KeyError: raise HTTPException( - status_code=404, detail=f"No node named '{node_id}' in flow '{name}'" + status_code=404, detail=f"No node named '{node_id}' in flow '{flow_id}'" ) if error: # The canvas already has the failure from the bus; the person who @@ -960,22 +1031,23 @@ async def trigger_node( return _flow_state(controller, name, hidden) -@router.post("/{name}/nodes/{node_id}/cancel", response_model=Message) -def cancel_node(name: str, node_id: str, controller: FlowControllerDep) -> Any: +@router.post("/{flow_id}/nodes/{node_id}/cancel", response_model=Message) +def cancel_node(flow_id: str, node_id: str, controller: FlowControllerDep) -> Any: """Stop a node that is running right now, by killing the worker running it. Idempotent on purpose: by the time a click reaches here the node may well have finished, and that is the outcome that was asked for. """ + name = _read_flow(controller, flow_id).name pool = controller.workers if pool is not None and pool.cancel(f"{name}.{node_id}"): return Message(message=f"Stopped '{node_id}'") return Message(message=f"'{node_id}' was not running") -@router.post("/{name}/nodes/{node_id}/acknowledge", response_model=Message) +@router.post("/{flow_id}/nodes/{node_id}/acknowledge", response_model=Message) def acknowledge_node_error( - name: str, node_id: str, controller: FlowControllerDep + flow_id: str, node_id: str, controller: FlowControllerDep ) -> Any: """Dismiss what a node last failed with, so the canvas stops marking it. @@ -983,18 +1055,19 @@ def acknowledge_node_error( an alert leaves no trace by the time anyone looks. Reading the traceback is what says it has been seen. """ + node = f"{_read_flow(controller, flow_id).name}.{node_id}" try: - controller.acknowledge_error(f"{name}.{node_id}") + controller.acknowledge_error(node) except KeyError: raise HTTPException( - status_code=404, detail=f"No node named '{node_id}' in flow '{name}'" + status_code=404, detail=f"No node named '{node_id}' in flow '{flow_id}'" ) from None # Another browser holds the same marker and has no reason to refetch, so it # would keep showing a failure that is gone until its next snapshot. event_bus.publish( { "type": "node_error_acknowledged", - "node": f"{name}.{node_id}", + "node": node, "ts": time.time(), } ) @@ -1006,21 +1079,23 @@ def acknowledge_node_error( # ----------------------------------------------------------------------------- -def _node_def(controller: FlowController, name: str, node_id: str) -> NodeDef: +def _node_def(controller: FlowController, flow_id: str, node_id: str) -> NodeDef: """A node as the editor shows it: from the draft when there is one.""" node = next( - (n for n in _read_flow(controller, name).nodes if n.id == node_id), None + (n for n in _read_flow(controller, flow_id).nodes if n.id == node_id), None ) if node is None: raise HTTPException( - status_code=404, detail=f"No node named '{node_id}' in flow '{name}'" + status_code=404, detail=f"No node named '{node_id}' in flow '{flow_id}'" ) return node -@router.post("/{name}/nodes/{node_id}/train", response_model=Training, status_code=202) +@router.post( + "/{flow_id}/nodes/{node_id}/train", response_model=Training, status_code=202 +) async def train_node( - name: str, + flow_id: str, node_id: str, request: Request, controller: FlowControllerDep, @@ -1030,9 +1105,10 @@ async def train_node( It learns the node's targets, its ``examples`` and its ``tests``, as the editor shows them. The node keeps running its model until the new one is - set as its ``model``. + set as its ``model``. The run names the node it trains by the flow's id, + ``.``, which is what its panel finds it by. """ - node = _node_def(controller, name, node_id) + node = _node_def(controller, flow_id, node_id) if node.type != "wren": raise HTTPException(status_code=400, detail=f"'{node_id}' is not a wren node") try: @@ -1054,7 +1130,7 @@ async def train_node( cause="train", actor=user.email, no_cache=True, - for_node=f"{name}.{node_id}", + for_node=f"{flow_id}.{node_id}", ) except FlowNotFound: raise HTTPException( @@ -1069,19 +1145,19 @@ async def train_node( return Training(id=run.id, status=run.status) -@router.get("/{name}/nodes/{node_id}/train", response_model=Training | None) +@router.get("/{flow_id}/nodes/{node_id}/train", response_model=Training | None) def read_training( - name: str, + flow_id: str, node_id: str, request: Request, controller: FlowControllerDep, session: SessionDep, ) -> Any: """A wren node's latest training run, or null when it has none.""" - _node_def(controller, name, node_id) + _node_def(controller, flow_id, node_id) run = session.exec( select(Run) - .where(col(Run.flow) == TRAIN_FLOW, col(Run.for_node) == f"{name}.{node_id}") + .where(col(Run.flow) == TRAIN_FLOW, col(Run.for_node) == f"{flow_id}.{node_id}") .order_by(col(Run.created_at).desc()) ).first() if run is None: @@ -1106,27 +1182,27 @@ def read_training( return training -@router.get("/{name}/state", response_model=FlowStatePublic) -def read_flow_state(name: str, controller: FlowControllerDep, hidden: HiddenDep) -> Any: +@router.get("/{flow_id}/state", response_model=FlowStatePublic) +def read_flow_state( + flow_id: str, controller: FlowControllerDep, hidden: HiddenDep +) -> Any: """The last value seen on every message of this flow.""" - _read_flow(controller, name) - return _flow_state(controller, name, hidden) + return _flow_state(controller, _read_flow(controller, flow_id).name, hidden) -@router.get("/{name}/history/{message}", response_model=MessageHistory) +@router.get("/{flow_id}/history/{message}", response_model=MessageHistory) def read_message_history( - name: str, + flow_id: str, message: str, controller: FlowControllerDep, hidden: HiddenDep, ) -> Any: """The recent values of one message, for plotting. - ``message`` may be given bare or qualified; a message that never carried a - number or a flag comes back with an empty series. + ``message`` may be given bare, or qualified with the flow's key; a message + that never carried a number or a flag comes back with an empty series. """ - _read_flow(controller, name) - key = qualify(name, message) + key = qualify(_read_flow(controller, flow_id).name, message) if key in hidden: raise HTTPException(status_code=403, detail=CAMERA_IS_SUPERUSERS) return MessageHistory( diff --git a/backend/fluksio/api/routes/observability.py b/backend/fluksio/api/routes/observability.py index a24db46..c42961d 100644 --- a/backend/fluksio/api/routes/observability.py +++ b/backend/fluksio/api/routes/observability.py @@ -204,7 +204,8 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any: if queue.get("error"): problems.append("work queue unreachable") - names = await run_in_threadpool(controller.store.list_flows) + # Keys, which is what the engine's gates and quarantine go by. + names = list(await run_in_threadpool(controller.store.key_index)) quarantined = controller.quarantined paused = set(controller.paused_flows()) entries = list(controller.loaded.values()) diff --git a/backend/fluksio/flow/controller.py b/backend/fluksio/flow/controller.py index 0f42bcb..288a087 100644 --- a/backend/fluksio/flow/controller.py +++ b/backend/fluksio/flow/controller.py @@ -72,6 +72,7 @@ from fluksio.flow.pipeline import ( ValueSource, ) from fluksio.flow.placement import Placer +from fluksio.flow.queue import WorkItem, owned_by from fluksio.flow.remote import RemoteWorkerHub from fluksio.flow.resources import ( ResourceAccountant, @@ -160,7 +161,10 @@ class LoadedNode: """ id: str + #: The flow's key, which ``id`` is qualified with. flow: str + #: The flow's id, which its store paths and API routes use. + flow_id: str = "" status: NodeStatus = NodeStatus.ACTIVE node: Node | None = None error: str | None = None @@ -556,6 +560,9 @@ class FlowController: self.disabled: set[str] = set() #: Flows that only run when a run asks them to. self.batch: set[str] = set() + #: The key each built flow runs under, by its id. What a rebuild of a + #: flow that is gone takes out, since its document can no longer say. + self._flow_keys: dict[str, str] = {} #: Each flow's declared inputs, kept so rebuilding one flow can #: validate the whole graph without re-reading every flow off the disk. self._flow_inputs: dict[str, dict[str, bool]] = {} @@ -642,15 +649,16 @@ class FlowController: entry.last_error_ts = None entry.failures_in_a_row = 0 - async def set_enabled(self, flow: str, enabled: bool) -> None: + async def set_enabled(self, flow_id: str, flow: str, enabled: bool) -> None: """Stop or start one flow, without building anything. Its nodes are built whether or not it runs — being stopped means having no subscriptions, schedules or webhooks, not being absent — so all this has to do is the lifecycle call and the gate that goes with - it. Nothing is compiled and no other flow is touched. + it. Nothing is compiled and no other flow is touched. *flow_id* is + where that is stored, *flow* the key the gate goes by. """ - await run_in_threadpool(self.store.write_enabled, flow, enabled) + await run_in_threadpool(self.store.write_enabled, flow_id, enabled) if self.pipeline is None: # Nothing built yet, so there are no nodes to start or stop. await self.reload() @@ -726,10 +734,9 @@ class FlowController: self.supervisor = Supervisor(self.events) published = self.store.read_all() + self._flow_keys = {flow.id: flow.name for flow in published} self.disabled = { - flow.name - for flow in published - if not self.store.read_enabled(flow.name) + flow.name for flow in published if not self.store.read_enabled(flow.id) } # A batch flow is built — the canvas draws it and validation covers # it — but never activated: it runs when a run asks it to, and a @@ -795,7 +802,7 @@ class FlowController: self._rewired() async def reload_flow( - self, name: str, wait: float | None = FLOW_REBUILD_WAIT + self, flow_id: str, wait: float | None = FLOW_REBUILD_WAIT ) -> None: """Rebuild one flow, leaving every other flow's nodes connected. @@ -806,7 +813,8 @@ class FlowController: on a populated instance is every node opening its socket again, and only one flow's have changed. - A flow the store no longer has is taken out instead of replaced. + A flow the store no longer has is taken out instead of replaced. The + flow is named by its id; everything about running it goes by its key. The failure shape is the whole-pipeline one narrowed to a flow, and deliberately no better: a build that raises changes nothing, and @@ -826,11 +834,14 @@ class FlowController: # flow is not running is its own teardown and nothing more. Off # the loop for the same reason the full build is: compiling a # python node waits for a worker slot. - built = await run_in_threadpool(self._build_one, name) + built = await run_in_threadpool(self._build_one, flow_id) + # A flow never built has nothing to take out, so no key either. + name = built[0].name if built else self._flow_keys.pop(flow_id, "") await self._teardown(name) if built is None: - pipeline.remove_flow(name) + if name: + pipeline.remove_flow(name) self.loaded = { node_id: entry for node_id, entry in self.loaded.items() @@ -843,6 +854,7 @@ class FlowController: self.batch = self.batch - {name} else: flow, nodes, loaded, initial_values, enabled = built + self._flow_keys[flow_id] = name pipeline.replace_flow(name, nodes, initial_values) # A rebuild is a fresh set of nodes, but not a fresh history: @@ -953,7 +965,7 @@ class FlowController: """ broken = sorted( { - entry.flow + entry.flow_id for entry in self.loaded.values() if entry.status is NodeStatus.ERROR } @@ -963,7 +975,10 @@ class FlowController: return broken async def reload_lib_users(self, ref: str) -> list[str]: - """Rebuild the flows running a shared node whose source has changed.""" + """Rebuild the flows running a shared node whose source has changed. + + Returns their ids, which is what `usages` qualifies with. + """ usages = self.store.usages(ref, draft=False) flows = sorted({usage.split(".", 1)[0] for usage in usages}) for flow in flows: @@ -971,18 +986,18 @@ class FlowController: return flows def _build_one( - self, name: str + self, flow_id: str ) -> tuple[FlowDef, list[Node], dict[str, LoadedNode], dict[str, Any], bool] | None: """Read one published flow and build its nodes. None when it is gone. Blocking: this is the part of a rebuild that talks to the worker pool. """ try: - flow = self.store.read_flow(name) + flow = self.store.read_flow(flow_id) except FlowNotFound: return None nodes, loaded, initial_values, _ = self._build_flows([(flow, False)]) - return flow, nodes, loaded, initial_values, self.store.read_enabled(name) + return flow, nodes, loaded, initial_values, self.store.read_enabled(flow_id) def _all_flow_inputs(self) -> dict[str, bool]: """Every flow's declared inputs, merged the way one build would see them. @@ -1145,7 +1160,7 @@ class FlowController: for flow, draft in flows: for node_def in flow.nodes: - entry = self._build_node(flow.name, node_def, draft=draft, run=run) + entry = self._build_node(flow, node_def, draft=draft, run=run) loaded[entry.id] = entry if entry.node is not None: nodes.append(entry.node) @@ -1157,13 +1172,16 @@ class FlowController: def _build_node( self, - flow: str, + flow_def: FlowDef, node_def: NodeDef, draft: bool = False, run: RunContext | None = None, ) -> LoadedNode: + """One node of *flow_def*: its code read by the flow's id, and run, + named and wired by its key.""" + flow, flow_id = flow_def.name, flow_def.id node_id = f"{flow}.{node_def.id}" - entry = LoadedNode(id=node_id, flow=flow) + entry = LoadedNode(id=node_id, flow=flow, flow_id=flow_id) try: node_type = NODE_TYPES.get(node_def.type) if node_type is None: @@ -1192,19 +1210,21 @@ class FlowController: code = self.store.read_lib_source(node_def.source_ref) else: owner, local = flow, node_def.id - code = self.store.read_node_source(flow, node_def.id, draft=draft) + code = self.store.read_node_source( + flow_id, node_def.id, draft=draft + ) # The store answers the new-node template when nothing was # written, which is what an editor should open with and not # something to run. A draft legitimately has none yet. entry.missing_source = not draft and not self.store.has_node_source( - flow, node_def.id + flow_id, node_def.id ) # Only a body somebody wrote: the template a new node opens # with takes nothing, and a node declared before it is written # is reported as missing its source rather than as wrong. written = bool(node_def.source_ref) or self.store.has_node_source( - flow, node_def.id, draft=draft + flow_id, node_def.id, draft=draft ) if written: gap = _signature_gap( @@ -1376,8 +1396,10 @@ class FlowController: node._on_health = self._health_changed if _is_hook(node): - # Hooks live under their flow, away from the JSON API. - node.url = f"{HOOK_PREFIX}/{flow}/{node.url.lstrip('/')}" + # Hooks live under their flow, away from the JSON API: under + # its id, so a URL handed out keeps working when the key + # changes. + node.url = f"{HOOK_PREFIX}/{flow_id}/{node.url.lstrip('/')}" entry.node = node except (SecretNotFound, LibNotFound) as exc: @@ -1493,7 +1515,7 @@ class FlowController: issue for issue in self.issues if not issue.flow or issue.flow == flow ] + self._health_issues(flow) - def preview(self, name: str) -> Preview: + def preview(self, flow_id: str) -> Preview: """Build a flow's unpublished draft without deploying it. The draft stands in for its published self among the other flows, so @@ -1501,13 +1523,15 @@ class FlowController: graph is thrown away afterwards and seeded into a throwaway state, so nothing here touches what is running. """ - flows: list[tuple[FlowDef, bool]] = [ - (flow, False) for flow in self.store.read_all() if flow.name != name - ] try: - flows.append((self.store.read_flow(name, draft=True), True)) + draft = self.store.read_flow(flow_id, draft=True) except FlowNotFound: return Preview(nodes=[], issues=[]) + name = draft.name + flows: list[tuple[FlowDef, bool]] = [ + (flow, False) for flow in self.store.read_all() if flow.id != flow_id + ] + flows.append((draft, True)) nodes, loaded, initial_values, flow_inputs = self._build_flows(flows) pipeline = Pipeline( @@ -1856,21 +1880,35 @@ class FlowController: return self.execution.stats() if self.execution is not None else {} def forget_flow(self, flow: str) -> None: - """Drop what a deleted flow left behind. Blocking.""" + """Drop what a deleted flow left behind, by its key. Blocking. + + After its nodes are torn down, or they put some of it straight back. + """ if self.execution is not None: self.execution.queue.clear_flow(flow) - prefix = f"{flow}." with self.state.lock(): - stale = [ - key - for key in self.state.keys() - # Both the messages themselves and the engine's own bookkeeping - # about them, which is keyed by message name too. - if key.startswith(prefix) or f":{prefix}" in key - ] - for key in stale: + for key in [k for k in self.state.keys() if _in_namespace(k, flow)]: self.state.delete(key) + def key_residue(self, flow: str) -> list[str]: + """What still lives under the flow key *flow*. Blocking. + + State and live queue work in its namespace: left by a flow that held + the key before, or put there by anything still writing to it. A new + flow taking the key would inherit it, so creating one is refused while + this is not empty, and so is renaming a flow onto it. Work that was + acknowledged, dead-lettered or marked done is history and not counted. + Names what it found, state keys first. + """ + found = [key for key in self.state.keys() if _in_namespace(key, flow)] + if self.execution is not None: + found += [ + f"queued work of '{item.node}'" + for item in self.execution.queue.live_items() + if _touches(item, flow) + ] + return found + def run_flow(self, flow: str, inputs: dict[str, Any] | None = None) -> None: """Run every node of one flow. Blocking — call from a worker thread.""" if self.pipeline is None: @@ -1912,7 +1950,7 @@ class FlowController: pipeline.history_limits = self.history_limits return pipeline - def run_preview(self, flow_name: str, inputs: dict[str, Any] | None = None) -> None: + def run_preview(self, flow_id: str, inputs: dict[str, Any] | None = None) -> None: """Run a flow's unpublished draft once, as the editor shows it. The nodes are built and run but never activated, so a draft never @@ -1920,7 +1958,7 @@ class FlowController: that. State and events are the live ones, so values land on the canvas exactly as they do for a published run. """ - flow = self.store.read_flow(flow_name, draft=True) + flow = self.store.read_flow(flow_id, draft=True) nodes, _, initial_values, _ = self._build_flows([(flow, True)]) pipeline = Pipeline( nodes=nodes, @@ -1960,6 +1998,20 @@ class FlowController: self.events.publish(event) +def _in_namespace(state_key: str, flow: str) -> bool: + """Whether a state key is about the flow key *flow*: one of its messages, + or the engine's own bookkeeping on one of its messages or nodes, which is + keyed by those names too.""" + prefix = f"{flow}." + return state_key.startswith(prefix) or f":{prefix}" in state_key + + +def _touches(item: WorkItem, flow: str) -> bool: + """Whether a queued item names anything of the flow key *flow*: as its + flow, its node or one of the messages it carries.""" + return owned_by(item, flow) or any(flow_of(name) == flow for name in item.outputs) + + def _bound(specs: list[MessageSpec]) -> list[MessageSpec]: """Ports without a message name are not wired into the graph.""" return [spec for spec in specs if spec.name] diff --git a/backend/fluksio/flow/dashboards.py b/backend/fluksio/flow/dashboards.py index 21ee215..7b7521d 100644 --- a/backend/fluksio/flow/dashboards.py +++ b/backend/fluksio/flow/dashboards.py @@ -28,11 +28,15 @@ from fluksio.flow.messages import DType, qualify from fluksio.flow.schemas import FlowDef, _validate_id, _validate_name from fluksio.flow.store import ( FlowStore, + IdTaken, StaleVersion, _updated_at, + _version, checked, claim, copy_name, + named, + new_id, read_keys, ) @@ -619,6 +623,27 @@ class DashboardDef(BaseModel): return shapes +class DashboardCreate(BaseModel): + """A new dashboard: one empty grid under a key, filled in by saves after.""" + + #: Empty for an id the server issues, or one a superuser pins, as for a + #: flow (`FlowCreate`). Dashboards keep no history, so it only has to be free. + id: str = "" + name: str + #: Empty reads the key as words. + title: str = "" + + @field_validator("id") + @classmethod + def _check_id(cls, value: str) -> str: + return _validate_id(value) + + @field_validator("name") + @classmethod + def _check_name(cls, value: str) -> str: + return _validate_name(value) + + class DashboardSummary(BaseModel): """A dashboard in a list, without its contents.""" @@ -766,14 +791,17 @@ class DashboardStore: self._keys = (at, keys) return keys - def _claim(self, defn: DashboardDef) -> DashboardDef: + def _claim(self, defn: DashboardDef, create: bool = False) -> DashboardDef: """*defn* with its id, once it is certain to be a save the store allows. - Under the flow store's write lock; see `claim`. An id left out is the - key, which is how every dashboard is created for now. + Under the flow store's write lock; see `claim`. A save is to a + dashboard that exists, named by its id or, from before ids, its key. + Only *create* may make a new one. """ dash_id = checked(defn.id or defn.name) current = self.read(dash_id, draft=True) if self.exists(dash_id) else None + if current is None and not create: + raise DashboardNotFound(dash_id) claim( dash_id, defn.name, @@ -807,21 +835,50 @@ class DashboardStore: """Publish a dashboard directly, skipping the draft. The API never does: it creates a draft and promotes it. This is for a - caller that already has the finished document — a test, or a seed. + caller that already has the finished document, a test or a seed, and + creates one too, under the id it carries or else its key. """ with self._lock, self.flows._write_lock: - defn = self._claim(defn) + defn = self._claim(defn, create=True) path = self._file(defn.id) - current = 0 - if path.exists(): - current = DashboardDef.model_validate_json(path.read_text()).version + current = _version(path) if base_version is not None and base_version != current: raise StaleVersion(defn.name, current) - saved = defn.model_copy(update={"version": current + 1, "has_draft": False}) + # Above the draft too, which an editor may be holding. + top = max(current, _version(self._draft_file(defn.id))) + saved = defn.model_copy(update={"version": top + 1, "has_draft": False}) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(self._dump(saved)) - self.flows._commit(f"Save dashboard '{defn.name}'") + self.flows._commit(f"Save dashboard {named(defn.name, defn.id)}") + return saved + + def create( + self, defn: DashboardDef, pin: str = "", published: bool = False + ) -> DashboardDef: + """A new dashboard, only a draft unless *published* says otherwise. + + Its id is issued here unless *pin* names one, a superuser's to ask for + as for a flow (`FlowStore.create`). Dashboards have no history to + adopt, so a pin only has to be free. + """ + with self._lock, self.flows._write_lock: + keys = self.key_index() + if pin: + dash_id = checked(pin) + if (self.root / dash_id).exists(): + raise IdTaken(dash_id, "dashboard") + else: + dash_id = new_id("d", lambda c: (self.root / c).exists() or c in keys) + defn = self._claim(defn.model_copy(update={"id": dash_id}), create=True) + if keys.get(dash_id, dash_id) != dash_id: + raise IdTaken(dash_id, "dashboard") + + saved = defn.model_copy(update={"version": 1, "has_draft": not published}) + path = self._file(dash_id) if published else self._draft_file(dash_id) + path.parent.mkdir(parents=True) + path.write_text(self._dump(saved)) + self.flows._commit(f"Create dashboard {named(defn.name, dash_id)}") return saved def write_draft( @@ -829,23 +886,19 @@ class DashboardStore: ) -> DashboardDef: """Save unpublished changes, refusing to overwrite someone else's. - ``base_version`` is the version the editor last saw — of the working - copy, which is the draft once there is one, and 0 for a dashboard that - does not exist yet: creating one is its first draft. + ``base_version`` is the version the editor last saw, of the working + copy, which is the draft once there is one. The dashboard has to + exist: creating one is `create`. """ with self._lock, self.flows._write_lock: defn = self._claim(defn) - current = 0 - if self.exists(defn.id): - current = self.read(defn.id, draft=True).version + current = self.read(defn.id, draft=True).version if base_version is not None and base_version != current: raise StaleVersion(defn.name, current) saved = defn.model_copy(update={"version": current + 1, "has_draft": True}) - path = self._draft_file(defn.id) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(self._dump(saved)) - self.flows._commit(f"Update draft of dashboard '{defn.name}'") + self._draft_file(defn.id).write_text(self._dump(saved)) + self.flows._commit(f"Update draft of dashboard {named(defn.name, defn.id)}") return saved def publish(self, name: str, base_version: int | None = None) -> DashboardDef: @@ -859,14 +912,24 @@ class DashboardStore: if draft.exists(): self._file(name).write_text(self._dump(current)) draft.unlink() - self.flows._commit(f"Publish dashboard '{name}'") + self.flows._commit(f"Publish dashboard {named(current.name, name)}") return current.model_copy(update={"has_draft": False}) def discard_draft(self, name: str) -> DashboardDef: - """Throw the unpublished changes away and go back to what is shown.""" + """Throw the unpublished changes away and go back to what is shown. + + What is shown comes back a version above the draft, as a flow's does + (`FlowStore.discard_draft`). + """ with self._lock, self.flows._write_lock: draft = self._draft_file(name) if draft.exists(): + shown = self.read(name) + top = _version(draft) + if shown.version <= top: + self._file(name).write_text( + self._dump(shown.model_copy(update={"version": top + 1})) + ) draft.unlink() self.flows._commit(f"Discard draft of dashboard '{name}'") return self.read(name) @@ -887,27 +950,32 @@ class DashboardStore: def duplicate(self, name: str) -> DashboardDef: """Copy the working copy into a new dashboard, as a draft only. - Named ``{name}_copy``, then ``_copy_2`` and on. Unpublished like a new - dashboard, so no panel shows it until someone publishes it. + Under an id of its own and the key ``{name}_copy``, then ``_copy_2`` + and on. Unpublished like a new dashboard, so no panel shows it until + someone publishes it. """ with self._lock, self.flows._write_lock: source = self.read(name, draft=True) - # The copy's id is its key, so it has to be free as both. keys = self.key_index() - new_name = copy_name(source.name, lambda n: self.exists(n) or n in keys) + new_name = copy_name(source.name, lambda n: n in keys) + copy_id = new_id("d", lambda c: (self.root / c).exists() or c in keys) copy = source.model_copy( update={ - "id": new_name, + "id": copy_id, "name": new_name, "title": f"{source.title} (copy)" if source.title else "", "version": 1, "has_draft": True, } ) - path = self._draft_file(new_name) - path.parent.mkdir(parents=True, exist_ok=True) + copy = self._claim(copy, create=True) + path = self._draft_file(copy_id) + path.parent.mkdir(parents=True) path.write_text(self._dump(copy)) - self.flows._commit(f"Duplicate dashboard '{name}' as '{new_name}'") + self.flows._commit( + f"Duplicate dashboard {named(source.name, name)} as " + f"{named(new_name, copy_id)}" + ) return copy def bindings_for(self, flow: str) -> Bindings: @@ -984,11 +1052,11 @@ class DashboardStore: return limits -def default_dashboard(name: str) -> DashboardDef: - """A new dashboard: one page, one section, nothing in it yet.""" +def default_dashboard(name: str, title: str = "") -> DashboardDef: + """A new dashboard: one empty grid, titled after its key unless told.""" return DashboardDef( name=name, - title=name.replace("_", " ").capitalize(), + title=title or name.replace("_", " ").capitalize(), ) @@ -1094,6 +1162,7 @@ __all__ = [ "INPUT_WIDGETS", "SETTING_DTYPES", "WIDGET_DTYPES", + "DashboardCreate", "DashboardDef", "DashboardExists", "DashboardNotFound", diff --git a/backend/fluksio/flow/devices.py b/backend/fluksio/flow/devices.py index cafef27..3012406 100644 --- a/backend/fluksio/flow/devices.py +++ b/backend/fluksio/flow/devices.py @@ -445,7 +445,7 @@ class DeviceStore: if self._wiring_at != revision: found: dict[str, list[Binding]] = {} for flow in self.flows.read_all(): - if not self.flows.read_enabled(flow.name): + if not self.flows.read_enabled(flow.id): continue for binding in bindings_of(flow): found.setdefault(binding.device, []).append(binding) diff --git a/backend/fluksio/flow/metrics.py b/backend/fluksio/flow/metrics.py index 88b8c61..719c54c 100644 --- a/backend/fluksio/flow/metrics.py +++ b/backend/fluksio/flow/metrics.py @@ -24,6 +24,7 @@ from sqlmodel import Session, col from fluksio.core.config import settings from fluksio.core.db import engine +from fluksio.flow import tombstones from fluksio.flow.events import EventBus from fluksio.models import EngineEvent, FlowRun, MetricBucket @@ -456,6 +457,9 @@ class MetricsCollector: session.execute(delete(MetricBucket).where(col(MetricBucket.bucket) < cutoff)) session.execute(delete(EngineEvent).where(col(EngineEvent.ts) < cutoff)) session.execute(delete(FlowRun).where(col(FlowRun.started_at) < cutoff)) + # After the rows above: a deleted flow's name is kept as long as any + # of its history is. + tombstones.prune(session) # A cascade still open long after it started did not finish; saying so # is more honest than leaving it running forever. A batch run is not # swept: it is allowed to take hours, and `Run` has its own lease sweep diff --git a/backend/fluksio/flow/nodes/http.py b/backend/fluksio/flow/nodes/http.py index 0877cf1..0205ae8 100644 --- a/backend/fluksio/flow/nodes/http.py +++ b/backend/fluksio/flow/nodes/http.py @@ -161,7 +161,7 @@ class HttpNode(Node): url: str = Field( description=( - "A path such as /tick, served as a webhook at /hooks//, " + "A path such as /tick, served as a webhook at /hooks//, " "or the full URL a sender requests." ) ) @@ -197,7 +197,7 @@ class HttpNode(Node): default="", description=( "Shared secret for a webhook, appended to its URL: " - "/hooks///. Empty leaves the webhook open " + "/hooks///. Empty leaves the webhook open " "to anyone." ), json_schema_extra={"x-secret": True}, diff --git a/backend/fluksio/flow/nodes/websocket.py b/backend/fluksio/flow/nodes/websocket.py index dfdbc0b..26adbf4 100644 --- a/backend/fluksio/flow/nodes/websocket.py +++ b/backend/fluksio/flow/nodes/websocket.py @@ -31,7 +31,7 @@ class WebSocketNode(Node): The URL picks the role: ``ws://`` or ``wss://`` dials out and holds the connection, a path such as ``/ticker`` is served at - ``/hooks//[/]`` while the flow runs. The ports pick the + ``/hooks//[/]`` while the flow runs. The ports pick the direction, as on the mqtt node: outputs only make a **listener**, inputs a **sender**. @@ -79,7 +79,7 @@ class WebSocketNode(Node): url: str = Field( description=( "ws:// or wss:// to connect to a server, or a path such as " - "/ticker to serve one at /hooks//." + "/ticker to serve one at /hooks//." ) ) headers: dict[str, str] = Field( @@ -100,7 +100,7 @@ class WebSocketNode(Node): default="", description=( "Shared secret for a served socket, appended to its URL: " - "/hooks///. Empty leaves it open to anyone." + "/hooks///. Empty leaves it open to anyone." ), json_schema_extra={"x-secret": True}, ) diff --git a/backend/fluksio/flow/queue.py b/backend/fluksio/flow/queue.py index 1b3a605..cb68b82 100644 --- a/backend/fluksio/flow/queue.py +++ b/backend/fluksio/flow/queue.py @@ -151,6 +151,17 @@ class WorkItem: ) +def owned_by(item: WorkItem, flow: str) -> bool: + """Whether *item* is work of the flow with key *flow*, not one reaching it. + + A run item names its run's flow by identity rather than by key, and a + run is never removed with a flow's queued work, so it is nobody's here. + """ + return item.kind != "run" and ( + item.flow == flow or item.node.split(".", 1)[0] == flow + ) + + class WorkQueue(ABC): """What the execution service pulls from.""" @@ -236,7 +247,21 @@ class WorkQueue(ABC): @abstractmethod def clear_flow(self, flow: str) -> None: - """Forget anything held for a flow that no longer exists.""" + """Drop the work of a flow that no longer exists (`owned_by`). + + What it parked, what waits on a timer, and what is queued or claimed + and not yet acknowledged, so a new flow under the same key never runs + it. A cascade already running finishes; its acknowledgement then has + nothing left to remove. + """ + + @abstractmethod + def live_items(self) -> list[WorkItem]: + """Every item still to run: queued, claimed, delayed or parked. + + Not what was acknowledged, dead-lettered or marked done, which is + history. Reads the whole queue, so it is for rare questions. + """ @abstractmethod def stats(self) -> dict[str, Any]: @@ -388,6 +413,23 @@ class MemoryWorkQueue(WorkQueue): def clear_flow(self, flow: str) -> None: with self._lock: self._parked.pop(flow, None) + for lane in (self._items, self._due): + kept = [item for item in lane if not owned_by(item, flow)] + lane.clear() + lane.extend(kept) + self._delayed = [e for e in self._delayed if not owned_by(e[2], flow)] + heapq.heapify(self._delayed) + + def live_items(self) -> list[WorkItem]: + """What is waiting. A claimed item is gone from here once it is handed + out, so unlike Redis this cannot say what is running.""" + with self._lock: + return [ + *self._items, + *self._due, + *(entry[2] for entry in self._delayed), + *(item for held in self._parked.values() for item in held), + ] def backlog(self) -> int: with self._lock: @@ -683,8 +725,68 @@ class RedisWorkQueue(WorkQueue): pipe = self._redis.pipeline() pipe.delete(self._parked_key(flow)) pipe.srem(self._parked_flows_key, flow) + # Timers before the streams: one promoted in between is in the due + # stream by the time that is read. + for raw in cast(list[str], self._redis.zrange(self._delayed_key, 0, -1)): + if owned_by(WorkItem.from_fields(orjson.loads(raw), ""), flow): + pipe.zrem(self._delayed_key, raw) + for stream in (self._stream, self._due_stream): + ids = [ + entry_id + for entry_id, fields in self._live_entries(stream) + if owned_by(WorkItem.from_fields(fields, entry_id), flow) + ] + if ids: + pipe.xack(stream, GROUP, *ids) + pipe.xdel(stream, *ids) pipe.execute() + def _live_entries(self, stream: str) -> list[tuple[str, dict[str, str]]]: + """The entries of *stream* still to run: claimed and unacknowledged, + then never handed out. Acknowledged ones stay in a stream until it is + trimmed, and are history.""" + group = self._group_info(stream) + if not group: + return [] + last = str(group.get("last-delivered-id") or "0-0") + live: list[tuple[str, dict[str, str]]] = [] + pending = int(group.get("pending") or 0) + if pending: + rows = cast( + list[dict[str, Any]], + self._redis.xpending_range( + stream, GROUP, min="-", max="+", count=pending + ), + ) + ids = {row["message_id"] for row in rows} + if ids: + # In id order, so the first is the oldest still held. + entries = self._redis.xrange( + stream, min=rows[0]["message_id"], max=last + ) + live += [(i, f) for i, f in cast(Any, entries) if i in ids] + live += cast(Any, self._redis.xrange(stream, min=f"({last}", max="+")) + return live + + def live_items(self) -> list[WorkItem]: + items = [ + WorkItem.from_fields(fields, prefix + entry_id) + for stream, prefix in ((self._due_stream, DUE_PREFIX), (self._stream, "")) + for entry_id, fields in self._live_entries(stream) + ] + items += [ + WorkItem.from_fields(orjson.loads(raw), "") + for raw in cast(list[str], self._redis.zrange(self._delayed_key, 0, -1)) + ] + for flow in cast(set[str], self._redis.smembers(self._parked_flows_key)): + items += [ + WorkItem.from_fields(orjson.loads(raw), "") + for raw in cast( + list[str], self._redis.lrange(self._parked_key(flow), 0, -1) + ) + ] + return items + def _group_info(self, stream: str = "") -> dict[str, Any]: """This consumer group's row of ``XINFO GROUPS``, or an empty one.""" try: diff --git a/backend/fluksio/flow/runs.py b/backend/fluksio/flow/runs.py index 31ecee0..5b98a04 100644 --- a/backend/fluksio/flow/runs.py +++ b/backend/fluksio/flow/runs.py @@ -155,7 +155,7 @@ def batch_issues( "the value would be dropped. Remove the interval, or mark " "the port as streaming if it is a curve being thinned out." ) - if store is not None and _has_no_body(store, flow.name, node, draft): + if store is not None and _has_no_body(store, flow.id, node, draft): issues.append( f"Node '{node.id}' has no stored code. It would run as an " "empty node and publish nothing, so the run is refused." diff --git a/backend/fluksio/flow/schemas.py b/backend/fluksio/flow/schemas.py index c345de8..63ba469 100644 --- a/backend/fluksio/flow/schemas.py +++ b/backend/fluksio/flow/schemas.py @@ -352,6 +352,36 @@ class FlowDef(BaseModel): return _validate_name(value) +class FlowCreate(BaseModel): + """A new flow as a client hands it over: a document but for what the + server owns. The fields mean what they do on `FlowDef`, but ``id``.""" + + #: Empty for an id the server issues. Anything else pins it, which only a + #: superuser may: an id that is free, or whose history this flow resumes. + id: str = "" + name: str + title: str = "" + nodes: list[NodeDef] = Field(default_factory=list) + inputs: list[FlowInput] = Field(default_factory=list) + mode: Literal["live", "batch"] = "live" + outputs: list[str] = Field(default_factory=list) + origin: FlowOrigin | None = None + + @field_validator("id") + @classmethod + def _check_id(cls, value: str) -> str: + return _validate_id(value) + + @field_validator("name") + @classmethod + def _check_name(cls, value: str) -> str: + return _validate_name(value) + + def document(self) -> FlowDef: + """The flow to store, before the store gives it an id.""" + return FlowDef.model_validate(self.model_dump(exclude={"id"})) + + class NodeSource(BaseModel): """The Python source of a node.""" diff --git a/backend/fluksio/flow/store.py b/backend/fluksio/flow/store.py index 9b75fe2..3e0f445 100644 --- a/backend/fluksio/flow/store.py +++ b/backend/fluksio/flow/store.py @@ -14,8 +14,9 @@ unpublished — which is what every flow written before this existed looks like. A flow is two names. Its id is which flow it is: the directory, the argument every method here takes, and never changed once given. Its key, ``name``, is the namespace of its messages and unique among flows. A flow from before ids -existed has its key as its id (`fluksio.flow.identity`), and so does every new -one for now. Published and draft documents always share both. +existed has its key as its id (`fluksio.flow.identity`). A new one is given an +opaque id by `create`, or the one a superuser pinned. Published and draft +documents always share both. """ from __future__ import annotations @@ -31,10 +32,13 @@ import threading from collections.abc import Callable, Iterable from datetime import UTC, datetime from pathlib import Path -from typing import Literal +from typing import TYPE_CHECKING, Literal from fluksio.flow.schemas import FlowDef, _validate_name +if TYPE_CHECKING: + from fluksio.flow.tombstones import Tombstones + logger = logging.getLogger(__name__) #: Where the sources of nodes shared between flows live. @@ -97,6 +101,37 @@ class IdentityChange(ValueError): """ +class RecordedKey(IdentityChange): + """A pinned id restored under another key than its history is filed under. + + The history names the old key's messages, so taking a new key with it is + a rename, not a create. + """ + + def __init__(self, obj_id: str, key: str) -> None: + super().__init__(obj_id) + self.id = obj_id + self.key = key + + def __str__(self) -> str: + return ( + f"'{self.id}' has history filed under the key '{self.key}', so it " + "can only be restored with that key" + ) + + +class IdTaken(ValueError): + """A pinned id that is in use: an object's, a directory's or another key.""" + + def __init__(self, obj_id: str, kind: str = "flow") -> None: + super().__init__(obj_id) + self.id = obj_id + self.kind = kind + + def __str__(self) -> str: + return f"The id '{self.id}' is already taken by a {self.kind}" + + def claim( obj_id: str, key: str, @@ -146,6 +181,11 @@ def read_keys(directories: Iterable[Path], document: str) -> dict[str, str]: return keys +def named(key: str, obj_id: str) -> str: + """How a commit message names an object: by key, and by id when it differs.""" + return f"'{key}'" if key == obj_id else f"'{key}' ({obj_id})" + + def new_id(kind: Literal["f", "d"], taken: Callable[[str], bool]) -> str: """A fresh id for a flow (``f``) or a dashboard (``d``), e.g. ``f_k3q7m2xa``. @@ -222,6 +262,14 @@ def _updated_at(paths: Iterable[Path]) -> datetime | None: return datetime.fromtimestamp(max(stamps), UTC) if stamps else None +def _version(path: Path) -> int: + """The version a stored document carries, 0 for none or an unreadable one.""" + try: + return int(json.loads(path.read_text()).get("version", 1)) + except (OSError, ValueError, TypeError, AttributeError): + return 0 + + def _same_content(left: FlowDef, right: FlowDef) -> bool: """Equal but for the version counter, which the server owns.""" return left.model_copy(update={"version": 0}) == right.model_copy( @@ -252,8 +300,11 @@ class FlowStore: #: a store reached 1898 commits, 5863 loose objects and no pack at all. _GC_EVERY = 500 - def __init__(self, root: Path) -> None: + def __init__(self, root: Path, tombstones: Tombstones | None = None) -> None: self.root = root + #: Where a deleted flow's key is kept, and what history an id has. + #: None leaves both out: a store without a database, as in tests. + self.tombstones = tombstones self.root.mkdir(parents=True, exist_ok=True) # Draft writes are check-and-set, so two clients saving at once must not # interleave between reading the current version and writing the next. @@ -679,14 +730,16 @@ class FlowStore: self._keys = (at, keys) return keys - def _claim(self, flow: FlowDef) -> FlowDef: + def _claim(self, flow: FlowDef, create: bool = False) -> FlowDef: """*flow* with its id, once it is certain to be a save the store allows. - Under `_write_lock`. An id left out is the key, which is how every - flow is created for now. + Under `_write_lock`. A save is to a flow that exists, named by its id + or, from before ids, by its key. Only *create* may make a new one. """ flow_id = checked(flow.id or flow.name) current = self.read_flow(flow_id, draft=True) if self.exists(flow_id) else None + if current is None and not create: + raise FlowNotFound(flow_id) claim( flow_id, flow.name, @@ -696,6 +749,22 @@ class FlowStore: ) return flow.model_copy(update={"id": flow_id}) + def _issue(self, keys: dict[str, str]) -> str: + """A fresh flow id. Under `_write_lock`, with *keys* the key index. + + Free as a directory, a key and in history: an id a deleted flow's runs + are still filed under would hand them to the new one. + """ + graves = self.tombstones + return new_id( + "f", + lambda c: ( + self._flow_dir(c).exists() + or c in keys + or (graves is not None and graves.recorded(c) is not None) + ), + ) + def read_all(self) -> list[FlowDef]: """Every published flow — what the engine runs. @@ -718,9 +787,14 @@ class FlowStore: return list(self._read_all) def write_flow(self, flow: FlowDef) -> bool: - """Publish a flow directly. Returns False when nothing actually changed.""" + """Publish a flow directly. Returns False when nothing actually changed. + + Creates one too, under the id it carries or else its key: what a test + or a seed holding a finished document wants. The API creates only + through `create`. + """ with self._write_lock: - flow = self._claim(flow) + flow = self._claim(flow, create=True) path = self._flow_file(flow.id) content = flow.model_dump_json(indent=2) + "\n" if path.exists() and path.read_text() == content: @@ -728,122 +802,188 @@ class FlowStore: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content) - self._commit(f"Update flow '{flow.name}'") + self._commit(f"Update flow {named(flow.name, flow.id)}") return True + def create(self, flow: FlowDef, pin: str = "") -> FlowDef: + """A new flow, only a draft until someone publishes it. + + Its id is issued here unless *pin* names one, which is a superuser's + to ask for: the Demo, a test, a restore. A pinned id whose history + outlived its flow adopts that history, so it has to take the key the + history is filed under, and its tombstone goes once the document is on + disk. + """ + with self._write_lock: + keys = self.key_index() + if pin: + flow_id = checked(pin) + # The directory rather than `exists`: anything left in it would + # become the new flow's. + if self._flow_dir(flow_id).exists(): + raise IdTaken(flow_id) + recorded = ( + self.tombstones.recorded(flow_id) if self.tombstones else None + ) + if recorded is not None and recorded != flow.name: + raise RecordedKey(flow_id, recorded) + else: + flow_id = self._issue(keys) + flow = self._claim(flow.model_copy(update={"id": flow_id}), create=True) + # Another flow's key as this one's id would make naming either + # ambiguous. Checked after the key, which is the likelier mistake. + if keys.get(flow_id, flow_id) != flow_id: + raise IdTaken(flow_id) + + stored = flow.model_copy(update={"version": 1}) + path = self._draft_file(flow_id) + path.parent.mkdir(parents=True) + path.write_text(stored.model_dump_json(indent=2) + "\n") + self._commit(f"Create flow {named(flow.name, flow_id)}") + if pin and self.tombstones is not None: + self.tombstones.lift(flow_id) + return stored + def write_draft(self, flow: FlowDef, base_version: int) -> FlowDef: """Save unpublished changes, refusing to overwrite someone else's. ``base_version`` is the version the editor last saw. It has to match what is stored, otherwise another client saved in between and this - write would silently discard their work. + write would silently discard their work. The flow has to exist: + creating one is `create`. """ with self._write_lock: flow = self._claim(flow) - current: FlowDef | None = None - if self.exists(flow.id): - current = self.read_flow(flow.id, draft=True) - if base_version != current.version: - raise StaleVersion(flow.name, current.version) - if _same_content(current, flow): - return current + current = self.read_flow(flow.id, draft=True) + if base_version != current.version: + raise StaleVersion(flow.name, current.version) + if _same_content(current, flow): + return current - stored = flow.model_copy( - update={"version": (current.version if current else 0) + 1} - ) + stored = flow.model_copy(update={"version": current.version + 1}) path = self._draft_file(flow.id) - path.parent.mkdir(parents=True, exist_ok=True) path.write_text(stored.model_dump_json(indent=2) + "\n") - self._commit(f"Update draft of flow '{flow.name}'") + self._commit(f"Update draft of flow {named(flow.name, flow.id)}") return stored - def publish_flow(self, name: str, base_version: int) -> FlowDef: + def publish_flow(self, flow_id: str, base_version: int) -> FlowDef: """Promote the working copy to what the engine runs.""" with self._write_lock: - current = self.read_flow(name, draft=True) + current = self.read_flow(flow_id, draft=True) if base_version != current.version: - raise StaleVersion(name, current.version) + raise StaleVersion(current.name, current.version) - draft = self._draft_file(name) + draft = self._draft_file(flow_id) if draft.exists(): - self._flow_file(name).write_text( + self._flow_file(flow_id).write_text( current.model_dump_json(indent=2) + "\n" ) draft.unlink() - drafts = self._draft_nodes_dir(name) + drafts = self._draft_nodes_dir(flow_id) if drafts.exists(): - target = self._flow_dir(name) / "nodes" + target = self._flow_dir(flow_id) / "nodes" target.mkdir(parents=True, exist_ok=True) for source in sorted(drafts.glob("*.py")): shutil.copy2(source, target / source.name) shutil.rmtree(drafts) - self._commit(f"Publish flow '{name}'") + self._commit(f"Publish flow {named(current.name, flow_id)}") return current - def discard_draft(self, name: str) -> FlowDef: - """Throw the unpublished changes away and go back to what is running.""" - with self._write_lock: - draft = self._draft_file(name) - if draft.exists(): - draft.unlink() - drafts = self._draft_nodes_dir(name) - if drafts.exists(): - shutil.rmtree(drafts) - self._commit(f"Discard draft of flow '{name}'") - return self.read_flow(name) + def discard_draft(self, flow_id: str) -> FlowDef: + """Throw the unpublished changes away and go back to what is running. - def delete_flow(self, name: str) -> None: - with self._write_lock: - directory = self._flow_dir(name) - if not directory.exists(): - raise FlowNotFound(name) - shutil.rmtree(directory) - self._commit(f"Delete flow '{name}'") - - def duplicate_flow(self, name: str) -> FlowDef: - """Copy a flow's working copy into a new flow that is only a draft. - - Like a new flow, the copy runs nothing until someone publishes it. It is - named ``{name}_copy``, then ``_copy_2`` and on. ``origin`` is dropped, - since the copy is a local flow ``fluksio sync`` must not claim, and - ``runtime.json`` stays behind with everything else about running. + What is running comes back a version above the draft, so an editor + still holding the draft is refused rather than matching the next save + made from the published copy. """ with self._write_lock: - source = self.read_flow(name, draft=True) - # The directory rather than `exists`: anything left in it would - # otherwise end up in the copy. The copy's id is its key, so that - # has to be free among the keys too. + draft = self._draft_file(flow_id) + if draft.exists(): + running = self.read_flow(flow_id) + top = _version(draft) + if running.version <= top: + running.version = top + 1 + self._flow_file(flow_id).write_text( + running.model_dump_json(indent=2) + "\n" + ) + draft.unlink() + drafts = self._draft_nodes_dir(flow_id) + if drafts.exists(): + shutil.rmtree(drafts) + published = self.read_flow(flow_id) + self._commit(f"Discard draft of flow {named(published.name, flow_id)}") + return published + + def delete_flow(self, flow_id: str) -> str: + """Remove a flow's files. Returns the key it had. + + The key is buried first, so a delete that stops half-way never leaves + history without a name, and the live document wins while it is there. + """ + with self._write_lock: + directory = self._flow_dir(flow_id) + if not directory.exists(): + raise FlowNotFound(flow_id) + key = next( + (k for k, i in self.key_index().items() if i == flow_id), flow_id + ) + if self.tombstones is not None: + self.tombstones.bury(flow_id, key) + shutil.rmtree(directory) + self._commit(f"Delete flow {named(key, flow_id)}") + return key + + def duplicate_flow( + self, flow_id: str, busy: Callable[[str], bool] | None = None + ) -> FlowDef: + """Copy a flow's working copy into a new flow that is only a draft. + + Like a new flow, the copy runs nothing until someone publishes it. It + gets an id of its own and the key ``{name}_copy``, then ``_copy_2`` and + on, past any key *busy* says something still holds. ``origin`` is + dropped, since the copy is a local flow ``fluksio sync`` must not + claim, and ``runtime.json`` stays behind with everything else about + running. + """ + with self._write_lock: + source = self.read_flow(flow_id, draft=True) keys = self.key_index() new_name = copy_name( - source.name, lambda n: self._flow_dir(n).exists() or n in keys + source.name, lambda n: n in keys or (busy is not None and busy(n)) ) - copy = source.model_copy( update={ - "id": new_name, + "id": self._issue(keys), "name": new_name, "title": f"{source.title} (copy)" if source.title else "", "version": 1, "origin": None, } ) + copy = self._claim(copy, create=True) # Its own messages named with its own key are the copy's now. self._retarget(copy, f"{source.name}.", f"{new_name}.") - path = self._draft_file(new_name) + path = self._draft_file(copy.id) path.parent.mkdir(parents=True) path.write_text(copy.model_dump_json(indent=2) + "\n") # The bodies the source's editor shows: published, edits over them. # As drafts, so publishing the copy promotes them like a new flow's. - target = self._draft_nodes_dir(new_name) - for folder in (self._flow_dir(name) / "nodes", self._draft_nodes_dir(name)): + target = self._draft_nodes_dir(copy.id) + for folder in ( + self._flow_dir(flow_id) / "nodes", + self._draft_nodes_dir(flow_id), + ): for body in sorted(folder.glob("*.py")): target.mkdir(exist_ok=True) shutil.copy2(body, target / body.name) - self._commit(f"Duplicate flow '{name}' as '{new_name}'") + self._commit( + f"Duplicate flow {named(source.name, flow_id)} as " + f"{named(new_name, copy.id)}" + ) return copy @staticmethod diff --git a/backend/fluksio/flow/tombstones.py b/backend/fluksio/flow/tombstones.py new file mode 100644 index 0000000..b40e7db --- /dev/null +++ b/backend/fluksio/flow/tombstones.py @@ -0,0 +1,130 @@ +"""What is left of a deleted flow: its last key, while its history is kept. + +A flow's runs, events and metrics are filed under its id and outlive it. A new +flow's id is opaque, so a tombstone keeps the key it last had, and `names_of` +reads that back for anything showing the history. A flow deleted before +tombstones existed has its old key as its id, which already reads right. + +The flow store is the one writer: it buries a flow before removing its files, +asks before issuing an id, and lifts the tombstone of a flow restored under a +pinned id once the restored document is on disk. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import TYPE_CHECKING + +from sqlalchemy import delete, or_ +from sqlmodel import Session, col, select + +from fluksio.core.db import engine +from fluksio.models import EngineEvent, FlowRun, FlowTombstone, MetricBucket, Run + +if TYPE_CHECKING: + from fluksio.flow.store import FlowStore + + +def referenced(session: Session, flow_id: str) -> bool: + """Whether any history row names *flow_id*, as its flow or a node's. + + A training run's ``for_node`` names the node it trains, in a flow other + than the run's own, so the four ``flow`` columns are not enough. + """ + nodes = f"{flow_id}." + queries = [ + select(Run.id).where( + or_( + col(Run.flow) == flow_id, + col(Run.for_node).startswith(nodes, autoescape=True), + ) + ), + select(FlowRun.id).where(col(FlowRun.flow) == flow_id), + select(MetricBucket.flow).where( + or_( + col(MetricBucket.flow) == flow_id, + col(MetricBucket.node).startswith(nodes, autoescape=True), + ) + ), + select(EngineEvent.id).where( + or_( + col(EngineEvent.flow) == flow_id, + col(EngineEvent.node).startswith(nodes, autoescape=True), + ) + ), + ] + return any(session.exec(query.limit(1)).first() is not None for query in queries) + + +class Tombstones: + """Tombstones and the history behind an id, as the flow store asks.""" + + def bury(self, flow_id: str, name: str) -> None: + """Keep *name* for *flow_id*, before its files go. + + Written again by a retried delete, which may find another name if the + first attempt never got as far as removing anything. + """ + with Session(engine) as session: + stone = session.get(FlowTombstone, flow_id) or FlowTombstone( + id=flow_id, name=name + ) + stone.name = name + session.add(stone) + session.commit() + + def lift(self, flow_id: str) -> None: + """Forget the tombstone of a flow that is live again.""" + with Session(engine) as session: + session.execute( + delete(FlowTombstone).where(col(FlowTombstone.id) == flow_id) + ) + session.commit() + + def recorded(self, flow_id: str) -> str | None: + """The key the history of a flow that is not live is filed under. + + Its tombstone's, or the id itself for history from before tombstones, + whose id was its key. None when nothing names the id at all. + """ + with Session(engine) as session: + stone = session.get(FlowTombstone, flow_id) + if stone is not None: + return stone.name + return flow_id if referenced(session, flow_id) else None + + +def names_of(ids: Iterable[str], store: FlowStore) -> dict[str, str]: + """Each flow id's key: the live flow's, else its tombstone's, else the id. + + One read of the key index and one query, however many ids a response + carries. + """ + wanted = set(ids) + live = {flow_id: key for key, flow_id in store.key_index().items()} + names = {flow_id: live[flow_id] for flow_id in wanted if flow_id in live} + gone = wanted - names.keys() + if gone: + with Session(engine) as session: + for stone in session.exec( + select(FlowTombstone).where(col(FlowTombstone.id).in_(gone)) + ): + names[stone.id] = stone.name + return {flow_id: names.get(flow_id, flow_id) for flow_id in wanted} + + +def name_of(flow_id: str, store: FlowStore) -> str: + """One flow id's key, as `names_of`.""" + return names_of([flow_id], store)[flow_id] + + +def prune(session: Session) -> int: + """Drop the tombstones no history row names any more. Returns how many.""" + stale = [ + stone.id + for stone in session.exec(select(FlowTombstone)) + if not referenced(session, stone.id) + ] + if stale: + session.execute(delete(FlowTombstone).where(col(FlowTombstone.id).in_(stale))) + return len(stale) diff --git a/backend/fluksio/main.py b/backend/fluksio/main.py index 811e96f..febc112 100644 --- a/backend/fluksio/main.py +++ b/backend/fluksio/main.py @@ -60,6 +60,7 @@ from fluksio.flow.runs import RUN_STATE_TTL, RunService, sweep_artifacts from fluksio.flow.secrets import init_secrets from fluksio.flow.state import MemoryState, RedisState, StateBackend from fluksio.flow.store import FlowStore, IdentityChange, InvalidName +from fluksio.flow.tombstones import Tombstones from fluksio.flow.watchdog import LoopWatchdog from fluksio.flow.workers import PythonWorkerPool @@ -69,7 +70,7 @@ logger = logging.getLogger(__name__) #: worker's `?token=`, the pairing poll's `?secret=`. _CREDENTIAL_IN_QUERY = re.compile(r"([?&](?:[\w-]*token|[\w-]*secret|ticket)=)[^&\s]+") -#: A hook's secret, the last segment of `/hooks///`. A hook +#: A hook's secret, the last segment of `/hooks///`. A hook #: without one loses its last path segment from the log too, since the log #: cannot tell which hooks carry a secret. _SECRET_IN_HOOK_PATH = re.compile(r"(/hooks/[^/\s?]+/(?:[^/\s?]*/)*)[^/\s?]+") @@ -306,7 +307,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # store's own `_migrate`, the device migrations below and every cache # after them write documents back with whatever id they read. await run_in_threadpool(assign_ids, settings.FLOWS_DIR) - store = FlowStore(settings.FLOWS_DIR) + store = FlowStore(settings.FLOWS_DIR, tombstones=Tombstones()) # The packages node code imports. In the background, because a first # install of anything substantial takes minutes and `uv` is given five # of them twice over — held here, the container never answers its diff --git a/backend/fluksio/models.py b/backend/fluksio/models.py index baa6897..402b26d 100644 --- a/backend/fluksio/models.py +++ b/backend/fluksio/models.py @@ -607,6 +607,25 @@ class AlertSent(SQLModel, table=True): ts: float = Field(index=True) +# ----------------------------------------------------------------------------- +# Flow tombstones +# +# A deleted flow's runs, events and metrics stay, filed under its id. New ids +# are opaque, so without the key the flow last had they would name nothing +# anyone recognises. +# ----------------------------------------------------------------------------- + + +class FlowTombstone(SQLModel, table=True): + """A deleted flow's last key, kept while history names its id.""" + + __tablename__ = "flow_tombstone" + + id: str = Field(primary_key=True, max_length=255) + name: str = Field(max_length=255) + deleted_at: datetime = Field(default_factory=get_datetime_utc, sa_type=UTCDateTime) + + # ----------------------------------------------------------------------------- # Worker tokens # diff --git a/backend/fluksio/sdk/client.py b/backend/fluksio/sdk/client.py index 07d7b5b..9a607ad 100644 --- a/backend/fluksio/sdk/client.py +++ b/backend/fluksio/sdk/client.py @@ -238,7 +238,7 @@ class Client: # -- flows ------------------------------------------------------------- def get_flow(self, name: str) -> dict[str, Any] | None: - """The stored flow, draft included, or ``None`` if there is none.""" + """The stored flow with the id *name*, draft included, or ``None``.""" try: result: dict[str, Any] = self._call("GET", f"/flows/{seg(name)}") except ApiError as exc: @@ -247,9 +247,23 @@ class Client: raise return result + def find_flow(self, key: str) -> dict[str, Any] | None: + """The listing of the flow holding the key *key*, or ``None``.""" + result = self._call("GET", "/flows/", params={"name": key}) + found = list(result.get("data") or []) + return found[0] if found else None + + def create_flow(self, document: dict[str, Any]) -> dict[str, Any]: + """A new flow, under the id the engine issues for it.""" + result: dict[str, Any] = self._call("POST", "/flows/", json=document) + return result + def put_flow(self, document: dict[str, Any]) -> dict[str, Any]: + """Save over the flow *document* names by its id, or by its key for + one whose id is its key.""" + flow_id = document.get("id") or document["name"] result: dict[str, Any] = self._call( - "PUT", f"/flows/{seg(document['name'])}", json=document + "PUT", f"/flows/{seg(flow_id)}", json=document ) return result @@ -821,7 +835,11 @@ def _sync_one( force: bool, ) -> SyncReport: report = SyncReport(target.name) - stored = client.get_flow(target.name) + # The declared key is what an instance shares; the id is this engine's, + # and every request after the lookup names the flow by it. + found = client.find_flow(target.name) + flow_id = str(found["id"]) if found else "" + stored = client.get_flow(flow_id) if flow_id else None version = 1 # What the drift check already read, so the write loop below does not ask # the engine for every node's body a second time. @@ -836,7 +854,11 @@ def _sync_one( document = target.document(origin) moved = _moved_code(stored, document) - saved = client.put_flow(document | {"version": version}) + if stored is None: + saved = client.create_flow(document) + flow_id = str((saved.get("definition") or {}).get("id") or target.name) + else: + saved = client.put_flow(document | {"id": flow_id, "version": version}) # An engine older than the per-node digest parses the document without # those fields and stores it without them, silently: no error, no 404, and # a cache still keyed on the whole repository. @@ -860,16 +882,16 @@ def _sync_one( if not report.created: stored_code = known.get(node_id) if stored_code is None: - stored_code = client.get_source(target.name, node_id) + stored_code = client.get_source(flow_id, node_id) if stored_code == code: continue - client.put_source(target.name, node_id, code) + client.put_source(flow_id, node_id, code) if node_id not in report.changed: report.changed.append(node_id) - has_draft = bool((client.get_flow(target.name) or {}).get("has_draft")) + has_draft = bool((client.get_flow(flow_id) or {}).get("has_draft")) if publish and has_draft: - client.publish(target.name, version) + client.publish(flow_id, version) report.published = True else: # Only when one is actually there. "Nothing needed publishing" and @@ -934,7 +956,9 @@ def _refuse_on_drift( for stored_node in definition.get("nodes") or []: node_id = str(stored_node.get("id")) try: - entry = client.get_source_entry(target.name, node_id) + entry = client.get_source_entry( + str(definition.get("id") or target.name), node_id + ) except ApiError: continue if entry.get("missing"): diff --git a/backend/scripts/bench_startup.py b/backend/scripts/bench_startup.py index 9f39550..3610ee8 100644 --- a/backend/scripts/bench_startup.py +++ b/backend/scripts/bench_startup.py @@ -89,9 +89,10 @@ def ensure_flow(api: Api) -> None: try: current = api.call("GET", f"/api/v1/flows/{BENCH_FLOW}") definition["version"] = current["definition"]["version"] + api.call("PUT", f"/api/v1/flows/{BENCH_FLOW}", definition) except urllib.error.HTTPError: - pass - api.call("PUT", f"/api/v1/flows/{BENCH_FLOW}", definition) + # Pinned to its key, so every path below can name it. + api.call("POST", "/api/v1/flows/", {**definition, "id": BENCH_FLOW}) api.call( "PUT", f"/api/v1/flows/{BENCH_FLOW}/nodes/noop/source", {"code": NODE_SOURCE} ) diff --git a/backend/scripts/soak.py b/backend/scripts/soak.py index e601e37..fcbfebc 100644 --- a/backend/scripts/soak.py +++ b/backend/scripts/soak.py @@ -354,9 +354,13 @@ def create_flow( definition: dict[str, Any], sources: dict[str, str] | None = None, ) -> None: - """Draft, node source, publish — the sequence the editor uses.""" + """Draft, node source, publish — the sequence the editor uses. + + Pinned to its key, which the superuser running this may, so the hooks and + paths below can name it. + """ name = definition["name"] - response = client.put(f"/api/v1/flows/{name}", json=definition) + response = client.post("/api/v1/flows/", json={**definition, "id": name}) response.raise_for_status() version = response.json()["definition"]["version"] diff --git a/backend/tests/api/routes/test_code_rights.py b/backend/tests/api/routes/test_code_rights.py index 84ed5fd..a69c12b 100644 --- a/backend/tests/api/routes/test_code_rights.py +++ b/backend/tests/api/routes/test_code_rights.py @@ -36,10 +36,10 @@ def save( ) -> Callable[..., Any]: """Seed the flow as a superuser, then save a change as someone else.""" client.delete(f"{PREFIX}/{FLOW}", headers=superuser_token_headers) - seeded = client.put( - f"{PREFIX}/{FLOW}", headers=superuser_token_headers, json=a_flow(0) + seeded = client.post( + f"{PREFIX}/", headers=superuser_token_headers, json={**a_flow(0), "id": FLOW} ) - assert seeded.status_code == 200, seeded.text + assert seeded.status_code == 201, seeded.text version = seeded.json()["definition"]["version"] def put(headers: dict[str, str], **changes: Any) -> Any: diff --git a/backend/tests/api/routes/test_dashboards.py b/backend/tests/api/routes/test_dashboards.py index 20a98fd..b4fbe4d 100644 --- a/backend/tests/api/routes/test_dashboards.py +++ b/backend/tests/api/routes/test_dashboards.py @@ -1,8 +1,10 @@ -"""Dashboards over HTTP: saving one, which is also how the first one is made.""" +"""Dashboards over HTTP: creating one under an issued id, and saving it.""" from fastapi.testclient import TestClient from fluksio.core.config import settings +from fluksio.flow.store import ISSUED_ID +from tests.utils.flows import put_dashboard PREFIX = f"{settings.API_V1_STR}/dashboards" @@ -26,37 +28,78 @@ def a_dashboard(name: str) -> dict: } -def test_a_save_creates_a_dashboard_that_does_not_exist_yet( +def test_a_dashboard_is_created_with_an_id_and_saved_by_it( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: - """A first draft, not a 404: creating one *is* saving it at version 0.""" - body = a_dashboard("put_creates") - - response = client.put( - f"{PREFIX}/put_creates", headers=superuser_token_headers, json=body + """A first draft under an id the server issues, which every path then names.""" + created = client.post( + f"{PREFIX}/", headers=superuser_token_headers, json={"name": "made_here"} ) - - assert response.status_code == 200, response.text - saved = response.json() - assert saved["version"] == 1 and saved["has_draft"] is True + assert created.status_code == 201, created.text + made = created.json() + assert ISSUED_ID.fullmatch(made["id"]) and made["name"] == "made_here" + assert made["version"] == 1 and made["has_draft"] is True # A draft alone: nothing was published, so no panel can be shown it. assert ( - client.get(f"{PREFIX}/put_creates", headers=superuser_token_headers).status_code + client.get( + f"{PREFIX}/{made['id']}", headers=superuser_token_headers + ).status_code == 404 ) + body = {**a_dashboard("made_here"), "version": made["version"]} + saved = client.put( + f"{PREFIX}/{made['id']}", headers=superuser_token_headers, json=body + ) + assert saved.status_code == 200, saved.text draft = client.get( - f"{PREFIX}/put_creates?draft=true", headers=superuser_token_headers + f"{PREFIX}/{made['id']}?draft=true", headers=superuser_token_headers ) assert draft.json()["widgets"] == body["widgets"] + (listed,) = client.get( + f"{PREFIX}/?name=made_here", headers=superuser_token_headers + ).json()["data"] + assert listed["id"] == made["id"] + + copy = client.post( + f"{PREFIX}/{made['id']}/duplicate", headers=superuser_token_headers + ).json() + assert copy["name"] == "made_here_copy" and copy["id"] != made["id"] + for dashboard_id in (made["id"], copy["id"]): + client.delete(f"{PREFIX}/{dashboard_id}", headers=superuser_token_headers) + + +def test_a_save_neither_creates_nor_renames( + client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + missing = client.put( + f"{PREFIX}/nowhere", + headers=superuser_token_headers, + json={**a_dashboard("nowhere"), "version": 0}, + ) + assert missing.status_code == 404 + saved = put_dashboard( + client, superuser_token_headers, a_dashboard("put_fixed") + ).json() + renamed = client.put( + f"{PREFIX}/put_fixed", + headers=superuser_token_headers, + json={**saved, "name": "renamed"}, + ) + assert renamed.status_code == 409 + elsewhere = client.put( + f"{PREFIX}/put_fixed", + headers=superuser_token_headers, + json={**saved, "id": "elsewhere"}, + ) + assert elsewhere.status_code == 422 + client.delete(f"{PREFIX}/put_fixed", headers=superuser_token_headers) def test_a_save_of_an_existing_dashboard_keeps_what_it_did_not_change( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: seeded = a_dashboard("put_updates") - first = client.put( - f"{PREFIX}/put_updates", headers=superuser_token_headers, json=seeded - ).json() + first = put_dashboard(client, superuser_token_headers, seeded).json() renamed = {**first, "widgets": [{**first["widgets"][0], "title": "Outside"}]} second = client.put( @@ -78,37 +121,38 @@ def test_a_save_based_on_a_version_someone_moved_past_is_refused( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: body = a_dashboard("put_conflicts") - client.put(f"{PREFIX}/put_conflicts", headers=superuser_token_headers, json=body) + put_dashboard(client, superuser_token_headers, body) stale = client.put( - f"{PREFIX}/put_conflicts", headers=superuser_token_headers, json=body + f"{PREFIX}/put_conflicts", + headers=superuser_token_headers, + json={**body, "version": 1}, ) assert stale.status_code == 409 - assert stale.json()["detail"]["current_version"] == 1 + assert stale.json()["detail"]["current_version"] == 2 -def test_a_dashboard_is_known_by_an_id_no_save_changes( - client: TestClient, superuser_token_headers: dict[str, str] +def test_only_a_superuser_pins_a_dashboard_id( + client: TestClient, + superuser_token_headers: dict[str, str], + normal_user_token_headers: dict[str, str], ) -> None: - saved = client.put( - f"{PREFIX}/put_identified", + pinned = {"id": "pinned_hall", "name": "pinned_hall"} + refused = client.post(f"{PREFIX}/", headers=normal_user_token_headers, json=pinned) + assert refused.status_code == 403 + made = client.post(f"{PREFIX}/", headers=superuser_token_headers, json=pinned) + assert made.status_code == 201 and made.json()["id"] == "pinned_hall" + again = client.post( + f"{PREFIX}/", headers=superuser_token_headers, - json=a_dashboard("put_identified"), - ).json() - assert saved["id"] == "put_identified" - - elsewhere = client.put( - f"{PREFIX}/put_identified", - headers=superuser_token_headers, - json={**saved, "id": "elsewhere"}, + json={"id": "pinned_hall", "name": "other_hall"}, ) - assert elsewhere.status_code == 422 - renamed = client.post( - f"{PREFIX}/put_identified/rename", - headers=superuser_token_headers, - json={"name": "renamed"}, + assert again.status_code == 409 + # Anyone may make one under an id of the server's. + issued = client.post( + f"{PREFIX}/", headers=normal_user_token_headers, json={"name": "their_hall"} ) - assert renamed.status_code in (404, 405) - - client.delete(f"{PREFIX}/put_identified", headers=superuser_token_headers) + assert issued.status_code == 201 + for dashboard_id in ("pinned_hall", issued.json()["id"]): + client.delete(f"{PREFIX}/{dashboard_id}", headers=superuser_token_headers) diff --git a/backend/tests/api/routes/test_devices.py b/backend/tests/api/routes/test_devices.py index d2199c8..daae687 100644 --- a/backend/tests/api/routes/test_devices.py +++ b/backend/tests/api/routes/test_devices.py @@ -12,6 +12,7 @@ from fluksio.cloud import config as cloud_config from fluksio.core.config import settings from fluksio.flow.devices import DeviceState from fluksio.main import app +from tests.utils.flows import put_flow PREFIX = f"{settings.API_V1_STR}/devices" #: What the openpanel firmware calls, kept as aliases. @@ -54,8 +55,10 @@ def _dashboard(client: TestClient, headers: dict[str, str], name: str) -> dict: A new one is a draft, and a panel is only ever shown what is published. """ - created = client.post(f"{DASHBOARDS}/{name}", headers=headers) - assert created.status_code == 200, created.text + created = client.post( + f"{DASHBOARDS}/", headers=headers, json={"id": name, "name": name} + ) + assert created.status_code == 201, created.text published = client.post( f"{DASHBOARDS}/{name}/publish", headers=headers, @@ -95,7 +98,11 @@ def test_assign_and_read_back( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: for name in ("hall_a", "hall_b"): - client.post(f"{DASHBOARDS}/{name}", headers=superuser_token_headers) + client.post( + f"{DASHBOARDS}/", + headers=superuser_token_headers, + json={"id": name, "name": name}, + ) _panels( client, @@ -132,7 +139,11 @@ def test_touch_belongs_to_the_device( The same dashboard may hang on a hallway tablet and on a desk browser, so the flag rides on the panel and each one answers for itself. """ - client.post(f"{DASHBOARDS}/shared", headers=superuser_token_headers) + client.post( + f"{DASHBOARDS}/", + headers=superuser_token_headers, + json={"id": "shared", "name": "shared"}, + ) _panels( client, superuser_token_headers, @@ -553,12 +564,8 @@ def _publish_flow( ) -> dict: """Write a flow and publish it, replacing whatever was there.""" client.delete(f"{FLOWS}/{name}", headers=headers) - saved = client.put( - f"{FLOWS}/{name}", - headers=headers, - json={"name": name, "nodes": nodes, "version": 0, **more}, - ) - assert saved.status_code == 200, saved.text + saved = put_flow(client, headers, {"name": name, "nodes": nodes, **more}, name) + assert saved.is_success, saved.text published = client.post( f"{FLOWS}/{name}/publish", headers=headers, @@ -590,10 +597,10 @@ def test_a_devices_io_comes_from_its_device_nodes( _panels(client, superuser_token_headers, {"panels": [{"id": "porch_io"}]}) _publish_flow(client, superuser_token_headers, "porch", [PORCH_NODE]) # A draft wires nothing until it is published. - client.put( - f"{FLOWS}/porch_draft", - headers=superuser_token_headers, - json={ + put_flow( + client, + superuser_token_headers, + { "name": "porch_draft", "nodes": [ _device_node( @@ -603,6 +610,7 @@ def test_a_devices_io_comes_from_its_device_nodes( ], "version": 0, }, + "porch_draft", ) first = client.get(f"{PREFIX}/porch_io", headers=superuser_token_headers) @@ -792,10 +800,10 @@ def test_a_port_wired_twice_or_an_unknown_device_is_an_issue( "ports": [{"port": "touch", "direction": "in", "dtype": "bool"}], } store._write_states(states) - client.put( - f"{FLOWS}/porch_too", - headers=superuser_token_headers, - json={ + put_flow( + client, + superuser_token_headers, + { "name": "porch_too", "version": client.get( f"{FLOWS}/porch_too", headers=superuser_token_headers @@ -816,6 +824,7 @@ def test_a_port_wired_twice_or_an_unknown_device_is_an_issue( ) ], }, + "porch_too", ) assert codes("porch_too") == { ("device_port", "relay", True), @@ -1440,7 +1449,11 @@ def test_a_panel_credential_cannot_move_artifacts( the scope check has to be handed the request there too — without it a panel would act as whoever approved it. """ - client.post(f"{DASHBOARDS}/panel_art", headers=superuser_token_headers) + client.post( + f"{DASHBOARDS}/", + headers=superuser_token_headers, + json={"id": "panel_art", "name": "panel_art"}, + ) _panels( client, superuser_token_headers, diff --git a/backend/tests/api/routes/test_flavors.py b/backend/tests/api/routes/test_flavors.py index 8c2fce4..b87481c 100644 --- a/backend/tests/api/routes/test_flavors.py +++ b/backend/tests/api/routes/test_flavors.py @@ -7,6 +7,7 @@ from sqlmodel import Session, select from fluksio.core.config import settings from fluksio.core.db import seed_flavors from fluksio.models import Flavor +from tests.utils.flows import put_flow def test_an_instance_starts_with_sizes_to_pick_from( @@ -97,12 +98,11 @@ def test_a_flow_cannot_name_a_size_that_does_not_exist( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: """Caught at the save, which covers the canvas and `fluksio sync` at once.""" - response = client.put( - f"{settings.API_V1_STR}/flows/flavor_check", + response = client.post( + f"{settings.API_V1_STR}/flows/", headers=superuser_token_headers, json={ "name": "flavor_check", - "version": 1, "nodes": [ {"id": "fit", "type": "python", "resources": {"flavor": "imaginary"}} ], @@ -120,12 +120,11 @@ def test_a_size_a_node_still_asks_for_is_not_deleted( headers=superuser_token_headers, json={"name": "in-use", "cpus": 2}, ) - client.put( - f"{settings.API_V1_STR}/flows/flavor_user", - headers=superuser_token_headers, - json={ + put_flow( + client, + superuser_token_headers, + { "name": "flavor_user", - "version": 1, "nodes": [ {"id": "fit", "type": "python", "resources": {"flavor": "in-use"}} ], diff --git a/backend/tests/api/routes/test_flow_ids.py b/backend/tests/api/routes/test_flow_ids.py new file mode 100644 index 0000000..5f0fb70 --- /dev/null +++ b/backend/tests/api/routes/test_flow_ids.py @@ -0,0 +1,341 @@ +"""Flows over HTTP by id, with keys apart from them. + +A flow made through the API gets an opaque id unless a superuser pins one, so +most tests here run on both: a pinned ``x_`` and an issued id. A path +that quietly read the key where it meant the id passes for a flow whose id is +its key, and nowhere else. +""" + +import uuid +from collections.abc import Callable +from datetime import UTC, datetime +from typing import Any + +import pytest +from fastapi.testclient import TestClient +from sqlmodel import Session, delete, select + +from fluksio.core.config import settings +from fluksio.flow.store import ISSUED_ID +from fluksio.flow.tombstones import names_of +from fluksio.models import FlowTombstone, Run + +FLOWS = f"{settings.API_V1_STR}/flows" +DASHBOARDS = f"{settings.API_V1_STR}/dashboards" + +SENSOR = "def process():\n return {'reading': 21.5}\n" +DOUBLE = "def process(reading):\n return {'doubled': reading * 2}\n" + + +def a_flow(key: str, mode: str = "live") -> dict[str, Any]: + return { + "name": key, + "mode": mode, + "nodes": [ + { + "id": "sensor", + "type": "python", + "provides": [{"name": "reading", "dtype": "float"}], + }, + { + "id": "double", + "type": "python", + "requires": [{"name": "reading", "dtype": "float"}], + "provides": [{"name": "doubled", "dtype": "float"}], + }, + { + "id": "hook", + "type": "http", + "params": {"url": "/tick", "method": "POST"}, + "provides": [{"name": "tick", "dtype": "float"}], + }, + ], + "outputs": ["doubled"] if mode == "batch" else [], + } + + +def a_key() -> str: + return f"ids_{uuid.uuid4().hex[:8]}" + + +Make = Callable[..., dict[str, Any]] + + +@pytest.fixture(params=["pinned", "issued"]) +def make( + request: pytest.FixtureRequest, + client: TestClient, + superuser_token_headers: dict[str, str], +) -> Make: + """Create a flow, pinned to ``x_`` or under an issued id; its detail.""" + made: list[str] = [] + + def create(key: str, **document: Any) -> dict[str, Any]: + body = {**a_flow(key), **document} + if request.param == "pinned": + body["id"] = f"x_{key}" + response = client.post(f"{FLOWS}/", headers=superuser_token_headers, json=body) + assert response.status_code == 201, response.text + detail: dict[str, Any] = response.json() + made.append(detail["definition"]["id"]) + return detail + + yield create # type: ignore[misc] + for flow_id in made: + client.delete(f"{FLOWS}/{flow_id}", headers=superuser_token_headers) + + +def test_a_flow_is_made_found_and_saved_by_its_id( + make: Make, client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + key = a_key() + definition = make(key)["definition"] + flow_id = definition["id"] + assert flow_id != key and definition["name"] == key + assert flow_id == f"x_{key}" or ISSUED_ID.fullmatch(flow_id) + + read = client.get(f"{FLOWS}/{flow_id}", headers=superuser_token_headers) + assert read.json()["definition"]["name"] == key + # By its key it is not there: only a client resolves a key. + assert client.get( + f"{FLOWS}/{key}", headers=superuser_token_headers + ).status_code == (404) + (listed,) = client.get( + f"{FLOWS}/?name={key}", headers=superuser_token_headers + ).json()["data"] + assert (listed["id"], listed["name"]) == (flow_id, key) + assert ( + client.get(f"{FLOWS}/?name={key}_no", headers=superuser_token_headers).json()[ + "data" + ] + == [] + ) + + saved = client.put( + f"{FLOWS}/{flow_id}", + headers=superuser_token_headers, + json={**definition, "title": "Retitled"}, + ) + assert saved.status_code == 200, saved.text + assert saved.json()["definition"]["title"] == "Retitled" + renamed = client.put( + f"{FLOWS}/{flow_id}", + headers=superuser_token_headers, + json={**saved.json()["definition"], "name": f"{key}_b"}, + ) + assert renamed.status_code == 409 + + copy = client.post( + f"{FLOWS}/{flow_id}/duplicate", headers=superuser_token_headers + ).json()["definition"] + assert copy["name"] == f"{key}_copy" and ISSUED_ID.fullmatch(copy["id"]) + client.delete(f"{FLOWS}/{copy['id']}", headers=superuser_token_headers) + + +def test_a_flow_runs_under_its_key_and_is_reached_by_its_id( + make: Make, client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + """Publish, a message, the nodes running, the state under the key.""" + key = a_key() + flow_id = make(key)["definition"]["id"] + for node, code in (("sensor", SENSOR), ("double", DOUBLE)): + saved = client.put( + f"{FLOWS}/{flow_id}/nodes/{node}/source", + headers=superuser_token_headers, + json={"code": code}, + ) + assert saved.json()["id"] == f"{key}.{node}" + version = client.get(f"{FLOWS}/{flow_id}", headers=superuser_token_headers).json()[ + "definition" + ]["version"] + published = client.post( + f"{FLOWS}/{flow_id}/publish", + headers=superuser_token_headers, + json={"version": version}, + ) + assert published.status_code == 200, published.text + assert {n["id"]: n["status"] for n in published.json()["nodes"]} == { + f"{key}.sensor": "active", + f"{key}.double": "active", + f"{key}.hook": "active", + } + + ran = client.post( + f"{FLOWS}/{flow_id}/run", headers=superuser_token_headers, json={"inputs": {}} + ) + assert ran.json()["values"][f"{key}.doubled"]["value"] == 43.0 + triggered = client.post( + f"{FLOWS}/{flow_id}/nodes/double/trigger", + headers=superuser_token_headers, + json={"values": {f"{key}.reading": 1.0}}, + ) + assert triggered.json()["values"][f"{key}.doubled"]["value"] == 2.0 + history = client.get( + f"{FLOWS}/{flow_id}/history/doubled", headers=superuser_token_headers + ).json() + assert history["message"] == f"{key}.doubled" and history["points"] + + # The webhook is under the id, which a later key change leaves alone. + hooked = client.post(f"/hooks/{flow_id}/tick", json={"tick": 7.0}) + assert hooked.status_code == 200, hooked.text + assert client.post(f"/hooks/{key}/tick", json={"tick": 8.0}).status_code == 404 + state = client.get(f"{FLOWS}/{flow_id}/state", headers=superuser_token_headers) + assert state.json()["values"][f"{key}.tick"]["value"] == 7.0 + + paused = client.post(f"{FLOWS}/{flow_id}/pause", headers=superuser_token_headers) + assert paused.status_code == 200 + assert client.get(f"{FLOWS}/{flow_id}", headers=superuser_token_headers).json()[ + "paused" + ] + client.post(f"{FLOWS}/{flow_id}/resume", headers=superuser_token_headers) + stopped = client.post(f"{FLOWS}/{flow_id}/stop", headers=superuser_token_headers) + assert stopped.json()["enabled"] is False + assert ( + client.post( + f"{FLOWS}/{flow_id}/run", + headers=superuser_token_headers, + json={"inputs": {}}, + ).status_code + == 409 + ) + started = client.post(f"{FLOWS}/{flow_id}/start", headers=superuser_token_headers) + assert started.json()["enabled"] is True + acknowledged = client.post( + f"{FLOWS}/{flow_id}/nodes/double/acknowledge", headers=superuser_token_headers + ) + assert acknowledged.status_code == 200 + + +def test_a_results_dashboard_is_made_from_a_flow_id_under_the_flows_key( + make: Make, client: TestClient, superuser_token_headers: dict[str, str] +) -> None: + key = a_key() + flow_id = make(key, mode="batch")["definition"]["id"] + client.post( + f"{FLOWS}/{flow_id}/publish", + headers=superuser_token_headers, + json={"version": 1}, + ) + + made = client.post( + f"{DASHBOARDS}/from-flow/{flow_id}", headers=superuser_token_headers + ) + assert made.status_code == 201, made.text + board = made.json() + assert board["name"] == f"{key}_results" and ISSUED_ID.fullmatch(board["id"]) + assert board["has_draft"] is False + again = client.post( + f"{DASHBOARDS}/from-flow/{flow_id}", headers=superuser_token_headers + ) + assert again.status_code == 409 + by_key = client.post( + f"{DASHBOARDS}/from-flow/{key}", headers=superuser_token_headers + ) + assert by_key.status_code == 404 + client.delete(f"{DASHBOARDS}/{board['id']}", headers=superuser_token_headers) + + +def test_who_may_create_what( + client: TestClient, + superuser_token_headers: dict[str, str], + normal_user_token_headers: dict[str, str], +) -> None: + key = a_key() + plain = {"name": key, "nodes": [{"id": "hold", "type": "delay"}]} + + # Anyone makes a flow under an issued id, as anyone could save one before. + made = client.post(f"{FLOWS}/", headers=normal_user_token_headers, json=plain) + assert made.status_code == 201, made.text + flow_id = made.json()["definition"]["id"] + assert ISSUED_ID.fullmatch(flow_id) + # Code stays a superuser's to put there, and so does choosing an id. + coded = client.post( + f"{FLOWS}/", + headers=normal_user_token_headers, + json={**plain, "name": f"{key}_code", "nodes": [{"id": "fn"}]}, + ) + assert coded.status_code == 403 + pinned = client.post( + f"{FLOWS}/", + headers=normal_user_token_headers, + json={**plain, "name": f"{key}_pin", "id": f"x_{key}"}, + ) + assert pinned.status_code == 403 + + def pin(flow: str, name: str) -> int: + return client.post( + f"{FLOWS}/", + headers=superuser_token_headers, + json={**plain, "name": name, "id": flow}, + ).status_code + + assert pin("Not-An-Id", f"{key}_a") == 422 + assert pin("graph", f"{key}_b") == 422 # behind `GET /flows/graph` + assert pin(flow_id, f"{key}_c") == 409 # a live flow's + assert pin(f"x_{key}", key) == 409 # the key is taken + assert pin(f"x_{key}", "f_k3q7m2xa") == 400 # shaped like an issued id + + client.delete(f"{FLOWS}/{flow_id}", headers=superuser_token_headers) + + +def test_a_deleted_flow_keeps_its_name_and_frees_its_key( + make: Make, + client: TestClient, + superuser_token_headers: dict[str, str], + db: Session, +) -> None: + key = a_key() + old = make(key)["definition"]["id"] + db.add(Run(id=f"run-{key}", flow=old, status="ok", created_at=datetime.now(UTC))) + db.commit() + controller = client.app.state.flow_controller # type: ignore[attr-defined] + controller.state.set(f"{key}.doubled", 1.0) + + deleted = client.delete(f"{FLOWS}/{old}", headers=superuser_token_headers) + assert deleted.status_code == 200 + assert db.get(FlowTombstone, old).name == key + assert names_of([old], controller.store) == {old: key} + # What the old flow left under its key went with it. + assert controller.key_residue(key) == [] + + # Something still writing there keeps the key from a new flow. + controller.state.set(f"{key}.stray", 1.0) + refused = client.post( + f"{FLOWS}/", headers=superuser_token_headers, json=a_flow(key) + ) + assert refused.status_code == 409 and f"{key}.stray" in refused.json()["detail"] + controller.forget_flow(key) + + # Under an id of its own, not the one the runs are filed under. + made = client.post(f"{FLOWS}/", headers=superuser_token_headers, json=a_flow(key)) + assert made.status_code == 201, made.text + new = made.json()["definition"]["id"] + assert new != old + assert db.exec(select(Run).where(Run.flow == new)).first() is None + client.delete(f"{FLOWS}/{new}", headers=superuser_token_headers) + db.exec(delete(Run).where(Run.id == f"run-{key}")) # type: ignore[call-overload] + db.commit() + + +def test_a_pinned_restore_takes_its_history_back( + client: TestClient, superuser_token_headers: dict[str, str], db: Session +) -> None: + key = a_key() + pin = f"x_{key}" + plain = {"name": key, "nodes": []} + client.post(f"{FLOWS}/", headers=superuser_token_headers, json={**plain, "id": pin}) + client.delete(f"{FLOWS}/{pin}", headers=superuser_token_headers) + + other = client.post( + f"{FLOWS}/", + headers=superuser_token_headers, + json={**plain, "name": f"{key}_b", "id": pin}, + ) + assert other.status_code == 409 + restored = client.post( + f"{FLOWS}/", headers=superuser_token_headers, json={**plain, "id": pin} + ) + assert restored.status_code == 201, restored.text + db.expire_all() + assert db.get(FlowTombstone, pin) is None + client.delete(f"{FLOWS}/{pin}", headers=superuser_token_headers) diff --git a/backend/tests/api/routes/test_flows.py b/backend/tests/api/routes/test_flows.py index 74a31b1..25ee7a5 100644 --- a/backend/tests/api/routes/test_flows.py +++ b/backend/tests/api/routes/test_flows.py @@ -11,6 +11,7 @@ from fluksio.api.routes import flows as flows_route from fluksio.core.config import settings from fluksio.flow.events import event_bus from fluksio.models import Run, RunArtifact, RunMetric, RunNode +from tests.utils.flows import put_flow PREFIX = f"{settings.API_V1_STR}/flows" @@ -51,10 +52,8 @@ def test_flows_require_authentication(client: TestClient) -> None: def test_save_then_read_flow( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: - response = client.put( - f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow() - ) - assert response.status_code == 200 + response = put_flow(client, superuser_token_headers, a_flow(), "demo") + assert response.status_code == 201 response = client.get(f"{PREFIX}/demo", headers=superuser_token_headers) assert response.status_code == 200 @@ -68,58 +67,35 @@ def test_save_then_read_flow( assert datetime.fromisoformat(summary["updated_at"]).tzinfo is not None -def test_name_mismatch_is_rejected( +def test_a_save_neither_creates_nor_renames( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: - response = client.put( - f"{PREFIX}/other", headers=superuser_token_headers, json=a_flow("demo") + put_flow(client, superuser_token_headers, a_flow(), "demo") + current = client.get(f"{PREFIX}/demo", headers=superuser_token_headers).json() + document = current["definition"] + + missing = client.put( + f"{PREFIX}/nowhere", headers=superuser_token_headers, json=a_flow("nowhere") ) - assert response.status_code == 400 - - -def test_a_flow_is_known_by_an_id_no_save_changes( - client: TestClient, superuser_token_headers: dict[str, str] -) -> None: - """For now the id of a new flow is its key, and the URL names both.""" - saved = client.put( - f"{PREFIX}/identified", + assert missing.status_code == 404 + renamed = client.put( + f"{PREFIX}/demo", headers=superuser_token_headers, - json=a_flow("identified"), - ).json()["definition"] - assert saved["id"] == "identified" - listed = client.get(f"{PREFIX}/", headers=superuser_token_headers).json()["data"] - assert {"id": "identified", "name": "identified"}.items() <= next( - f for f in listed if f["name"] == "identified" - ).items() - + json={**document, "name": "other"}, + ) + assert renamed.status_code == 409 elsewhere = client.put( - f"{PREFIX}/identified", + f"{PREFIX}/demo", headers=superuser_token_headers, - json={**saved, "id": "elsewhere"}, + json={**document, "id": "elsewhere"}, ) assert elsewhere.status_code == 400 - # A key changes by a rename, and that route is not there yet. - renamed = client.post( - f"{PREFIX}/identified/rename", - headers=superuser_token_headers, - json={"new_name": "renamed"}, - ) - assert renamed.status_code in (404, 405) - # Shaped like an id the server issues, which a new key may not be. - issued = client.put( - f"{PREFIX}/f_k3q7m2xa", - headers=superuser_token_headers, - json=a_flow("f_k3q7m2xa"), - ) - assert issued.status_code == 400 - - client.delete(f"{PREFIX}/identified", headers=superuser_token_headers) def test_broken_node_is_reported_and_siblings_stay_active( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: - client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow()) + put_flow(client, superuser_token_headers, a_flow(), "demo") client.put( f"{PREFIX}/demo/nodes/sensor/source", headers=superuser_token_headers, @@ -143,7 +119,7 @@ def test_broken_node_is_reported_and_siblings_stay_active( def test_running_a_flow_produces_values( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: - client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow()) + put_flow(client, superuser_token_headers, a_flow(), "demo") client.put( f"{PREFIX}/demo/nodes/sensor/source", headers=superuser_token_headers, @@ -165,9 +141,7 @@ def test_running_a_flow_produces_values( def test_editing_does_not_deploy_until_published( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: - saved = client.put( - f"{PREFIX}/staged", headers=superuser_token_headers, json=a_flow("staged") - ).json() + saved = put_flow(client, superuser_token_headers, a_flow("staged"), "staged").json() assert saved["has_draft"] is True # Nothing is running yet, so the engine knows no nodes of this flow. assert ( @@ -197,8 +171,8 @@ def test_editing_does_not_deploy_until_published( def test_a_stale_save_is_refused( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: - saved = client.put( - f"{PREFIX}/contested", headers=superuser_token_headers, json=a_flow("contested") + saved = put_flow( + client, superuser_token_headers, a_flow("contested"), "contested" ).json() stale = saved["definition"] @@ -222,8 +196,8 @@ def test_a_stale_save_is_refused( def test_discarding_a_draft_restores_what_is_running( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: - saved = client.put( - f"{PREFIX}/reverted", headers=superuser_token_headers, json=a_flow("reverted") + saved = put_flow( + client, superuser_token_headers, a_flow("reverted"), "reverted" ).json() client.post( f"{PREFIX}/reverted/publish", @@ -232,10 +206,11 @@ def test_discarding_a_draft_restores_what_is_running( ) published = client.get(f"{PREFIX}/reverted", headers=superuser_token_headers).json() - client.put( - f"{PREFIX}/reverted", - headers=superuser_token_headers, - json={**published["definition"], "title": "Scratch that"}, + put_flow( + client, + superuser_token_headers, + {**published["definition"], "title": "Scratch that"}, + "reverted", ) response = client.post( @@ -251,9 +226,7 @@ def test_discarding_a_draft_restores_what_is_running( def test_stopping_a_flow_takes_it_off_the_engine( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: - saved = client.put( - f"{PREFIX}/halted", headers=superuser_token_headers, json=a_flow("halted") - ).json() + saved = put_flow(client, superuser_token_headers, a_flow("halted"), "halted").json() client.post( f"{PREFIX}/halted/publish", headers=superuser_token_headers, @@ -286,9 +259,7 @@ def test_stopping_a_flow_takes_it_off_the_engine( def test_pausing_is_reported_back( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: - saved = client.put( - f"{PREFIX}/held", headers=superuser_token_headers, json=a_flow("held") - ).json() + saved = put_flow(client, superuser_token_headers, a_flow("held"), "held").json() client.post( f"{PREFIX}/held/publish", headers=superuser_token_headers, @@ -318,7 +289,7 @@ def test_unconnected_input_is_surfaced( ) -> None: flow = a_flow() flow["nodes"][0]["provides"] = [] # nothing produces "reading" any more - client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=flow) + put_flow(client, superuser_token_headers, flow, "demo") issues = client.post( f"{PREFIX}/demo/validate", headers=superuser_token_headers @@ -329,7 +300,7 @@ def test_unconnected_input_is_surfaced( def test_delete_flow( client: TestClient, superuser_token_headers: dict[str, str], db: Session ) -> None: - client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow()) + put_flow(client, superuser_token_headers, a_flow(), "demo") # Finished: a flow with a live run refuses to be deleted, which is its own # test below. db.add(Run(id="run-1", flow="demo", status="ok", created_at=datetime.now(UTC))) @@ -362,7 +333,7 @@ def test_delete_flow( def test_delete_flow_is_refused_while_a_run_is_live( client: TestClient, superuser_token_headers: dict[str, str], db: Session ) -> None: - client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow()) + put_flow(client, superuser_token_headers, a_flow(), "demo") db.add( Run(id="run-live", flow="demo", status="running", created_at=datetime.now(UTC)) ) @@ -390,8 +361,8 @@ def test_a_node_that_raises_answers_with_its_error( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: """A manual run fails the way a scheduled one does, not as a 500.""" - saved = client.put( - f"{PREFIX}/failing", headers=superuser_token_headers, json=a_flow("failing") + saved = put_flow( + client, superuser_token_headers, a_flow("failing"), "failing" ).json() client.put( f"{PREFIX}/failing/nodes/sensor/source", @@ -422,9 +393,7 @@ def test_acknowledging_a_failure_is_announced_on_the_bus( monkeypatch: pytest.MonkeyPatch, ) -> None: """Another browser holds the same marker; the event is what clears it.""" - saved = client.put( - f"{PREFIX}/acked", headers=superuser_token_headers, json=a_flow("acked") - ).json() + saved = put_flow(client, superuser_token_headers, a_flow("acked"), "acked").json() client.post( f"{PREFIX}/acked/publish", headers=superuser_token_headers, @@ -505,9 +474,7 @@ async def test_the_live_socket_ends_when_its_client_goes( def test_saving_a_header_moves_the_node_ports_in_the_draft( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: - saved = client.put( - f"{PREFIX}/header", headers=superuser_token_headers, json=a_flow("header") - ).json() + saved = put_flow(client, superuser_token_headers, a_flow("header"), "header").json() response = client.put( f"{PREFIX}/header/nodes/logger/source", @@ -539,9 +506,7 @@ def test_saving_a_header_moves_the_node_ports_in_the_draft( def test_a_shared_node_in_use_is_retired_only_with_detach( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: - client.put( - f"{PREFIX}/shared", headers=superuser_token_headers, json=a_flow("shared") - ) + put_flow(client, superuser_token_headers, a_flow("shared"), "shared") client.put( f"{PREFIX}/shared/nodes/sensor/source", headers=superuser_token_headers, @@ -571,9 +536,7 @@ def test_a_shared_node_only_the_published_flow_uses_is_kept( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: """An unshare in the draft leaves the running flow on the source.""" - client.put( - f"{PREFIX}/unshared", headers=superuser_token_headers, json=a_flow("unshared") - ) + put_flow(client, superuser_token_headers, a_flow("unshared"), "unshared") shared = client.post( f"{PREFIX}/unshared/nodes/sensor/share", headers=superuser_token_headers, @@ -603,9 +566,7 @@ def test_a_shared_source_write_is_audited( monkeypatch: pytest.MonkeyPatch, ) -> None: """It goes live at once, so it is recorded like a publish.""" - client.put( - f"{PREFIX}/audited", headers=superuser_token_headers, json=a_flow("audited") - ) + put_flow(client, superuser_token_headers, a_flow("audited"), "audited") client.post( f"{PREFIX}/audited/nodes/sensor/share", headers=superuser_token_headers, diff --git a/backend/tests/api/routes/test_observability.py b/backend/tests/api/routes/test_observability.py index fd123c8..5a56b42 100644 --- a/backend/tests/api/routes/test_observability.py +++ b/backend/tests/api/routes/test_observability.py @@ -96,10 +96,10 @@ def test_a_flow_that_cannot_run_makes_the_summary_degraded( controller = client.app.state.flow_controller before = controller.issues - before_flows = controller.store.list_flows + before_flows = controller.store.key_index # The looping flow has to be one the store lists, otherwise "running" # counts nothing either way and the assertion below proves nothing. - controller.store.list_flows = lambda: ["looping"] + controller.store.key_index = lambda: {"looping": "f_looping"} controller.issues = [ ValidationIssue( code="cycle", @@ -119,7 +119,7 @@ def test_a_flow_that_cannot_run_makes_the_summary_degraded( body = client.get(f"{PREFIX}/summary", headers=superuser_token_headers).json() finally: controller.issues = before - controller.store.list_flows = before_flows + controller.store.key_index = before_flows assert body["status"] == "degraded" assert body["flows"]["invalid"] == 1 diff --git a/backend/tests/api/routes/test_search.py b/backend/tests/api/routes/test_search.py index c63aab1..459bc09 100644 --- a/backend/tests/api/routes/test_search.py +++ b/backend/tests/api/routes/test_search.py @@ -7,6 +7,7 @@ from fastapi.testclient import TestClient from fluksio.api.routes.search import MASK, setting_text from fluksio.core.config import settings +from tests.utils.flows import put_dashboard, put_flow PREFIX = f"{settings.API_V1_STR}/search" FLOWS = f"{settings.API_V1_STR}/flows" @@ -22,19 +23,20 @@ def test_index_reaches_inside_flows_and_dashboards( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: """A node and a widget are the point: neither is on any list endpoint.""" - client.put( - f"{FLOWS}/searchable", - headers=superuser_token_headers, - json={ + put_flow( + client, + superuser_token_headers, + { "name": "searchable", "title": "Searchable", "nodes": [{"id": "sensor", "type": "python", "title": "Hall sensor"}], }, + "searchable", ) - client.put( - f"{DASHBOARDS}/hall", - headers=superuser_token_headers, - json={ + put_dashboard( + client, + superuser_token_headers, + { "name": "hall", "title": "Hall", "widgets": [ @@ -48,6 +50,7 @@ def test_index_reaches_inside_flows_and_dashboards( ], "version": 0, }, + "hall", ) entries = client.get(f"{PREFIX}/", headers=superuser_token_headers).json() @@ -69,15 +72,16 @@ def test_index_lists_a_flows_inputs_and_outputs( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: """Named by the qualified message, which is the label the canvas draws.""" - client.put( - f"{FLOWS}/fit", - headers=superuser_token_headers, - json={ + put_flow( + client, + superuser_token_headers, + { "name": "fit", "mode": "batch", "inputs": [{"spec": {"name": "epochs", "dtype": "int"}}], "outputs": ["accuracy"], }, + "fit", ) entries = client.get(f"{PREFIX}/", headers=superuser_token_headers).json() @@ -151,10 +155,10 @@ def test_index_lists_a_nodes_settings( normal_user_token_headers: dict[str, str], ) -> None: """A setting is found by its node and name, with its value beside it.""" - client.put( - f"{FLOWS}/tuned", - headers=superuser_token_headers, - json={ + put_flow( + client, + superuser_token_headers, + { "name": "tuned", "nodes": [ {"id": "gain", "type": "python", "params": {"factor": 2.5}}, @@ -169,6 +173,7 @@ def test_index_lists_a_nodes_settings( }, ], }, + "tuned", ) def settings_of(headers: dict[str, str]) -> tuple[dict[str, dict], str]: diff --git a/backend/tests/api/routes/test_sync.py b/backend/tests/api/routes/test_sync.py index ddacfc0..3cee2d1 100644 --- a/backend/tests/api/routes/test_sync.py +++ b/backend/tests/api/routes/test_sync.py @@ -38,6 +38,13 @@ def api(client: TestClient, superuser_token_headers: dict[str, str]) -> Client: return Client(http=client, token=superuser_token_headers["Authorization"][7:]) +def flow_id(api: Client, key: str) -> str: + """The id sync stored the flow with key *key* under.""" + found = api.find_flow(key) + assert found is not None, key + return str(found["id"]) + + def commits() -> int: """How many commits the flow store has made.""" from fluksio.flow.store import FlowStore @@ -62,14 +69,14 @@ def test_sync_stores_a_runnable_flow_with_its_origin(api, flows): assert [r.flow for r in reports] == ["train"] assert reports[0].created and reports[0].published - stored = api.get_flow("train") + stored = api.get_flow(flow_id(api, "train")) definition = stored["definition"] assert definition["origin"]["commit"] == "a1b2c3d4" assert [n["id"] for n in definition["nodes"]] == ["prepare", "fit", "evaluate"] assert not stored["has_draft"] # The store holds a complete definition, and the bodies say where the real # code is rather than being a copy of it. - source = api.get_source("train", "fit") + source = api.get_source(definition["id"], "fit") assert source.startswith(MARKER) assert "from myresearch.train import fit" in source @@ -79,7 +86,7 @@ def test_a_flow_read_by_another_process_syncs_as_the_flow_does(api, flows): carried = Declared(flows["train"].declaration()) sync([carried], api, origin=ORIGIN) - stored = api.get_flow("train")["definition"] + stored = api.get_flow(flow_id(api, "train"))["definition"] assert stored["origin"]["declared_in"] == "myresearch/pipeline.py" # Nothing differs from what the flow itself would have stored. @@ -90,7 +97,8 @@ def test_a_node_carries_the_modules_its_code_reaches(api, flows): """The shim names one function; the cache has to key on what it calls.""" sync([flows["train"]], api, origin=ORIGIN) - nodes = {n["id"]: n for n in api.get_flow("train")["definition"]["nodes"]} + definition = api.get_flow(flow_id(api, "train"))["definition"] + nodes = {n["id"]: n for n in definition["nodes"]} files = nodes["fit"]["code_files"] # The module the function is in, and the package whose `__init__` runs on @@ -156,14 +164,15 @@ def test_a_second_sync_changes_nothing_and_commits_nothing(api, flows): def test_sync_refuses_to_overwrite_a_canvas_edit(api, flows): sync([flows["train"]], api, origin=ORIGIN) - api.put_source("train", "evaluate", "def process(weights):\n return {}\n") + train = flow_id(api, "train") + api.put_source(train, "evaluate", "def process(weights):\n return {}\n") with pytest.raises(SyncError, match="edited on the canvas"): sync([flows["train"]], api, origin=ORIGIN) reports = sync([flows["train"]], api, origin=ORIGIN, force=True) assert "evaluate" in reports[0].changed - assert api.get_source("train", "evaluate").startswith(MARKER) + assert api.get_source(train, "evaluate").startswith(MARKER) def test_an_interrupted_sync_finishes_on_the_next_one(api, flows): @@ -177,23 +186,23 @@ def test_an_interrupted_sync_finishes_on_the_next_one(api, flows): target = flows["train"] target.name = "resumed" try: - # Exactly what the document PUT leaves behind before any body is written. - api.put_flow(target.document(ORIGIN) | {"version": 1}) - assert api.get_source_entry("resumed", "fit")["missing"] + # Exactly what creating the document leaves before any body is written. + resumed = api.create_flow(target.document(ORIGIN))["definition"]["id"] + assert api.get_source_entry(resumed, "fit")["missing"] reports = sync([target], api, origin=ORIGIN) assert "fit" in reports[0].changed - assert api.get_source("resumed", "fit").startswith(MARKER) - assert not api.get_source_entry("resumed", "fit")["missing"] - assert not api.get_flow("resumed")["has_draft"] + assert api.get_source(resumed, "fit").startswith(MARKER) + assert not api.get_source_entry(resumed, "fit")["missing"] + assert not api.get_flow(resumed)["has_draft"] finally: target.name = "train" def test_sync_refuses_a_flow_it_did_not_create(api, flows): - api.put_flow({"name": "drawn", "version": 1, "nodes": [], "mode": "batch"}) - api.publish("drawn", 1) + drawn = api.create_flow({"name": "drawn", "nodes": [], "mode": "batch"}) + api.publish(drawn["definition"]["id"], 1) theirs = flows["train"] theirs.name = "drawn" @@ -207,7 +216,8 @@ def test_sync_refuses_a_flow_it_did_not_create(api, flows): def test_use_stores_the_rewiring_it_was_given(api, flows): sync([flows["finetune"]], api, origin=ORIGIN) - nodes = {n["id"]: n for n in api.get_flow("finetune")["definition"]["nodes"]} + definition = api.get_flow(flow_id(api, "finetune"))["definition"] + nodes = {n["id"]: n for n in definition["nodes"]} assert [p["name"] for p in nodes["fit"]["requires"]] == ["augmented", "lr"] assert nodes["fit"]["params"] == {"epochs": 3} diff --git a/backend/tests/api/routes/test_wren_train.py b/backend/tests/api/routes/test_wren_train.py index 6a2b73f..cb53972 100644 --- a/backend/tests/api/routes/test_wren_train.py +++ b/backend/tests/api/routes/test_wren_train.py @@ -16,6 +16,7 @@ from fluksio.core.config import settings from fluksio.core.db import engine as db_engine from fluksio.models import Run, RunArtifact from tests.flow.conftest import FakeWren +from tests.utils.flows import put_flow FLOW = "wren_house" PREFIX = f"{settings.API_V1_STR}/flows/{FLOW}/nodes" @@ -70,10 +71,8 @@ def house( {"id": "sensor", "type": "python"}, ] url = f"{settings.API_V1_STR}/flows/{FLOW}" - saved = client.put( - url, headers=superuser_token_headers, json={"name": FLOW, "nodes": nodes} - ) - assert saved.status_code == 200, saved.text + saved = put_flow(client, superuser_token_headers, {"name": FLOW, "nodes": nodes}) + assert saved.is_success, saved.text yield client.delete(url, headers=superuser_token_headers) diff --git a/backend/tests/flow/test_dashboards.py b/backend/tests/flow/test_dashboards.py index bf4636f..9495276 100644 --- a/backend/tests/flow/test_dashboards.py +++ b/backend/tests/flow/test_dashboards.py @@ -19,7 +19,8 @@ from fluksio.flow.nodes import Node from fluksio.flow.pipeline import Pipeline from fluksio.flow.schemas import FlowDef, NodeDef from fluksio.flow.state import MemoryState -from fluksio.flow.store import FlowStore, StaleVersion +from fluksio.flow.store import ISSUED_ID, FlowStore, StaleVersion +from tests.utils.stores import created @pytest.fixture @@ -93,25 +94,25 @@ def test_discarding_leaves_what_is_published(store: DashboardStore): def test_a_new_dashboard_is_a_draft_until_it_is_published(store: DashboardStore): """Creating one does not put it on a wall; publishing is what does.""" - created = store.write_draft(default_dashboard("house"), 0) + made = created(store, default_dashboard("house")) assert not store.is_published("house") with pytest.raises(DashboardNotFound): store.read("house") - assert store.read("house", draft=True).title == created.title + assert store.read("house", draft=True).title == made.title # Listed all the same, so the editor can find what it just made. assert [(d.name, d.has_draft, d.version) for d in store.list()] == [ - ("house", True, created.version) + ("house", True, made.version) ] - store.publish("house", created.version) + store.publish("house", made.version) assert store.is_published("house") - assert store.read("house").title == created.title + assert store.read("house").title == made.title def test_an_unpublished_dashboard_can_be_deleted(store: DashboardStore): - store.write_draft(default_dashboard("house"), 0) + created(store, default_dashboard("house")) store.delete("house") assert not store.exists("house") @@ -128,7 +129,8 @@ def test_a_duplicate_is_an_unpublished_copy_of_the_working_copy( copy = store.duplicate("house") assert (copy.name, copy.title, copy.version) == ("house_copy", "Edited (copy)", 1) - assert store.has_draft("house_copy") and not store.is_published("house_copy") + assert ISSUED_ID.fullmatch(copy.id) + assert store.has_draft(copy.id) and not store.is_published(copy.id) assert store.read("house", draft=True).version == edited.version assert store.duplicate("house").name == "house_copy_2" diff --git a/backend/tests/flow/test_drafts.py b/backend/tests/flow/test_drafts.py index bc833fb..1c163ba 100644 --- a/backend/tests/flow/test_drafts.py +++ b/backend/tests/flow/test_drafts.py @@ -8,7 +8,8 @@ import pytest from fluksio.flow.controller import FlowController from fluksio.flow.messages import MessageSpec from fluksio.flow.schemas import FlowDef, FlowOrigin, NodeDef -from fluksio.flow.store import FlowStore, StaleVersion +from fluksio.flow.store import ISSUED_ID, FlowStore, StaleVersion +from tests.utils.stores import created SOURCE = "def process():\n return {}\n" EDITED = "def process():\n return {'temp': 1}\n" @@ -28,7 +29,7 @@ def a_flow(title: str = "") -> FlowDef: def test_a_draft_is_invisible_to_the_engine(store: FlowStore): - store.write_draft(a_flow(), 0) + created(store, a_flow()) assert store.has_draft("heating") assert store.list_flows() == ["heating"] # the editor sees it @@ -36,7 +37,7 @@ def test_a_draft_is_invisible_to_the_engine(store: FlowStore): def test_the_version_counter_advances_per_save(store: FlowStore): - first = store.write_draft(a_flow(), 0) + first = created(store, a_flow()) assert first.version == 1 second = store.write_draft(a_flow(title="Warm"), first.version) @@ -44,14 +45,14 @@ def test_the_version_counter_advances_per_save(store: FlowStore): def test_resaving_identical_content_neither_bumps_nor_commits(store: FlowStore): - stored = store.write_draft(a_flow(), 0) + stored = created(store, a_flow()) again = store.write_draft(a_flow(), stored.version) assert again.version == stored.version def test_a_stale_write_is_refused(store: FlowStore): - store.write_draft(a_flow(), 0) + created(store, a_flow()) store.write_draft(a_flow(title="Mine"), 1) # A second client still holding version 1 must not overwrite version 2. @@ -61,7 +62,7 @@ def test_a_stale_write_is_refused(store: FlowStore): def test_publishing_promotes_the_draft_and_its_sources(store: FlowStore): - stored = store.write_draft(a_flow(title="Warm"), 0) + stored = created(store, a_flow(title="Warm")) store.write_node_source("heating", "sensor", EDITED, draft=True) published = store.publish_flow("heating", stored.version) @@ -74,7 +75,7 @@ def test_publishing_promotes_the_draft_and_its_sources(store: FlowStore): def test_publishing_a_stale_draft_is_refused(store: FlowStore): - store.write_draft(a_flow(), 0) + created(store, a_flow()) with pytest.raises(StaleVersion): store.publish_flow("heating", 99) @@ -150,7 +151,7 @@ def test_a_worker_on_a_node_without_code_is_reported(store: FlowStore): def test_a_node_being_written_in_the_editor_is_not_reported(store: FlowStore): """A draft legitimately has no published body yet — that is what a draft is.""" - store.write_draft(a_flow(), 0) + created(store, a_flow()) controller = FlowController(store) assert controller.preview("heating").issues == [] @@ -175,12 +176,14 @@ def test_a_duplicate_is_a_local_draft_of_what_the_editor_shows(store: FlowStore) copy = store.duplicate_flow("heating") assert (copy.name, copy.title, copy.version) == ("heating_copy", "Warm (copy)", 1) + # An id of its own, which its key is not. + assert ISSUED_ID.fullmatch(copy.id) assert copy.origin is None assert copy.nodes[1].requires[0].name == "heating_copy.temp" - assert store.has_draft("heating_copy") and not store.is_published("heating_copy") - assert store.read_node_source("heating_copy", "sensor", draft=True) == EDITED - assert not (store.root / "heating_copy" / "runtime.json").exists() + assert store.has_draft(copy.id) and not store.is_published(copy.id) + assert store.read_node_source(copy.id, "sensor", draft=True) == EDITED + assert not (store.root / copy.id / "runtime.json").exists() assert store.duplicate_flow("heating").name == "heating_copy_2" - store.publish_flow("heating_copy", copy.version) - assert store.read_node_source("heating_copy", "sensor") == EDITED + store.publish_flow(copy.id, copy.version) + assert store.read_node_source(copy.id, "sensor") == EDITED diff --git a/backend/tests/flow/test_flow_ids.py b/backend/tests/flow/test_flow_ids.py new file mode 100644 index 0000000..50895d8 --- /dev/null +++ b/backend/tests/flow/test_flow_ids.py @@ -0,0 +1,157 @@ +"""The engine runs a flow by its key and finds it by its id. + +Every test runs twice, once with the id equal to the key the way a flow from +before ids has it, and once with them apart. A mix-up of the two is invisible +in the first run, which is why the second one exists. +""" + +import asyncio +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from fluksio.flow.controller import HOOK_PREFIX, FlowController +from fluksio.flow.messages import DType, MessageSpec +from fluksio.flow.queue import MemoryWorkQueue, WorkItem +from fluksio.flow.schemas import FlowDef, NodeDef +from fluksio.flow.store import FlowStore + +SENSOR = "def process():\n return {'reading': 21.5}\n" +DOUBLE = "def process(reading):\n return {'doubled': reading * 2}\n" + + +@pytest.fixture(params=["house", "x_house"], ids=["id=key", "id!=key"]) +def flow_id(request: pytest.FixtureRequest) -> str: + return str(request.param) + + +@pytest.fixture +def store(tmp_path: Path, flow_id: str) -> FlowStore: + store = FlowStore(tmp_path / "flows") + store.write_flow( + FlowDef( + id=flow_id, + name="house", + nodes=[ + NodeDef( + id="sensor", + provides=[MessageSpec(name="reading", dtype=DType.FLOAT)], + ), + NodeDef( + id="double", + requires=[MessageSpec(name="reading", dtype=DType.FLOAT)], + provides=[MessageSpec(name="doubled", dtype=DType.FLOAT)], + ), + NodeDef( + id="hook", + type="http", + params={"url": "/tick", "method": "POST"}, + provides=[MessageSpec(name="tick", dtype=DType.FLOAT)], + ), + ], + ) + ) + store.write_node_source(flow_id, "sensor", SENSOR) + store.write_node_source(flow_id, "double", DOUBLE) + return store + + +def test_a_flow_runs_under_its_key_from_the_code_its_id_holds( + store: FlowStore, flow_id: str +): + engine = FlowController(store) + asyncio.run(engine.reload()) + + entry = engine.loaded["house.double"] + assert (entry.flow, entry.flow_id) == ("house", flow_id) + # Read by the id: by the key it would be the template, and missing. + assert not entry.missing_source and entry.status.value == "active" + assert engine.loaded["house.hook"].node.url == f"{HOOK_PREFIX}/{flow_id}/tick" + + engine.run_flow("house") + + assert engine.values()["house.doubled"]["value"] == 43.0 + assert {s.id for s in engine.node_statuses("house")} >= {"house.double"} + + +def test_stopping_writes_under_the_id_and_gates_the_key(store: FlowStore, flow_id: str): + engine = FlowController(store) + + async def scenario() -> None: + await engine.reload() + await engine.set_enabled(flow_id, "house", False) + assert engine.disabled == {"house"} + assert not store.read_enabled(flow_id) + # Read back by id when everything is built again. + await engine.reload() + assert engine.disabled == {"house"} + + asyncio.run(scenario()) + assert (store.root / flow_id / "runtime.json").exists() + + +def test_a_flow_rebuilt_by_id_is_taken_out_by_its_key_once_gone( + store: FlowStore, flow_id: str +): + engine = FlowController(store) + + async def scenario() -> None: + await engine.reload() + assert "house.sensor" in engine.loaded + await engine.reload_flow(flow_id) + assert "house.sensor" in engine.loaded + + store.delete_flow(flow_id) + await engine.reload_flow(flow_id) + + asyncio.run(scenario()) + assert not [node for node in engine.loaded if node.startswith("house.")] + assert engine.pipeline is not None and not engine.pipeline.flow_nodes("house") + + +def test_a_draft_previews_by_id_and_reports_by_key(store: FlowStore, flow_id: str): + draft = store.read_flow(flow_id).model_copy(update={"title": "Edited"}) + store.write_draft(draft, draft.version) + engine = FlowController(store) + + preview = engine.preview(flow_id) + + assert {node.id for node in preview.nodes} == { + "house.sensor", + "house.double", + "house.hook", + } + assert all(node.status == "active" for node in preview.nodes) + engine.run_preview(flow_id) + assert engine.state.get("house.doubled") == 43.0 + + +def test_what_a_key_leaves_behind_is_found_and_forgotten(store: FlowStore): + engine = FlowController(store) + queue = MemoryWorkQueue() + engine.execution = SimpleNamespace(queue=queue) # type: ignore[assignment] + engine.state.set("house.reading", 1.0) + engine.state.set("__node__:house.double:last", 2) + engine.state.set("housework.reading", 3.0) + queue.add_delayed(WorkItem(kind="flush", node="house.double", flow="house"), 9e9) + queue.park("house", WorkItem(kind="cascade", node="house.sensor", flow="house")) + # Another flow's work, carrying one of this key's messages. + queue.add( + WorkItem( + kind="cascade", node="garden.pump", flow="garden", outputs={"house.x": 1} + ) + ) + + found = engine.key_residue("house") + + assert "house.reading" in found and "__node__:house.double:last" in found + assert "housework.reading" not in found + assert sum("queued work" in what for what in found) == 3 + assert engine.key_residue("garden") == ["queued work of 'garden.pump'"] + + engine.forget_flow("house") + + # What the other flow queued is its own, and runs as it would have. + assert engine.key_residue("house") == ["queued work of 'garden.pump'"] + assert engine.state.get("housework.reading") == 3.0 diff --git a/backend/tests/flow/test_identity.py b/backend/tests/flow/test_identity.py index be7dc84..2b3d244 100644 --- a/backend/tests/flow/test_identity.py +++ b/backend/tests/flow/test_identity.py @@ -8,6 +8,7 @@ is why the migration is driven through crashes, refusals and stores without git. import json import logging import subprocess +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import pytest @@ -18,17 +19,23 @@ from fluksio.flow.dashboards import ( DASHBOARD_DIR, DashboardDef, DashboardExists, + DashboardNotFound, DashboardStore, default_dashboard, + results_dashboard, ) from fluksio.flow.identity import COMMIT_MESSAGE, IdsRefused, assign_ids -from fluksio.flow.schemas import FlowDef +from fluksio.flow.schemas import FlowCreate, FlowDef from fluksio.flow.store import ( ISSUED_ID, FlowExists, + FlowNotFound, FlowStore, IdentityChange, + IdTaken, + RecordedKey, ReservedKey, + StaleVersion, new_id, ) @@ -37,23 +44,51 @@ from fluksio.flow.store import ( # ----------------------------------------------------------------------------- +class Graves: + """Tombstones and history, as `fluksio.flow.tombstones.Tombstones` keeps + them in the database, which store tests have none of.""" + + def __init__(self) -> None: + self.stones: dict[str, str] = {} + #: Ids that history rows name. + self.history: set[str] = set() + + def bury(self, flow_id: str, name: str) -> None: + self.stones[flow_id] = name + + def lift(self, flow_id: str) -> None: + self.stones.pop(flow_id, None) + + def recorded(self, flow_id: str) -> str | None: + if flow_id in self.stones: + return self.stones[flow_id] + return flow_id if flow_id in self.history else None + + @pytest.fixture -def store(tmp_path: Path) -> FlowStore: - return FlowStore(tmp_path / "flows") +def graves() -> Graves: + return Graves() -def test_a_flow_is_given_its_key_as_id_and_keeps_both(store: FlowStore): - created = store.write_draft(FlowDef(name="heating"), 0) - assert (created.id, created.name) == ("heating", "heating") +@pytest.fixture +def store(tmp_path: Path, graves: Graves) -> FlowStore: + return FlowStore(tmp_path / "flows", tombstones=graves) # type: ignore[arg-type] - published = store.publish_flow("heating", created.version) + +def test_a_new_flow_gets_an_issued_id_and_keeps_it(store: FlowStore): + created = store.create(FlowDef(name="heating")) + flow_id = created.id + assert ISSUED_ID.fullmatch(flow_id) and flow_id.startswith("f_") + assert created.name == "heating" and created.version == 1 + assert store.key_index() == {"heating": flow_id} + + published = store.publish_flow(flow_id, created.version) store.write_draft(published.model_copy(update={"title": "Warm"}), 1) - discarded = store.discard_draft("heating") + discarded = store.discard_draft(flow_id) - assert json.loads((store.root / "heating" / "flow.json").read_text())["id"] == ( - "heating" - ) - assert (discarded.id, discarded.name) == ("heating", "heating") + on_disk = json.loads((store.root / flow_id / "flow.json").read_text()) + assert (on_disk["id"], on_disk["name"]) == (flow_id, "heating") + assert (discarded.id, discarded.name) == (flow_id, "heating") def test_a_document_from_before_ids_reads_with_its_directory_as_id(store: FlowStore): @@ -70,9 +105,13 @@ def test_a_document_from_before_ids_reads_with_its_directory_as_id(store: FlowSt assert store.write_draft(FlowDef(name="heating", title="Warm"), 1).version == 2 -def test_a_save_cannot_change_an_id_or_a_key(store: FlowStore): - store.write_flow(FlowDef(id="x_heating", name="heating")) +def test_a_save_neither_creates_nor_changes_an_id_or_a_key(store: FlowStore): + with pytest.raises(FlowNotFound): + store.write_draft(FlowDef(name="nobody"), 0) + with pytest.raises(DashboardNotFound): + DashboardStore(store).write_draft(default_dashboard("nobody"), 0) + store.write_flow(FlowDef(id="x_heating", name="heating")) with pytest.raises(IdentityChange): store.write_draft(FlowDef(id="x_heating", name="warmth"), 1) with pytest.raises(IdentityChange): @@ -87,56 +126,218 @@ def test_a_save_cannot_change_an_id_or_a_key(store: FlowStore): def test_a_key_belongs_to_one_flow_drafts_included(store: FlowStore): """A draft nobody published holds its key, and a delete lets it go.""" - store.write_draft(FlowDef(id="x_heating", name="heating"), 0) - assert store.key_index() == {"heating": "x_heating"} + first = store.create(FlowDef(name="heating")) + assert store.key_index() == {"heating": first.id} with pytest.raises(FlowExists): - store.write_draft(FlowDef(name="heating"), 0) + store.create(FlowDef(name="heating")) + with pytest.raises(FlowExists): + store.create(FlowDef(name="heating"), "x_heating") with pytest.raises(FlowExists): store.write_flow(FlowDef(id="y_heating", name="heating")) - store.delete_flow("x_heating") - assert store.write_draft(FlowDef(name="heating"), 0).id == "heating" - assert store.key_index() == {"heating": "heating"} + store.delete_flow(first.id) + second = store.create(FlowDef(name="heating")) + assert second.id != first.id + assert store.key_index() == {"heating": second.id} def test_a_key_belongs_to_one_dashboard_drafts_included(store: FlowStore): dashboards = DashboardStore(store) - dashboards.write_draft(DashboardDef(id="x_home", name="home"), 0) + first = dashboards.create(default_dashboard("home")) + assert ISSUED_ID.fullmatch(first.id) and first.id.startswith("d_") # Its own id space: a flow may share the key. - store.write_draft(FlowDef(name="home"), 0) + store.create(FlowDef(name="home")) with pytest.raises(DashboardExists): - dashboards.write_draft(default_dashboard("home"), 0) + dashboards.create(default_dashboard("home")) with pytest.raises(DashboardExists): dashboards.write(DashboardDef(id="y_home", name="home")) - dashboards.delete("x_home") - assert dashboards.write(default_dashboard("home")).id == "home" + dashboards.delete(first.id) + assert dashboards.create(default_dashboard("home"), "home").id == "home" -def test_a_copy_takes_a_key_nobody_holds(store: FlowStore): +def test_a_pin_is_an_id_nothing_holds(store: FlowStore): + store.create(FlowDef(name="heating"), "x_heating") + with pytest.raises(IdTaken): + store.create(FlowDef(name="cooling"), "x_heating") + # Another flow's key would make naming either ambiguous. + with pytest.raises(IdTaken): + store.create(FlowDef(name="cooling"), "heating") + # And a directory left behind would hand the new flow what is in it. + (store.root / "leftover" / "nodes").mkdir(parents=True) + with pytest.raises(IdTaken): + store.create(FlowDef(name="leftover"), "leftover") + dashboards = DashboardStore(store) + dashboards.create(default_dashboard("home"), "x_home") + with pytest.raises(IdTaken): + dashboards.create(default_dashboard("house"), "x_home") + + +def test_an_issued_id_avoids_whatever_history_or_a_directory_holds( + store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch +): + candidates = iter(["f_aaaaaaaa", "f_bbbbbbbb", "f_cccccccc", "f_dddddddd"]) + monkeypatch.setattr( + "fluksio.flow.store.new_id", + lambda kind, taken: next(c for c in candidates if not taken(c)), + ) + graves.bury("f_aaaaaaaa", "gone") + graves.history.add("f_bbbbbbbb") + (store.root / "f_cccccccc").mkdir() + + assert store.create(FlowDef(name="heating")).id == "f_dddddddd" + + +def test_a_deleted_flow_is_buried_before_its_files_go( + store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch +): + flow_id = store.create(FlowDef(name="heating")).id + + def crash(path: Path) -> None: + raise OSError("disk gone") + + monkeypatch.setattr("fluksio.flow.store.shutil.rmtree", crash) + with pytest.raises(OSError): + store.delete_flow(flow_id) + # The name is safe, and the live document still wins over it. + assert graves.stones == {flow_id: "heating"} + assert store.read_flow(flow_id, draft=True).name == "heating" + + monkeypatch.undo() + assert store.delete_flow(flow_id) == "heating" + assert graves.stones == {flow_id: "heating"} + assert not store.exists(flow_id) + + +def test_a_delete_that_stops_after_the_files_has_kept_the_name( + store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch +): + flow_id = store.create(FlowDef(name="heating")).id + + def killed(message: str) -> None: + raise OSError("killed before the commit") + + monkeypatch.setattr(store, "_commit", killed) + with pytest.raises(OSError): + store.delete_flow(flow_id) + monkeypatch.undo() + + assert not store.exists(flow_id) + assert graves.stones == {flow_id: "heating"} + with pytest.raises(FlowNotFound): + store.delete_flow(flow_id) + + +def test_a_pinned_restore_takes_up_its_history_under_its_own_key( + store: FlowStore, graves: Graves +): + flow_id = store.create(FlowDef(name="heating")).id + store.delete_flow(flow_id) + + with pytest.raises(RecordedKey): + store.create(FlowDef(name="warmth"), flow_id) + assert graves.stones == {flow_id: "heating"} + + restored = store.create(FlowDef(name="heating"), flow_id) + assert restored.id == flow_id and graves.stones == {} + + # History from before tombstones is filed under an id that was its key. + graves.history.add("legacy") + with pytest.raises(RecordedKey): + store.create(FlowDef(name="other"), "legacy") + assert store.create(FlowDef(name="legacy"), "legacy").id == "legacy" + + +def test_a_key_reused_after_a_delete_starts_a_new_history( + store: FlowStore, graves: Graves +): + old = store.create(FlowDef(name="heating")).id + store.delete_flow(old) + + new = store.create(FlowDef(name="heating")).id + + assert new != old + assert graves.stones == {old: "heating"} + + +def test_two_creates_of_one_key_make_one_flow(store: FlowStore): + def create(_: int) -> str: + try: + return store.create(FlowDef(name="heating")).id + except FlowExists: + return "" + + with ThreadPoolExecutor(max_workers=8) as pool: + made = [flow_id for flow_id in pool.map(create, range(8)) if flow_id] + + assert len(made) == 1 + assert store.list_flows() == made + + +def test_two_results_dashboards_of_one_flow_make_one(store: FlowStore): + dashboards = DashboardStore(store) + flow = FlowDef(id="x_lab", name="lab", mode="batch") + + def generate(_: int) -> str: + try: + return dashboards.create(results_dashboard(flow), "", True).id + except DashboardExists: + return "" + + with ThreadPoolExecutor(max_workers=4) as pool: + made = [board for board in pool.map(generate, range(4)) if board] + + assert len(made) == 1 + assert dashboards.read(made[0]).name == "lab_results" + + +def test_a_discard_leaves_no_old_draft_version_to_match(store: FlowStore): + flow_id = store.create(FlowDef(name="heating")).id + published = store.publish_flow(flow_id, 1) + held = store.write_draft(published.model_copy(update={"title": "A"}), 1) + store.write_draft(held.model_copy(update={"title": "B"}), held.version) + + restored = store.discard_draft(flow_id) + + # Above every draft version handed out, so an editor holding one is stale. + assert restored.version == 4 and restored.name == "heating" + with pytest.raises(StaleVersion): + store.write_draft(held.model_copy(update={"title": "late"}), held.version) + assert store.write_draft(restored.model_copy(update={"title": "C"}), 4).version == 5 + + dashboards = DashboardStore(store) + board = dashboards.create(default_dashboard("home")) + dashboards.publish(board.id, 1) + dashboards.write_draft(dashboards.read(board.id).model_copy(), 1) + assert dashboards.discard_draft(board.id).version == 3 + # A direct write goes above a draft too. + dashboards.write_draft(dashboards.read(board.id).model_copy(), 3) + assert dashboards.write(dashboards.read(board.id)).version == 5 + + +def test_a_copy_gets_an_id_of_its_own_and_a_key_nobody_holds(store: FlowStore): store.write_flow(FlowDef(name="heating")) - store.write_draft(FlowDef(id="x_other", name="heating_copy"), 0) + store.create(FlowDef(name="heating_copy")) dashboards = DashboardStore(store) dashboards.write(default_dashboard("home")) - dashboards.write_draft(DashboardDef(id="x_other", name="home_copy"), 0) + dashboards.create(default_dashboard("home_copy")) - copy = store.duplicate_flow("heating") + copy = store.duplicate_flow("heating", busy=lambda key: key == "heating_copy_2") board = dashboards.duplicate("home") - assert (copy.id, copy.name) == ("heating_copy_2", "heating_copy_2") - assert (board.id, board.name) == ("home_copy_2", "home_copy_2") + # `_copy_2` still had work behind it, so the copy went past it. + assert copy.name == "heating_copy_3" and ISSUED_ID.fullmatch(copy.id) + assert board.name == "home_copy_2" and ISSUED_ID.fullmatch(board.id) def test_a_new_key_shaped_like_an_issued_id_is_refused(store: FlowStore): with pytest.raises(ReservedKey): - store.write_draft(FlowDef(name="f_k3q7m2xa"), 0) + store.create(FlowDef(name="f_k3q7m2xa")) with pytest.raises(ReservedKey): - DashboardStore(store).write_draft(default_dashboard("d_k3q7m2xa"), 0) - assert store.write_draft(FlowDef(name="f_k3q7m2xa_lab"), 0).name == ( - "f_k3q7m2xa_lab" - ) + DashboardStore(store).create(default_dashboard("d_k3q7m2xa")) + assert store.create(FlowDef(name="f_k3q7m2xa_lab")).name == "f_k3q7m2xa_lab" def test_an_existing_key_shaped_like_an_issued_id_stays_saveable(store: FlowStore): @@ -515,3 +716,10 @@ def test_ls_lists_ids_keys_and_kinds(tmp_path: Path, capsys): assert rows[0] == ["ID", "KEY", "KIND"] assert ["home", "home", "dashboard"] in rows and ["home", "home", "flow"] in rows assert ["warmth", "radiators", "flow"] in rows + + +def test_what_a_create_takes_is_a_document_but_for_its_version(): + """`FlowCreate` lists the fields again; one added to `FlowDef` belongs there.""" + assert set(FlowCreate.model_fields) == set(FlowDef.model_fields) - {"version"} + with pytest.raises(ValueError): + FlowCreate(name="Not A Key") diff --git a/backend/tests/flow/test_library.py b/backend/tests/flow/test_library.py index 501fff1..eb67bf2 100644 --- a/backend/tests/flow/test_library.py +++ b/backend/tests/flow/test_library.py @@ -9,6 +9,7 @@ from fluksio.flow.controller import FlowController from fluksio.flow.messages import MessageSpec from fluksio.flow.schemas import FlowDef, NodeDef from fluksio.flow.store import FlowStore, LibExists, LibNotFound +from tests.utils.stores import created SOURCE = "def process():\n return {'temp': 1}\n" @@ -79,7 +80,7 @@ def test_a_shared_source_change_rebuilds_only_what_is_published(store: FlowStore # Shared in a draft that was never published. cooling = a_flow("cooling") cooling.nodes[0].source_ref = "read_temp" - store.write_draft(cooling, 0) + created(store, cooling) controller = FlowController(store) rebuilt: list[str] = [] @@ -131,7 +132,7 @@ def test_retiring_a_shared_source_leaves_each_user_a_copy(store: FlowStore): # Shared in a draft that was never published. cooling = a_flow("cooling") cooling.nodes[0].source_ref = "read_temp" - store.write_draft(cooling, 0) + created(store, cooling) store.detach_lib("read_temp") diff --git a/backend/tests/flow/test_node_settings.py b/backend/tests/flow/test_node_settings.py index dd855df..323fb21 100644 --- a/backend/tests/flow/test_node_settings.py +++ b/backend/tests/flow/test_node_settings.py @@ -15,6 +15,7 @@ from fluksio.flow.pipeline import Pipeline from fluksio.flow.schemas import FlowDef, NodeDef from fluksio.flow.state import MemoryState from fluksio.flow.store import FlowStore +from tests.utils.stores import created SOURCE = "def process(reading, factor):\n return {'scaled': reading * factor}\n" @@ -53,7 +54,7 @@ def test_a_setting_reaches_the_function_as_a_keyword_argument(store: FlowStore): def test_a_setting_named_after_a_port_is_refused(store: FlowStore): - store.write_draft(a_flow(reading=3), 0) + created(store, a_flow(reading=3)) store.write_node_source("house", "scale", SOURCE, draft=True) controller = FlowController(store) @@ -81,7 +82,7 @@ READING = [MessageSpec(name="reading", dtype=DType.FLOAT)] def _preview(store: FlowStore, flow: FlowDef, source: str | None): - store.write_draft(flow, 0) + created(store, flow) if source is not None: store.write_node_source("house", "scale", source, draft=True) return FlowController(store).preview("house").nodes[0] diff --git a/backend/tests/flow/test_queue.py b/backend/tests/flow/test_queue.py index cc2a2f6..b0681b4 100644 --- a/backend/tests/flow/test_queue.py +++ b/backend/tests/flow/test_queue.py @@ -680,3 +680,28 @@ def test_done_markers_do_not_grow_without_bound(): # The newest are what redelivery would ask about. assert queue.was_done(f"entry-{queue_module.DONE_MARKERS + 499}", "f.node") assert not queue.was_done("entry-0", "f.node") + + +def test_clearing_a_flow_drops_its_work_and_nothing_else(): + """A new flow under the same key must never run what the old one queued.""" + queue = MemoryWorkQueue() + queue.add(WorkItem(kind="cascade", node="gone.a", flow="gone")) + # Another flow's work carrying one of its messages, and a run, which is + # filed under a flow's id rather than its key: neither is the flow's. + queue.add( + WorkItem(kind="cascade", node="kept.a", flow="kept", outputs={"gone.x": 1}) + ) + queue.add(WorkItem(kind="run", node="", flow="gone", run_id="r1")) + queue.add_delayed(WorkItem(kind="flush", node="gone.b", flow="gone"), 9e9) + queue.add_delayed(WorkItem(kind="flush", node="kept.b", flow="kept"), 9e9) + queue.park("gone", WorkItem(kind="cascade", node="gone.c", flow="gone")) + assert len(queue.live_items()) == 6 + + queue.clear_flow("gone") + + assert sorted((i.kind, i.node) for i in queue.live_items()) == [ + ("cascade", "kept.a"), + ("flush", "kept.b"), + ("run", ""), + ] + assert queue.next_due() == 9e9 diff --git a/backend/tests/flow/test_redis_queue.py b/backend/tests/flow/test_redis_queue.py index a6fd184..13d4847 100644 --- a/backend/tests/flow/test_redis_queue.py +++ b/backend/tests/flow/test_redis_queue.py @@ -66,3 +66,30 @@ def test_a_refused_stream_write_keeps_the_timer_for_retry(queue): assert queue.move_due(200.0) == 1 assert queue.move_due(200.0) == 0 assert [item.node for item in queue.claim(10, 10, due_only=True)] == ["f.stop"] + + +def test_clearing_a_flow_drops_its_queued_claimed_and_held_work(queue): + queue.add(WorkItem(kind="cascade", node="gone.p", flow="gone")) + (claimed,) = queue.claim(1, 10) + assert claimed.node == "gone.p" + queue.add(WorkItem(kind="cascade", node="gone.a", flow="gone")) + queue.add( + WorkItem(kind="cascade", node="kept.a", flow="kept", outputs={"gone.x": 1}) + ) + queue.add(WorkItem(kind="run", node="", flow="gone", run_id="r1")) + queue.add_delayed(WorkItem(kind="flush", node="gone.b", flow="gone"), 9e9) + queue.add_delayed(WorkItem(kind="flush", node="kept.b", flow="kept"), 9e9) + queue.park("gone", WorkItem(kind="cascade", node="gone.c", flow="gone")) + assert len(queue.live_items()) == 7 + + queue.clear_flow("gone") + + assert sorted((i.kind, i.node) for i in queue.live_items()) == [ + ("cascade", "kept.a"), + ("flush", "kept.b"), + ("run", ""), + ] + # Nothing comes back to be run again: not redelivered, not reclaimed. + assert queue.reclaim_stale(0) == [] + queue.ack(claimed) + assert [i.node for i in queue.claim(10, 10)] == ["kept.a", ""] diff --git a/backend/tests/flow/test_runtime_control.py b/backend/tests/flow/test_runtime_control.py index ff31c03..52f14c1 100644 --- a/backend/tests/flow/test_runtime_control.py +++ b/backend/tests/flow/test_runtime_control.py @@ -318,7 +318,7 @@ def test_stopping_one_flow_does_not_touch_the_others( before = {name: engine.loaded[name].node for name in ("a.io", "b.io")} lifecycle.clear() - await engine.set_enabled("a", False) + await engine.set_enabled("a", "a", False) assert lifecycle == ["stop a.io"] # Still built, still the same objects: only the lifecycle changed. @@ -326,14 +326,14 @@ def test_stopping_one_flow_does_not_touch_the_others( assert engine.disabled == {"a"} lifecycle.clear() - await engine.set_enabled("a", True) + await engine.set_enabled("a", "a", True) assert lifecycle == ["start a.io"] assert {name: engine.loaded[name].node for name in ("a.io", "b.io")} == before # Started while running is a restart: the cancelled loops come back. lifecycle.clear() - await engine.set_enabled("a", True) + await engine.set_enabled("a", "a", True) assert lifecycle == ["stop a.io", "start a.io"] diff --git a/backend/tests/test_artifacts.py b/backend/tests/test_artifacts.py index 2d8fd5e..cbb5068 100644 --- a/backend/tests/test_artifacts.py +++ b/backend/tests/test_artifacts.py @@ -15,6 +15,7 @@ from fluksio.flow.schemas import FlowDef, FlowInput, NodeDef from fluksio.flow.state import MemoryState from fluksio.flow.store import FlowStore from fluksio.models import Run, RunArtifact +from tests.utils.stores import created @pytest.fixture @@ -126,9 +127,7 @@ def test_a_sweep_keeps_what_a_flow_input_starts_with(store, tmp_path): flows.write_flow( FlowDef(name="f", inputs=[FlowInput(spec=clip, initial=published)]) ) - flows.write_draft( - FlowDef(name="g", inputs=[FlowInput(spec=clip, initial=drafted)]), 0 - ) + created(flows, FlowDef(name="g", inputs=[FlowInput(spec=clip, initial=drafted)])) assert sweep_artifacts(store, MemoryState(), grace_s=0, flows=flows) == 1 assert store.path(orphan["digest"]) is None diff --git a/backend/tests/test_cloud.py b/backend/tests/test_cloud.py index 56e7f97..84df2be 100644 --- a/backend/tests/test_cloud.py +++ b/backend/tests/test_cloud.py @@ -275,7 +275,9 @@ def test_a_panel_scoped_portal_token_reaches_only_its_panel( assert saved.status_code == 200, saved.text for name in ("kitchen", "bench"): created = client.post( - f"{settings.API_V1_STR}/dashboards/{name}", headers=superuser_token_headers + f"{settings.API_V1_STR}/dashboards/", + headers=superuser_token_headers, + json={"id": name, "name": name}, ) assert created.status_code in (200, 201, 409), created.text # A new dashboard is a draft, and a panel only reaches what is diff --git a/backend/tests/test_tombstones.py b/backend/tests/test_tombstones.py new file mode 100644 index 0000000..8e6c5c0 --- /dev/null +++ b/backend/tests/test_tombstones.py @@ -0,0 +1,65 @@ +"""A deleted flow's last key, kept for as long as history names its id.""" + +from datetime import UTC, datetime +from pathlib import Path + +from sqlmodel import Session, delete, select + +from fluksio.flow import tombstones +from fluksio.flow.schemas import FlowDef +from fluksio.flow.store import FlowStore +from fluksio.flow.tombstones import Tombstones, names_of +from fluksio.models import EngineEvent, FlowTombstone, MetricBucket, Run + + +def test_a_tombstone_stays_while_history_names_its_id(db: Session) -> None: + graves = Tombstones() + for flow_id in ("f_gonegone", "f_trainee", "f_measured", "f_forgotten"): + graves.bury(flow_id, flow_id.removeprefix("f_")) + now = datetime.now(UTC) + # A training run is filed under the training flow, and names the node it + # trains in another, by that flow's id. + db.add( + Run( + id="tomb-run", + flow="wren_train", + for_node="f_trainee.lights", + created_at=now, + ) + ) + db.add(Run(id="tomb-own", flow="f_gonegone", created_at=now)) + db.add(MetricBucket(flow="other", node="f_measured.sensor", bucket=now)) + db.add(EngineEvent(ts=now, type="audit", flow="f_forgotten_too")) + db.commit() + + assert graves.recorded("f_trainee") == "trainee" + assert graves.recorded("wren_train") == "wren_train" # history, no tombstone + assert graves.recorded("f_nothing") is None + + assert tombstones.prune(db) == 1 + db.commit() + kept = {stone.id for stone in db.exec(select(FlowTombstone))} + assert kept == {"f_gonegone", "f_trainee", "f_measured"} + + db.exec(delete(Run).where(Run.id.in_(["tomb-run", "tomb-own"]))) # type: ignore[attr-defined,call-overload] + db.exec(delete(MetricBucket)) # type: ignore[call-overload] + db.exec(delete(EngineEvent)) # type: ignore[call-overload] + db.commit() + assert tombstones.prune(db) == 3 + db.commit() + + +def test_a_name_comes_from_the_live_flow_then_its_tombstone_then_the_id( + db: Session, tmp_path: Path +) -> None: + store = FlowStore(tmp_path / "flows") + store.write_flow(FlowDef(id="f_livelive", name="heating")) + Tombstones().bury("f_deadbeef", "cooling") + + assert names_of(["f_livelive", "f_deadbeef", "old_key"], store) == { + "f_livelive": "heating", + "f_deadbeef": "cooling", + "old_key": "old_key", + } + Tombstones().lift("f_deadbeef") + assert db.get(FlowTombstone, "f_deadbeef") is None diff --git a/backend/tests/utils/flows.py b/backend/tests/utils/flows.py new file mode 100644 index 0000000..667c695 --- /dev/null +++ b/backend/tests/utils/flows.py @@ -0,0 +1,66 @@ +"""Flows and dashboards over HTTP, the way most tests want them there.""" + +from typing import Any + +import httpx +from fastapi.testclient import TestClient + +from fluksio.core.config import settings + +FLOWS = f"{settings.API_V1_STR}/flows" +DASHBOARDS = f"{settings.API_V1_STR}/dashboards" + + +def put_flow( + client: TestClient, + headers: dict[str, str], + definition: dict[str, Any], + flow_id: str = "", +) -> httpx.Response: + """Create the flow under *flow_id*, its key unless given, or save over it. + + Creating pins the id, so the test can address the flow by what it knows, + which needs a superuser's *headers*. Saving goes from whatever version is + stored. What a test meant by ``PUT`` before ``PUT`` stopped creating. + """ + flow_id = flow_id or definition["name"] + current = client.get(f"{FLOWS}/{flow_id}", headers=headers) + if current.status_code == 404: + return client.post( + f"{FLOWS}/", headers=headers, json={**definition, "id": flow_id} + ) + version = current.json()["definition"]["version"] + return client.put( + f"{FLOWS}/{flow_id}", + headers=headers, + json={**definition, "id": flow_id, "version": version}, + ) + + +def put_dashboard( + client: TestClient, + headers: dict[str, str], + document: dict[str, Any], + dashboard_id: str = "", +) -> httpx.Response: + """Create the dashboard under *dashboard_id*, its key unless given, and + save *document* over it, as `put_flow` does for a flow.""" + dashboard_id = dashboard_id or document["name"] + current = client.get(f"{DASHBOARDS}/{dashboard_id}?draft=true", headers=headers) + if current.status_code == 404: + created = client.post( + f"{DASHBOARDS}/", + headers=headers, + json={"id": dashboard_id, "name": document["name"]}, + ) + assert created.status_code == 201, created.text + current = created + return client.put( + f"{DASHBOARDS}/{dashboard_id}", + headers=headers, + json={ + **document, + "id": dashboard_id, + "version": current.json()["version"], + }, + ) diff --git a/backend/tests/utils/stores.py b/backend/tests/utils/stores.py new file mode 100644 index 0000000..e29c1c9 --- /dev/null +++ b/backend/tests/utils/stores.py @@ -0,0 +1,12 @@ +"""Store helpers for tests that work below the API.""" + +from typing import Any + + +def created(store: Any, document: Any) -> Any: + """*document* created as a draft, under its key as its id. + + How a store test made one before ids were issued: the id it addresses the + object by is the name it gave. Works for flows and dashboards alike. + """ + return store.create(document, document.name) diff --git a/docs/code/api.md b/docs/code/api.md index 08ea2d6..a28b08f 100644 --- a/docs/code/api.md +++ b/docs/code/api.md @@ -52,29 +52,47 @@ a token. Password recovery uses `POST /password-recovery/{email}` and | Method | Path | Action | |---|---|---| -| `GET` | `/flows/` | list flows and their state | -| `GET` | `/flows/{name}` | read the draft when present, with status and issues | -| `PUT` | `/flows/{name}` | save a draft; send the current `version` | -| `POST` | `/flows/{name}/publish` | publish; send `{"version": }` | -| `POST` | `/flows/{name}/discard-draft` | discard unpublished edits | -| `POST` | `/flows/{name}/duplicate` | copy into an unpublished `{name}_copy`; code nodes need a superuser | -| `DELETE` | `/flows/{name}` | delete; refused while a run is queued or running | +| `GET` | `/flows/` | list flows and their state; `?name=` lists the one with that key | +| `POST` | `/flows/` | create a flow from a document; it starts as a draft | +| `GET` | `/flows/{id}` | read the draft when present, with status and issues | +| `PUT` | `/flows/{id}` | save a draft of an existing flow; send the current `version` | +| `POST` | `/flows/{id}/publish` | publish; send `{"version": }` | +| `POST` | `/flows/{id}/discard-draft` | discard unpublished edits | +| `POST` | `/flows/{id}/duplicate` | copy into an unpublished `_copy` with an id of its own; code nodes need a superuser | +| `DELETE` | `/flows/{id}` | delete; refused while a run is queued or running | | `GET` | `/flows/node-types` | list types and parameter schemas | | `GET` | `/flows/graph` | read the graph shown on Home | -Deleting a flow keeps its run history. Delete runs separately. +A flow has an `id` and a key, its `name`. The id says which flow it is and +never changes. Every path takes the id, never the key. The key is the +namespace of the flow's messages (`house.temperature`) and is unique among +flows. A flow created before ids existed has its key as its id. -A flow document carries an `id` beside its `name`. The id never changes, and -for now it equals the name, which is the `{name}` in these paths. A save may -leave the id out, but cannot change the id or the name. +`POST /flows/` takes the same document as a save, without `version`, and +answers 201 with the new flow. The server issues an opaque id such as +`f_k3q7m2xa`. A superuser may pin one instead by sending `id`, which the Demo +and restores do. A pinned id has to be free and cannot be a path of this API +like `graph`. If runs or metrics of a deleted flow are still filed under it, +the new flow takes that history back and has to use the key it was recorded +under. Creating a flow with code nodes needs a superuser, as saving one does. +A key that still has values or queued work from an earlier flow is refused +with 409 until that is gone. + +`PUT /flows/{id}` only updates. It answers 404 for an id nobody holds, and 409 +for a body whose `name` differs from the stored key. Its body may leave `id` +out. + +Deleting a flow keeps its run history, which is shown under the key the flow +last had. Delete runs separately. The flow's values and queued work go with +it, so its key is free for a new flow, which starts with an empty history. ### Node source ```sh -curl "$FLUKSIO/flows/house/nodes/decide/source" \ +curl "$FLUKSIO/flows/f_k3q7m2xa/nodes/decide/source" \ -H "Authorization: Bearer $TOKEN" -curl -X PUT "$FLUKSIO/flows/house/nodes/decide/source" \ +curl -X PUT "$FLUKSIO/flows/f_k3q7m2xa/nodes/decide/source" \ -H "Authorization: Bearer $TOKEN" \ -H 'Content-Type: application/json' \ -d '{"code": "def process(x):\n return {\"y\": x * 2}\n"}' @@ -87,7 +105,7 @@ Function and Command nodes, changing Command settings, or changing Ordinary source edits stay in the draft until publishing. Shared library source changes affect its users immediately, and the audit trail records them like a publish. -`POST /flows/{name}/nodes/{id}/share` promotes a source to the library; +`POST /flows/{id}/nodes/{node}/share` promotes a source to the library; `/unshare` makes it private again. `GET /flows/library` lists shared sources. `DELETE /flows/library/{name}` retires one, and answers 409 while nodes still use it. With `?detach=true` each of those nodes keeps a copy of the code, in @@ -97,14 +115,18 @@ the published flow and the draft, and the shared source is deleted. | Method | Path | Action | |---|---|---| -| `POST` | `/flows/{name}/start`, `/flows/{name}/stop` | activate or stop a live flow | -| `POST` | `/flows/{name}/pause`, `/flows/{name}/resume` | hold or release incoming work | -| `POST` | `/flows/{name}/step` | release one held message | -| `POST` | `/flows/{name}/validate` | validate the current draft | -| `POST` | `/flows/{name}/run` | run the current draft once with `{"inputs": {...}}` | -| `POST` | `/flows/{name}/nodes/{id}/trigger` | feed a node with `{"values": {...}}` | -| `POST` | `/flows/{name}/nodes/{id}/cancel` | stop active node code | -| `POST` | `/flows/{name}/nodes/{id}/acknowledge` | clear its failure marker | +| `POST` | `/flows/{id}/start`, `/flows/{id}/stop` | activate or stop a live flow | +| `POST` | `/flows/{id}/pause`, `/flows/{id}/resume` | hold or release incoming work | +| `POST` | `/flows/{id}/step` | release one held message | +| `POST` | `/flows/{id}/validate` | validate the current draft | +| `POST` | `/flows/{id}/run` | run the current draft once with `{"inputs": {...}}` | +| `POST` | `/flows/{id}/nodes/{node}/trigger` | feed a node with `{"values": {...}}` | +| `POST` | `/flows/{id}/nodes/{node}/cancel` | stop active node code | +| `POST` | `/flows/{id}/nodes/{node}/acknowledge` | clear its failure marker | + +Values, states and node names in the answers are qualified with the key, as +in `house.temperature` and `house.decide`. Node ids in the paths are the +flow's own, unqualified. A preview run can have real side effects. On a batch flow, `/run` submits a recorded run; use the run endpoints below for parameters and tracking. @@ -192,13 +214,13 @@ reclaiming space sooner. | Method | Path | Action | |---|---|---| -| `GET` | `/dashboards/` | list dashboards | -| `GET` | `/dashboards/{name}?draft=true` | read a draft; omit the query for published content | -| `POST` | `/dashboards/{name}` | create | -| `POST` | `/dashboards/from-flow/{flow}` | create a batch flow's results dashboard | -| `PUT` | `/dashboards/{name}` | save a draft | -| `POST` | `/dashboards/{name}/publish`, `/dashboards/{name}/discard` | publish or discard | -| `POST` | `/dashboards/{name}/duplicate` | copy into an unpublished `{name}_copy` | +| `GET` | `/dashboards/` | list dashboards; `?name=` lists the one with that key | +| `POST` | `/dashboards/` | create an empty draft from `{"name": ..., "title": ...}` | +| `GET` | `/dashboards/{id}?draft=true` | read a draft; omit the query for published content | +| `POST` | `/dashboards/from-flow/{flow id}` | create and publish a batch flow's results dashboard, `_results` | +| `PUT` | `/dashboards/{id}` | save a draft of an existing dashboard | +| `POST` | `/dashboards/{id}/publish`, `/dashboards/{id}/discard` | publish or discard | +| `POST` | `/dashboards/{id}/duplicate` | copy into an unpublished `_copy` with an id of its own | | `GET` | `/devices/` | list devices, with `online`, `last_seen`, `report` and `reported_at` | | `POST`, `GET`, `PUT`, `DELETE` | `/devices/{id}` | create, read, save or delete one; a save sends the `version` it read and gets 409 when someone saved first. A read adds `io` and `bindings`, its ports as the device nodes of published flows wire them | | `POST` | `/devices/pair` | start device pairing, optionally with `{name, board, firmware}`; answers `code`, `secret` and the two-character `check` the device shows beside its code; unauthenticated, five a minute per address (the portal limits its own), the oldest of 50 waiting requests from the same side gives way | @@ -210,8 +232,10 @@ reclaiming space sooner. | `PUT` | `/devices/{id}/report` | what the device has, from its own credential only; at most 16 KB, and 64 KB once its named `config_schemas` are put in place (413 above either), replaces the last report, read leniently (see [What a device reports](../interface/dashboards.md#what-a-device-reports)) | | `GET` | `/devices/{id}/ice` | STUN and TURN servers for a live view of the device, as `{ice_servers: [{urls, username?, credential?}], ttl}`, with TURN credentials of the caller's own; 403 for whoever may not watch its camera, and for a device credential | -A dashboard document carries an `id` the way a flow does. Dashboards have ids -of their own, so a flow and a dashboard may share one. +A dashboard has an `id` and a key the way a flow does, and its paths take the +id. A new one gets an opaque id such as `d_k3q7m2xa`, or one a superuser pins +with `id`. Dashboards have ids of their own, so a flow and a dashboard may +share one. `PUT` only updates, as for a flow. `/panels/pair`, `/panels/pair/{code}` and `GET /panels/{id}` answer as their `/devices/` counterparts, for firmware that has not updated yet. diff --git a/docs/interface/portal.md b/docs/interface/portal.md index 522a5f0..cefb7d4 100644 --- a/docs/interface/portal.md +++ b/docs/interface/portal.md @@ -69,7 +69,7 @@ without an inbound port. Replace `portal.example.com` with your portal's API address in these examples: - A webhook or a served WebSocket: - `https://portal.example.com/i//hooks///`, + `https://portal.example.com/i//hooks///`, with `wss://` for a socket. - An agent: `https://portal.example.com/i//mcp`, see [Agents over MCP](../code/agents.md). diff --git a/docs/reference/node-types.md b/docs/reference/node-types.md index 6ec0be1..c717929 100644 --- a/docs/reference/node-types.md +++ b/docs/reference/node-types.md @@ -61,7 +61,9 @@ unwrapped only if it contains the port's name as a key. The ports pick the direction: outputs only make a **listener**, inputs a **sender**. A listener is a **webhook**: the engine mounts a route at -`/hooks/{flow}/{url}/{secret}` while the flow runs. A sender requests `url` +`/hooks/{flow id}/{url}/{secret}` while the flow runs. The flow's id is in the +URL rather than its key, so the URL stays the same when the key changes, and +for a flow made before ids existed it reads as the key. A sender requests `url` and publishes a JSON object response on its outputs, one key per output port. Keys no port declares are dropped, and a response that is not an object publishes nothing. @@ -93,7 +95,8 @@ one), while a reverse proxy in front of the engine logs the path as it sees it. The URL picks the role: `ws://…` or `wss://…` connects to a server and reconnects when it drops; a path such as `/ticker` is served at -`/hooks/{flow}/{url}/{secret}` while the flow runs. The ports pick the +`/hooks/{flow id}/{url}/{secret}` while the flow runs, under the flow's id like +a webhook. The ports pick the direction: outputs only make a **listener**, inputs a **sender**. A served sender sends to every connected client. diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 05434ce..8ab2581 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -668,6 +668,29 @@ chunk. A reference may carry a \`\`meta\`\` dict — sample rate, dimensions, a sequence number — which nothing here interprets.` } as const; +export const DashboardCreateSchema = { + properties: { + id: { + type: 'string', + title: 'Id', + default: '' + }, + name: { + type: 'string', + title: 'Name' + }, + title: { + type: 'string', + title: 'Title', + default: '' + } + }, + type: 'object', + required: ['name'], + title: 'DashboardCreate', + description: 'A new dashboard: one empty grid under a key, filled in by saves after.' +} as const; + export const DashboardDef_InputSchema = { properties: { id: { @@ -1504,6 +1527,67 @@ export const FlavorsPublicSchema = { title: 'FlavorsPublic' } as const; +export const FlowCreateSchema = { + properties: { + id: { + type: 'string', + title: 'Id', + default: '' + }, + name: { + type: 'string', + title: 'Name' + }, + title: { + type: 'string', + title: 'Title', + default: '' + }, + nodes: { + items: { + '$ref': '#/components/schemas/NodeDef-Input' + }, + type: 'array', + title: 'Nodes' + }, + inputs: { + items: { + '$ref': '#/components/schemas/FlowInput-Input' + }, + type: 'array', + title: 'Inputs' + }, + mode: { + type: 'string', + enum: ['live', 'batch'], + title: 'Mode', + default: 'live' + }, + outputs: { + items: { + type: 'string' + }, + type: 'array', + title: 'Outputs' + }, + origin: { + anyOf: [ + { + '$ref': '#/components/schemas/FlowOrigin' + }, + { + type: 'null' + } + ] + } + }, + type: 'object', + required: ['name'], + title: 'FlowCreate', + description: `A new flow as a client hands it over: a document but for what the +server owns. The fields mean what they do on \`FlowDef\`, but \`\`id\`\`.` +} as const; + export const FlowDef_InputSchema = { properties: { id: { diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index 8e04977..95e5484 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { AiReadAiConfigResponse, AiUpdateAiConfigData, AiUpdateAiConfigResponse, AiDeleteAiConfigResponse, AiListAiModelsData, AiListAiModelsResponse, AiReadAiStatusResponse, AiReadControlNodesResponse, AiReadChatResponse, AiChatData, AiChatResponse, AiClearChatResponse, AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsReadWebpushKeyResponse, AlertsAddWebpushSubscriptionData, AlertsAddWebpushSubscriptionResponse, AlertsRemoveWebpushSubscriptionData, AlertsRemoveWebpushSubscriptionResponse, AlertsStartTelegramLinkData, AlertsStartTelegramLinkResponse, AlertsReadTelegramLinkData, AlertsReadTelegramLinkResponse, AlertsCancelTelegramLinkData, AlertsCancelTelegramLinkResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, ArtifactsDeleteArtifactData, ArtifactsDeleteArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudReadMembersResponse, CloudLinkMemberData, CloudLinkMemberResponse, CloudRevokeMemberData, CloudRevokeMemberResponse, CloudDisconnectResponse, ConnectorsReadConnectorStateData, ConnectorsReadConnectorStateResponse, ConnectorsClearConnectorStateData, ConnectorsClearConnectorStateResponse, ConnectorsDeleteConnectorStateData, ConnectorsDeleteConnectorStateResponse, DashboardsReadDashboardsResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsDuplicateDashboardData, DashboardsDuplicateDashboardResponse, DevicesStartPairingData, DevicesStartPairingResponse, DevicesPollPairingData, DevicesPollPairingResponse, DevicesReadDevicesResponse, DevicesReadPendingResponse, DevicesPendingDeviceData, DevicesPendingDeviceResponse, DevicesCreateDeviceData, DevicesCreateDeviceResponse, DevicesReadDeviceData, DevicesReadDeviceResponse, DevicesSaveDeviceData, DevicesSaveDeviceResponse, DevicesDeleteDeviceData, DevicesDeleteDeviceResponse, DevicesReadIceData, DevicesReadIceResponse, DevicesPutReportData, DevicesPutReportResponse, DevicesApprovePairingData, DevicesApprovePairingResponse, DevicesUnpairDeviceData, DevicesUnpairDeviceResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsDuplicateFlowData, FlowsDuplicateFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsTrainNodeData, FlowsTrainNodeResponse, FlowsReadTrainingData, FlowsReadTrainingResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, FlowsIssueSocketTicketResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadNodeTypeTimingsData, ObservabilityReadNodeTypeTimingsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, ObservabilityFlushMetricsResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsImportRunsData, RunsImportRunsResponse, RunsReadRunsData, RunsReadRunsResponse, RunsDeleteRunsData, RunsDeleteRunsResponse, RunsReadOverviewResponse, RunsExportMetricsData, RunsExportMetricsResponse, RunsExportRunsData, RunsExportRunsResponse, RunsReadMetricNamesData, RunsReadMetricNamesResponse, RunsReadRunData, RunsReadRunResponse, RunsDeleteRunData, RunsDeleteRunResponse, RunsReadNodeOutputsData, RunsReadNodeOutputsResponse, RunsCancelRunData, RunsCancelRunResponse, RunsRetryRunData, RunsRetryRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SearchReadSearchIndexResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsReadConfigResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersReadTokensResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersRevokeTokenData, WorkersRevokeTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; +import type { AiReadAiConfigResponse, AiUpdateAiConfigData, AiUpdateAiConfigResponse, AiDeleteAiConfigResponse, AiListAiModelsData, AiListAiModelsResponse, AiReadAiStatusResponse, AiReadControlNodesResponse, AiReadChatResponse, AiChatData, AiChatResponse, AiClearChatResponse, AlertsReadAlertsConfigResponse, AlertsSaveAlertsConfigData, AlertsSaveAlertsConfigResponse, AlertsReadWebpushKeyResponse, AlertsAddWebpushSubscriptionData, AlertsAddWebpushSubscriptionResponse, AlertsRemoveWebpushSubscriptionData, AlertsRemoveWebpushSubscriptionResponse, AlertsStartTelegramLinkData, AlertsStartTelegramLinkResponse, AlertsReadTelegramLinkData, AlertsReadTelegramLinkResponse, AlertsCancelTelegramLinkData, AlertsCancelTelegramLinkResponse, AlertsTestChannelData, AlertsTestChannelResponse, ArtifactsPutArtifactData, ArtifactsPutArtifactResponse, ArtifactsGetArtifactData, ArtifactsGetArtifactResponse, ArtifactsDeleteArtifactData, ArtifactsDeleteArtifactResponse, CloudReadStatusResponse, CloudEnrollData, CloudEnrollResponse, CloudAddRemoteUserData, CloudAddRemoteUserResponse, CloudReadMembersResponse, CloudLinkMemberData, CloudLinkMemberResponse, CloudRevokeMemberData, CloudRevokeMemberResponse, CloudDisconnectResponse, ConnectorsReadConnectorStateData, ConnectorsReadConnectorStateResponse, ConnectorsClearConnectorStateData, ConnectorsClearConnectorStateResponse, ConnectorsDeleteConnectorStateData, ConnectorsDeleteConnectorStateResponse, DashboardsReadDashboardsData, DashboardsReadDashboardsResponse, DashboardsCreateDashboardData, DashboardsCreateDashboardResponse, DashboardsReadDashboardData, DashboardsReadDashboardResponse, DashboardsSaveDashboardData, DashboardsSaveDashboardResponse, DashboardsDeleteDashboardData, DashboardsDeleteDashboardResponse, DashboardsGenerateResultsDashboardData, DashboardsGenerateResultsDashboardResponse, DashboardsPublishDashboardData, DashboardsPublishDashboardResponse, DashboardsDiscardDashboardDraftData, DashboardsDiscardDashboardDraftResponse, DashboardsDuplicateDashboardData, DashboardsDuplicateDashboardResponse, DevicesStartPairingData, DevicesStartPairingResponse, DevicesPollPairingData, DevicesPollPairingResponse, DevicesReadDevicesResponse, DevicesReadPendingResponse, DevicesPendingDeviceData, DevicesPendingDeviceResponse, DevicesCreateDeviceData, DevicesCreateDeviceResponse, DevicesReadDeviceData, DevicesReadDeviceResponse, DevicesSaveDeviceData, DevicesSaveDeviceResponse, DevicesDeleteDeviceData, DevicesDeleteDeviceResponse, DevicesReadIceData, DevicesReadIceResponse, DevicesPutReportData, DevicesPutReportResponse, DevicesApprovePairingData, DevicesApprovePairingResponse, DevicesUnpairDeviceData, DevicesUnpairDeviceResponse, FlavorsReadFlavorsResponse, FlavorsCreateFlavorData, FlavorsCreateFlavorResponse, FlavorsUpdateFlavorData, FlavorsUpdateFlavorResponse, FlavorsDeleteFlavorData, FlavorsDeleteFlavorResponse, FlowsReadFlowsData, FlowsReadFlowsResponse, FlowsCreateFlowData, FlowsCreateFlowResponse, FlowsReadNodeTypesResponse, FlowsReadGraphResponse, FlowsReadLibraryResponse, FlowsDeleteSharedNodeData, FlowsDeleteSharedNodeResponse, FlowsReadFlowData, FlowsReadFlowResponse, FlowsSaveFlowData, FlowsSaveFlowResponse, FlowsDeleteFlowData, FlowsDeleteFlowResponse, FlowsPublishFlowData, FlowsPublishFlowResponse, FlowsDiscardDraftData, FlowsDiscardDraftResponse, FlowsDuplicateFlowData, FlowsDuplicateFlowResponse, FlowsReadNodeSourceData, FlowsReadNodeSourceResponse, FlowsSaveNodeSourceData, FlowsSaveNodeSourceResponse, FlowsShareNodeData, FlowsShareNodeResponse, FlowsUnshareNodeData, FlowsUnshareNodeResponse, FlowsStartFlowData, FlowsStartFlowResponse, FlowsStopFlowData, FlowsStopFlowResponse, FlowsPauseFlowData, FlowsPauseFlowResponse, FlowsResumeFlowData, FlowsResumeFlowResponse, FlowsStepFlowData, FlowsStepFlowResponse, FlowsValidateFlowData, FlowsValidateFlowResponse, FlowsRunFlowData, FlowsRunFlowResponse, FlowsTriggerNodeData, FlowsTriggerNodeResponse, FlowsCancelNodeData, FlowsCancelNodeResponse, FlowsAcknowledgeNodeErrorData, FlowsAcknowledgeNodeErrorResponse, FlowsTrainNodeData, FlowsTrainNodeResponse, FlowsReadTrainingData, FlowsReadTrainingResponse, FlowsReadFlowStateData, FlowsReadFlowStateResponse, FlowsReadMessageHistoryData, FlowsReadMessageHistoryResponse, FlowsIssueSocketTicketResponse, LoginLoginAccessTokenData, LoginLoginAccessTokenResponse, LoginTestTokenResponse, LoginRecoverPasswordData, LoginRecoverPasswordResponse, LoginResetPasswordData, LoginResetPasswordResponse, LoginRecoverPasswordHtmlContentData, LoginRecoverPasswordHtmlContentResponse, MessagesReadMessagesResponse, MessagesPublishMessageData, MessagesPublishMessageResponse, MessagesReadMessageHistoryData, MessagesReadMessageHistoryResponse, ModulesReadModulesResponse, ModulesApplyModulesData, ModulesApplyModulesResponse, ModulesRefreshModulesResponse, OauthRegisterClientData, OauthRegisterClientResponse, OauthAuthorizeValidateData, OauthAuthorizeValidateResponse, OauthAuthorizeData, OauthAuthorizeResponse, OauthTokenData, OauthTokenResponse, OauthReadClientsResponse, OauthRevokeClientData, OauthRevokeClientResponse, ObservabilityReadSummaryResponse, ObservabilityReadTimeseriesData, ObservabilityReadTimeseriesResponse, ObservabilityReadFlowRollupsData, ObservabilityReadFlowRollupsResponse, ObservabilityReadNodeTypeTimingsData, ObservabilityReadNodeTypeTimingsResponse, ObservabilityReadRunsData, ObservabilityReadRunsResponse, ObservabilityReadEventsData, ObservabilityReadEventsResponse, ObservabilityReadDeadLettersData, ObservabilityReadDeadLettersResponse, ObservabilityFlushMetricsResponse, PrivateCreateUserData, PrivateCreateUserResponse, RunsCreateRunData, RunsCreateRunResponse, RunsCreateSweepData, RunsCreateSweepResponse, RunsImportRunsData, RunsImportRunsResponse, RunsReadRunsData, RunsReadRunsResponse, RunsDeleteRunsData, RunsDeleteRunsResponse, RunsReadOverviewResponse, RunsExportMetricsData, RunsExportMetricsResponse, RunsExportRunsData, RunsExportRunsResponse, RunsReadMetricNamesData, RunsReadMetricNamesResponse, RunsReadRunData, RunsReadRunResponse, RunsDeleteRunData, RunsDeleteRunResponse, RunsReadNodeOutputsData, RunsReadNodeOutputsResponse, RunsCancelRunData, RunsCancelRunResponse, RunsRetryRunData, RunsRetryRunResponse, RunsReadMetricsData, RunsReadMetricsResponse, RunsCompareMetricData, RunsCompareMetricResponse, SearchReadSearchIndexResponse, SecretsReadSecretsResponse, SecretsSaveSecretData, SecretsSaveSecretResponse, SecretsDeleteSecretData, SecretsDeleteSecretResponse, UsersReadUsersData, UsersReadUsersResponse, UsersCreateUserData, UsersCreateUserResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersUpdatePasswordMeData, UsersUpdatePasswordMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UsersReadUserByIdData, UsersReadUserByIdResponse, UsersUpdateUserData, UsersUpdateUserResponse, UsersDeleteUserData, UsersDeleteUserResponse, UtilsTestEmailData, UtilsTestEmailResponse, UtilsReadConfigResponse, UtilsHealthCheckResponse, UtilsHealthResponse, WorkersReadWorkersResponse, WorkersReadResourcesResponse, WorkersReadTokensResponse, WorkersIssueTokenData, WorkersIssueTokenResponse, WorkersRevokeTokenData, WorkersRevokeTokenResponse, WorkersReadRuntimeResponse } from './types.gen'; export class AiService { /** @@ -650,14 +650,47 @@ export class ConnectorsService { export class DashboardsService { /** * Read Dashboards - * Every dashboard, without its contents. + * Every dashboard, without its contents, or with ``name`` the one with + * that key. + * @param data The data for the request. + * @param data.name * @returns DashboardsPublic Successful Response * @throws ApiError */ - public static readDashboards(): CancelablePromise { + public static readDashboards(data: DashboardsReadDashboardsData = {}): CancelablePromise { return __request(OpenAPI, { method: 'GET', - url: '/api/v1/dashboards/' + url: '/api/v1/dashboards/', + query: { + name: data.name + }, + errors: { + 422: 'Validation Error' + } + }); + } + + /** + * Create Dashboard + * Start a dashboard: one empty grid under a key of its own. + * + * The server gives it an opaque id; a superuser may pin one with ``id``. A + * draft, like every edit that follows it — a dashboard reaches a panel only + * once someone publishes it, so an empty one never does. + * @param data The data for the request. + * @param data.requestBody + * @returns DashboardDef_Output Successful Response + * @throws ApiError + */ + public static createDashboard(data: DashboardsCreateDashboardData): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/api/v1/dashboards/', + body: data.requestBody, + mediaType: 'application/json', + errors: { + 422: 'Validation Error' + } }); } @@ -665,7 +698,7 @@ export class DashboardsService { * Read Dashboard * What a panel shows, or with ``draft`` the copy the editor is on. * @param data The data for the request. - * @param data.name + * @param data.dashboardId * @param data.draft * @returns DashboardDef_Output Successful Response * @throws ApiError @@ -673,9 +706,9 @@ export class DashboardsService { public static readDashboard(data: DashboardsReadDashboardData): CancelablePromise { return __request(OpenAPI, { method: 'GET', - url: '/api/v1/dashboards/{name}', + url: '/api/v1/dashboards/{dashboard_id}', path: { - name: data.name + dashboard_id: data.dashboardId }, query: { draft: data.draft @@ -686,39 +719,17 @@ export class DashboardsService { }); } - /** - * Create Dashboard - * Start a dashboard: one page, one section, nothing on it yet. - * - * A draft, like every edit that follows it — a dashboard reaches a panel - * only once someone publishes it, so an empty one never does. - * @param data The data for the request. - * @param data.name - * @returns DashboardDef_Output Successful Response - * @throws ApiError - */ - public static createDashboard(data: DashboardsCreateDashboardData): CancelablePromise { - return __request(OpenAPI, { - method: 'POST', - url: '/api/v1/dashboards/{name}', - path: { - name: data.name - }, - errors: { - 422: 'Validation Error' - } - }); - } - /** * Save Dashboard * Save unpublished changes, refusing a save someone else has moved past. * - * This writes a draft: panels keep showing the published document until - * someone publishes, so nothing here can change what a wall is displaying — - * which is also why the engine's history limits are left alone. + * Only to a dashboard that exists: creating one is ``POST /dashboards``, and + * neither the id nor the key changes here. This writes a draft: panels keep + * showing the published document until someone publishes, so nothing here + * can change what a wall is displaying — which is also why the engine's + * history limits are left alone. * @param data The data for the request. - * @param data.name + * @param data.dashboardId * @param data.requestBody * @returns DashboardDef_Output Successful Response * @throws ApiError @@ -726,9 +737,9 @@ export class DashboardsService { public static saveDashboard(data: DashboardsSaveDashboardData): CancelablePromise { return __request(OpenAPI, { method: 'PUT', - url: '/api/v1/dashboards/{name}', + url: '/api/v1/dashboards/{dashboard_id}', path: { - name: data.name + dashboard_id: data.dashboardId }, body: data.requestBody, mediaType: 'application/json', @@ -741,16 +752,16 @@ export class DashboardsService { /** * Delete Dashboard * @param data The data for the request. - * @param data.name + * @param data.dashboardId * @returns Message Successful Response * @throws ApiError */ public static deleteDashboard(data: DashboardsDeleteDashboardData): CancelablePromise { return __request(OpenAPI, { method: 'DELETE', - url: '/api/v1/dashboards/{name}', + url: '/api/v1/dashboards/{dashboard_id}', path: { - name: data.name + dashboard_id: data.dashboardId }, errors: { 422: 'Validation Error' @@ -762,21 +773,23 @@ export class DashboardsService { * Generate Results Dashboard * Draw a batch flow's results as a dashboard, from the ports it declares. * - * Published straight away rather than left as a draft: there is nothing to - * review that the flow did not already say, and what makes it useful is - * being able to open it against a run immediately. It is an ordinary - * dashboard afterwards — editing it is how it stops being generic. + * Keyed after the flow's key, ``_results``, which a second one refuses + * (409) rather than numbering. Published straight away rather than left as a + * draft: there is nothing to review that the flow did not already say, and + * what makes it useful is being able to open it against a run immediately. + * It is an ordinary dashboard afterwards — editing it is how it stops being + * generic. * @param data The data for the request. - * @param data.flow + * @param data.flowId * @returns DashboardDef_Output Successful Response * @throws ApiError */ public static generateResultsDashboard(data: DashboardsGenerateResultsDashboardData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/dashboards/from-flow/{flow}', + url: '/api/v1/dashboards/from-flow/{flow_id}', path: { - flow: data.flow + flow_id: data.flowId }, errors: { 422: 'Validation Error' @@ -788,7 +801,7 @@ export class DashboardsService { * Publish Dashboard * Put the unpublished changes on the panels. * @param data The data for the request. - * @param data.name + * @param data.dashboardId * @param data.requestBody * @returns DashboardDef_Output Successful Response * @throws ApiError @@ -796,9 +809,9 @@ export class DashboardsService { public static publishDashboard(data: DashboardsPublishDashboardData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/dashboards/{name}/publish', + url: '/api/v1/dashboards/{dashboard_id}/publish', path: { - name: data.name + dashboard_id: data.dashboardId }, body: data.requestBody, mediaType: 'application/json', @@ -812,16 +825,16 @@ export class DashboardsService { * Discard Dashboard Draft * Throw the unpublished changes away and go back to what is shown. * @param data The data for the request. - * @param data.name + * @param data.dashboardId * @returns DashboardDef_Output Successful Response * @throws ApiError */ public static discardDashboardDraft(data: DashboardsDiscardDashboardDraftData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/dashboards/{name}/discard', + url: '/api/v1/dashboards/{dashboard_id}/discard', path: { - name: data.name + dashboard_id: data.dashboardId }, errors: { 422: 'Validation Error' @@ -833,18 +846,19 @@ export class DashboardsService { * Duplicate Dashboard * Copy a dashboard into a new one that is only a draft, named after it. * - * No history limits to apply, as for a new dashboard: nothing published moved. + * The copy gets an id of its own. No history limits to apply, as for a new + * dashboard: nothing published moved. * @param data The data for the request. - * @param data.name + * @param data.dashboardId * @returns DashboardDef_Output Successful Response * @throws ApiError */ public static duplicateDashboard(data: DashboardsDuplicateDashboardData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/dashboards/{name}/duplicate', + url: '/api/v1/dashboards/{dashboard_id}/duplicate', path: { - name: data.name + dashboard_id: data.dashboardId }, errors: { 422: 'Validation Error' @@ -1245,14 +1259,48 @@ export class FlavorsService { export class FlowsService { /** * Read Flows - * List every flow. + * List every flow, or with ``name`` the one holding that key. + * @param data The data for the request. + * @param data.name * @returns FlowsPublic Successful Response * @throws ApiError */ - public static readFlows(): CancelablePromise { + public static readFlows(data: FlowsReadFlowsData = {}): CancelablePromise { return __request(OpenAPI, { method: 'GET', - url: '/api/v1/flows/' + url: '/api/v1/flows/', + query: { + name: data.name + }, + errors: { + 422: 'Validation Error' + } + }); + } + + /** + * Create Flow + * Create a flow. It is only a draft until someone publishes it. + * + * The server gives it an opaque id. A superuser may pin one with ``id``: an + * id no flow holds, or one whose history this flow takes up again, under the + * key that history was kept under. A key with state or queued work still + * behind it is refused until that is gone, so a new flow never inherits what + * another left. + * @param data The data for the request. + * @param data.requestBody + * @returns FlowDetail Successful Response + * @throws ApiError + */ + public static createFlow(data: FlowsCreateFlowData): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/api/v1/flows/', + body: data.requestBody, + mediaType: 'application/json', + errors: { + 422: 'Validation Error' + } }); } @@ -1327,16 +1375,16 @@ export class FlowsService { * Read Flow * Read one flow, with the state of its nodes. * @param data The data for the request. - * @param data.name + * @param data.flowId * @returns FlowDetail Successful Response * @throws ApiError */ public static readFlow(data: FlowsReadFlowData): CancelablePromise { return __request(OpenAPI, { method: 'GET', - url: '/api/v1/flows/{name}', + url: '/api/v1/flows/{flow_id}', path: { - name: data.name + flow_id: data.flowId }, errors: { 422: 'Validation Error' @@ -1348,12 +1396,13 @@ export class FlowsService { * Save Flow * Save unpublished changes to a flow. * - * This writes a draft: the running pipeline keeps the published version until - * someone publishes. ``version`` is the one the editor last saw — a mismatch - * means another client saved in between and answers 409 rather than throwing - * their work away. + * Only to one that exists: creating one is ``POST /flows``, and neither the + * id nor the key changes here. This writes a draft: the running pipeline + * keeps the published version until someone publishes. ``version`` is the + * one the editor last saw — a mismatch means another client saved in + * between and answers 409 rather than throwing their work away. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.requestBody * @returns FlowDetail Successful Response * @throws ApiError @@ -1361,9 +1410,9 @@ export class FlowsService { public static saveFlow(data: FlowsSaveFlowData): CancelablePromise { return __request(OpenAPI, { method: 'PUT', - url: '/api/v1/flows/{name}', + url: '/api/v1/flows/{flow_id}', path: { - name: data.name + flow_id: data.flowId }, body: data.requestBody, mediaType: 'application/json', @@ -1376,17 +1425,21 @@ export class FlowsService { /** * Delete Flow * Delete a flow and everything in it, except the record of what it ran. + * + * Its key is kept with its id (`fluksio.flow.tombstones`) for as long as + * that record is, and what is left under the key, values and queued work, + * goes with the flow so the key is free for a new one. * @param data The data for the request. - * @param data.name + * @param data.flowId * @returns Message Successful Response * @throws ApiError */ public static deleteFlow(data: FlowsDeleteFlowData): CancelablePromise { return __request(OpenAPI, { method: 'DELETE', - url: '/api/v1/flows/{name}', + url: '/api/v1/flows/{flow_id}', path: { - name: data.name + flow_id: data.flowId }, errors: { 422: 'Validation Error' @@ -1398,7 +1451,7 @@ export class FlowsService { * Publish Flow * Deploy the unpublished changes: the engine picks them up from here. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.requestBody * @returns FlowDetail Successful Response * @throws ApiError @@ -1406,9 +1459,9 @@ export class FlowsService { public static publishFlow(data: FlowsPublishFlowData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/publish', + url: '/api/v1/flows/{flow_id}/publish', path: { - name: data.name + flow_id: data.flowId }, body: data.requestBody, mediaType: 'application/json', @@ -1422,16 +1475,16 @@ export class FlowsService { * Discard Draft * Throw the unpublished changes away and go back to what is running. * @param data The data for the request. - * @param data.name + * @param data.flowId * @returns FlowDetail Successful Response * @throws ApiError */ public static discardDraft(data: FlowsDiscardDraftData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/discard-draft', + url: '/api/v1/flows/{flow_id}/discard-draft', path: { - name: data.name + flow_id: data.flowId }, errors: { 422: 'Validation Error' @@ -1443,19 +1496,21 @@ export class FlowsService { * Duplicate Flow * Copy a flow into a new one that is only a draft, named after it. * - * Nothing to reload: a draft does not run. Its code nodes are new code in - * the copy, so copying them is a superuser's, like writing them. + * The copy gets an id of its own, and the first ``_copy`` key with nothing + * left behind it. Nothing to reload: a draft does not run. Its code nodes + * are new code in the copy, so copying them is a superuser's, like writing + * them. * @param data The data for the request. - * @param data.name + * @param data.flowId * @returns FlowDetail Successful Response * @throws ApiError */ public static duplicateFlow(data: FlowsDuplicateFlowData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/duplicate', + url: '/api/v1/flows/{flow_id}/duplicate', path: { - name: data.name + flow_id: data.flowId }, errors: { 422: 'Validation Error' @@ -1467,7 +1522,7 @@ export class FlowsService { * Read Node Source * Read a node's Python source, including unpublished edits. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.nodeId * @returns NodeSource Successful Response * @throws ApiError @@ -1475,9 +1530,9 @@ export class FlowsService { public static readNodeSource(data: FlowsReadNodeSourceData): CancelablePromise { return __request(OpenAPI, { method: 'GET', - url: '/api/v1/flows/{name}/nodes/{node_id}/source', + url: '/api/v1/flows/{flow_id}/nodes/{node_id}/source', path: { - name: data.name, + flow_id: data.flowId, node_id: data.nodeId }, errors: { @@ -1505,7 +1560,7 @@ export class FlowsService { * adds and renames from the header but keeps what it no longer names until * the editor is left. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.nodeId * @param data.requestBody * @param data.drop @@ -1515,9 +1570,9 @@ export class FlowsService { public static saveNodeSource(data: FlowsSaveNodeSourceData): CancelablePromise { return __request(OpenAPI, { method: 'PUT', - url: '/api/v1/flows/{name}/nodes/{node_id}/source', + url: '/api/v1/flows/{flow_id}/nodes/{node_id}/source', path: { - name: data.name, + flow_id: data.flowId, node_id: data.nodeId }, query: { @@ -1535,7 +1590,7 @@ export class FlowsService { * Share Node * Move this node's code into the library so other flows can run it too. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.nodeId * @param data.requestBody * @returns FlowDetail Successful Response @@ -1544,9 +1599,9 @@ export class FlowsService { public static shareNode(data: FlowsShareNodeData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/nodes/{node_id}/share', + url: '/api/v1/flows/{flow_id}/nodes/{node_id}/share', path: { - name: data.name, + flow_id: data.flowId, node_id: data.nodeId }, body: data.requestBody, @@ -1561,7 +1616,7 @@ export class FlowsService { * Unshare Node * Take a private copy of the shared code back into this flow. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.nodeId * @returns FlowDetail Successful Response * @throws ApiError @@ -1569,9 +1624,9 @@ export class FlowsService { public static unshareNode(data: FlowsUnshareNodeData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/nodes/{node_id}/unshare', + url: '/api/v1/flows/{flow_id}/nodes/{node_id}/unshare', path: { - name: data.name, + flow_id: data.flowId, node_id: data.nodeId }, errors: { @@ -1584,16 +1639,16 @@ export class FlowsService { * Start Flow * Let the engine run this flow again. * @param data The data for the request. - * @param data.name + * @param data.flowId * @returns FlowDetail Successful Response * @throws ApiError */ public static startFlow(data: FlowsStartFlowData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/start', + url: '/api/v1/flows/{flow_id}/start', path: { - name: data.name + flow_id: data.flowId }, errors: { 422: 'Validation Error' @@ -1605,16 +1660,16 @@ export class FlowsService { * Stop Flow * Take this flow off the engine: no subscriptions, schedules or webhooks. * @param data The data for the request. - * @param data.name + * @param data.flowId * @returns FlowDetail Successful Response * @throws ApiError */ public static stopFlow(data: FlowsStopFlowData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/stop', + url: '/api/v1/flows/{flow_id}/stop', path: { - name: data.name + flow_id: data.flowId }, errors: { 422: 'Validation Error' @@ -1626,16 +1681,16 @@ export class FlowsService { * Pause Flow * Hold the flow's nodes so its messages can be stepped through. * @param data The data for the request. - * @param data.name + * @param data.flowId * @returns Message Successful Response * @throws ApiError */ public static pauseFlow(data: FlowsPauseFlowData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/pause', + url: '/api/v1/flows/{flow_id}/pause', path: { - name: data.name + flow_id: data.flowId }, errors: { 422: 'Validation Error' @@ -1647,16 +1702,16 @@ export class FlowsService { * Resume Flow * Let the flow carry on, running whatever was held back. * @param data The data for the request. - * @param data.name + * @param data.flowId * @returns Message Successful Response * @throws ApiError */ public static resumeFlow(data: FlowsResumeFlowData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/resume', + url: '/api/v1/flows/{flow_id}/resume', path: { - name: data.name + flow_id: data.flowId }, errors: { 422: 'Validation Error' @@ -1668,16 +1723,16 @@ export class FlowsService { * Step Flow * Run one message a pause is holding back, leaving the flow paused. * @param data The data for the request. - * @param data.name + * @param data.flowId * @returns Message Successful Response * @throws ApiError */ public static stepFlow(data: FlowsStepFlowData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/step', + url: '/api/v1/flows/{flow_id}/step', path: { - name: data.name + flow_id: data.flowId }, errors: { 422: 'Validation Error' @@ -1689,16 +1744,16 @@ export class FlowsService { * Validate Flow * Report what would keep this flow from running. * @param data The data for the request. - * @param data.name + * @param data.flowId * @returns ValidationResult Successful Response * @throws ApiError */ public static validateFlow(data: FlowsValidateFlowData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/validate', + url: '/api/v1/flows/{flow_id}/validate', path: { - name: data.name + flow_id: data.flowId }, errors: { 422: 'Validation Error' @@ -1717,7 +1772,7 @@ export class FlowsService { * one means: it is what keeps the parameters, the series and the result, and * a button that quietly did something else would be a trap. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.requestBody * @returns FlowStatePublic Successful Response * @throws ApiError @@ -1725,9 +1780,9 @@ export class FlowsService { public static runFlow(data: FlowsRunFlowData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/run', + url: '/api/v1/flows/{flow_id}/run', path: { - name: data.name + flow_id: data.flowId }, body: data.requestBody, mediaType: 'application/json', @@ -1741,7 +1796,7 @@ export class FlowsService { * Trigger Node * Feed values into a single node. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.nodeId * @param data.requestBody * @returns FlowStatePublic Successful Response @@ -1750,9 +1805,9 @@ export class FlowsService { public static triggerNode(data: FlowsTriggerNodeData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/nodes/{node_id}/trigger', + url: '/api/v1/flows/{flow_id}/nodes/{node_id}/trigger', path: { - name: data.name, + flow_id: data.flowId, node_id: data.nodeId }, body: data.requestBody, @@ -1770,7 +1825,7 @@ export class FlowsService { * Idempotent on purpose: by the time a click reaches here the node may well * have finished, and that is the outcome that was asked for. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.nodeId * @returns Message Successful Response * @throws ApiError @@ -1778,9 +1833,9 @@ export class FlowsService { public static cancelNode(data: FlowsCancelNodeData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/nodes/{node_id}/cancel', + url: '/api/v1/flows/{flow_id}/nodes/{node_id}/cancel', path: { - name: data.name, + flow_id: data.flowId, node_id: data.nodeId }, errors: { @@ -1797,7 +1852,7 @@ export class FlowsService { * an alert leaves no trace by the time anyone looks. Reading the traceback is * what says it has been seen. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.nodeId * @returns Message Successful Response * @throws ApiError @@ -1805,9 +1860,9 @@ export class FlowsService { public static acknowledgeNodeError(data: FlowsAcknowledgeNodeErrorData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/nodes/{node_id}/acknowledge', + url: '/api/v1/flows/{flow_id}/nodes/{node_id}/acknowledge', path: { - name: data.name, + flow_id: data.flowId, node_id: data.nodeId }, errors: { @@ -1822,9 +1877,10 @@ export class FlowsService { * * It learns the node's targets, its ``examples`` and its ``tests``, as the * editor shows them. The node keeps running its model until the new one is - * set as its ``model``. + * set as its ``model``. The run names the node it trains by the flow's id, + * ``.``, which is what its panel finds it by. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.nodeId * @returns Training Successful Response * @throws ApiError @@ -1832,9 +1888,9 @@ export class FlowsService { public static trainNode(data: FlowsTrainNodeData): CancelablePromise { return __request(OpenAPI, { method: 'POST', - url: '/api/v1/flows/{name}/nodes/{node_id}/train', + url: '/api/v1/flows/{flow_id}/nodes/{node_id}/train', path: { - name: data.name, + flow_id: data.flowId, node_id: data.nodeId }, errors: { @@ -1847,7 +1903,7 @@ export class FlowsService { * Read Training * A wren node's latest training run, or null when it has none. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.nodeId * @returns unknown Successful Response * @throws ApiError @@ -1855,9 +1911,9 @@ export class FlowsService { public static readTraining(data: FlowsReadTrainingData): CancelablePromise { return __request(OpenAPI, { method: 'GET', - url: '/api/v1/flows/{name}/nodes/{node_id}/train', + url: '/api/v1/flows/{flow_id}/nodes/{node_id}/train', path: { - name: data.name, + flow_id: data.flowId, node_id: data.nodeId }, errors: { @@ -1870,16 +1926,16 @@ export class FlowsService { * Read Flow State * The last value seen on every message of this flow. * @param data The data for the request. - * @param data.name + * @param data.flowId * @returns FlowStatePublic Successful Response * @throws ApiError */ public static readFlowState(data: FlowsReadFlowStateData): CancelablePromise { return __request(OpenAPI, { method: 'GET', - url: '/api/v1/flows/{name}/state', + url: '/api/v1/flows/{flow_id}/state', path: { - name: data.name + flow_id: data.flowId }, errors: { 422: 'Validation Error' @@ -1891,10 +1947,10 @@ export class FlowsService { * Read Message History * The recent values of one message, for plotting. * - * ``message`` may be given bare or qualified; a message that never carried a - * number or a flag comes back with an empty series. + * ``message`` may be given bare, or qualified with the flow's key; a message + * that never carried a number or a flag comes back with an empty series. * @param data The data for the request. - * @param data.name + * @param data.flowId * @param data.message * @returns MessageHistory Successful Response * @throws ApiError @@ -1902,9 +1958,9 @@ export class FlowsService { public static readMessageHistory(data: FlowsReadMessageHistoryData): CancelablePromise { return __request(OpenAPI, { method: 'GET', - url: '/api/v1/flows/{name}/history/{message}', + url: '/api/v1/flows/{flow_id}/history/{message}', path: { - name: data.name, + flow_id: data.flowId, message: data.message }, errors: { diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index cbba3aa..231fac4 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -168,6 +168,15 @@ export type ControlWatch = { node?: string; }; +/** + * A new dashboard: one empty grid under a key, filled in by saves after. + */ +export type DashboardCreate = { + id?: string; + name: string; + title?: string; +}; + /** * A dashboard as stored, and as the API hands it over. */ @@ -424,6 +433,23 @@ export type FlavorUpdate = { description?: (string | null); }; +/** + * A new flow as a client hands it over: a document but for what the + * server owns. The fields mean what they do on `FlowDef`, but ``id``. + */ +export type FlowCreate = { + id?: string; + name: string; + title?: string; + nodes?: Array; + inputs?: Array; + mode?: 'live' | 'batch'; + outputs?: Array<(string)>; + origin?: (FlowOrigin | null); +}; + +export type mode = 'live' | 'batch'; + /** * One atomic flow. */ @@ -448,11 +474,6 @@ export type FlowDef_Input = { origin?: (FlowOrigin | null); }; -/** - * A live flow reacts to what arrives: its subscriptions, schedules and webhooks run until it is stopped. A batch flow only runs when a run asks it to, from its inputs to its outputs, and is never activated. - */ -export type mode = 'live' | 'batch'; - /** * One atomic flow. */ @@ -1846,55 +1867,59 @@ export type ConnectorsDeleteConnectorStateData = { export type ConnectorsDeleteConnectorStateResponse = (Message); -export type DashboardsReadDashboardsResponse = (DashboardsPublic); - -export type DashboardsReadDashboardData = { - draft?: boolean; - name: string; +export type DashboardsReadDashboardsData = { + name?: (string | null); }; -export type DashboardsReadDashboardResponse = (DashboardDef_Output); +export type DashboardsReadDashboardsResponse = (DashboardsPublic); export type DashboardsCreateDashboardData = { - name: string; + requestBody: DashboardCreate; }; export type DashboardsCreateDashboardResponse = (DashboardDef_Output); +export type DashboardsReadDashboardData = { + dashboardId: string; + draft?: boolean; +}; + +export type DashboardsReadDashboardResponse = (DashboardDef_Output); + export type DashboardsSaveDashboardData = { - name: string; + dashboardId: string; requestBody: DashboardDef_Input; }; export type DashboardsSaveDashboardResponse = (DashboardDef_Output); export type DashboardsDeleteDashboardData = { - name: string; + dashboardId: string; }; export type DashboardsDeleteDashboardResponse = (Message); export type DashboardsGenerateResultsDashboardData = { - flow: string; + flowId: string; }; export type DashboardsGenerateResultsDashboardResponse = (DashboardDef_Output); export type DashboardsPublishDashboardData = { - name: string; + dashboardId: string; requestBody: fluksio__api__routes__dashboards__PublishRequest; }; export type DashboardsPublishDashboardResponse = (DashboardDef_Output); export type DashboardsDiscardDashboardDraftData = { - name: string; + dashboardId: string; }; export type DashboardsDiscardDashboardDraftResponse = (DashboardDef_Output); export type DashboardsDuplicateDashboardData = { - name: string; + dashboardId: string; }; export type DashboardsDuplicateDashboardResponse = (DashboardDef_Output); @@ -1994,8 +2019,18 @@ export type FlavorsDeleteFlavorData = { export type FlavorsDeleteFlavorResponse = (Message); +export type FlowsReadFlowsData = { + name?: (string | null); +}; + export type FlowsReadFlowsResponse = (FlowsPublic); +export type FlowsCreateFlowData = { + requestBody: FlowCreate; +}; + +export type FlowsCreateFlowResponse = (FlowDetail); + export type FlowsReadNodeTypesResponse = (Array); export type FlowsReadGraphResponse = (BrainGraph); @@ -2010,45 +2045,45 @@ export type FlowsDeleteSharedNodeData = { export type FlowsDeleteSharedNodeResponse = (Message); export type FlowsReadFlowData = { - name: string; + flowId: string; }; export type FlowsReadFlowResponse = (FlowDetail); export type FlowsSaveFlowData = { - name: string; + flowId: string; requestBody: FlowDef_Input; }; export type FlowsSaveFlowResponse = (FlowDetail); export type FlowsDeleteFlowData = { - name: string; + flowId: string; }; export type FlowsDeleteFlowResponse = (Message); export type FlowsPublishFlowData = { - name: string; + flowId: string; requestBody: fluksio__api__routes__flows__PublishRequest; }; export type FlowsPublishFlowResponse = (FlowDetail); export type FlowsDiscardDraftData = { - name: string; + flowId: string; }; export type FlowsDiscardDraftResponse = (FlowDetail); export type FlowsDuplicateFlowData = { - name: string; + flowId: string; }; export type FlowsDuplicateFlowResponse = (FlowDetail); export type FlowsReadNodeSourceData = { - name: string; + flowId: string; nodeId: string; }; @@ -2056,7 +2091,7 @@ export type FlowsReadNodeSourceResponse = (NodeSource); export type FlowsSaveNodeSourceData = { drop?: boolean; - name: string; + flowId: string; nodeId: string; requestBody: NodeSource; }; @@ -2064,7 +2099,7 @@ export type FlowsSaveNodeSourceData = { export type FlowsSaveNodeSourceResponse = (NodeSourceSaved); export type FlowsShareNodeData = { - name: string; + flowId: string; nodeId: string; requestBody: ShareRequest; }; @@ -2072,57 +2107,57 @@ export type FlowsShareNodeData = { export type FlowsShareNodeResponse = (FlowDetail); export type FlowsUnshareNodeData = { - name: string; + flowId: string; nodeId: string; }; export type FlowsUnshareNodeResponse = (FlowDetail); export type FlowsStartFlowData = { - name: string; + flowId: string; }; export type FlowsStartFlowResponse = (FlowDetail); export type FlowsStopFlowData = { - name: string; + flowId: string; }; export type FlowsStopFlowResponse = (FlowDetail); export type FlowsPauseFlowData = { - name: string; + flowId: string; }; export type FlowsPauseFlowResponse = (Message); export type FlowsResumeFlowData = { - name: string; + flowId: string; }; export type FlowsResumeFlowResponse = (Message); export type FlowsStepFlowData = { - name: string; + flowId: string; }; export type FlowsStepFlowResponse = (Message); export type FlowsValidateFlowData = { - name: string; + flowId: string; }; export type FlowsValidateFlowResponse = (ValidationResult); export type FlowsRunFlowData = { - name: string; + flowId: string; requestBody: RunRequest; }; export type FlowsRunFlowResponse = (FlowStatePublic); export type FlowsTriggerNodeData = { - name: string; + flowId: string; nodeId: string; requestBody: TriggerRequest; }; @@ -2130,42 +2165,42 @@ export type FlowsTriggerNodeData = { export type FlowsTriggerNodeResponse = (FlowStatePublic); export type FlowsCancelNodeData = { - name: string; + flowId: string; nodeId: string; }; export type FlowsCancelNodeResponse = (Message); export type FlowsAcknowledgeNodeErrorData = { - name: string; + flowId: string; nodeId: string; }; export type FlowsAcknowledgeNodeErrorResponse = (Message); export type FlowsTrainNodeData = { - name: string; + flowId: string; nodeId: string; }; export type FlowsTrainNodeResponse = (Training); export type FlowsReadTrainingData = { - name: string; + flowId: string; nodeId: string; }; export type FlowsReadTrainingResponse = ((Training | null)); export type FlowsReadFlowStateData = { - name: string; + flowId: string; }; export type FlowsReadFlowStateResponse = (FlowStatePublic); export type FlowsReadMessageHistoryData = { + flowId: string; message: string; - name: string; }; export type FlowsReadMessageHistoryResponse = (MessageHistory); diff --git a/frontend/src/components/Dashboard/DashboardEditor.tsx b/frontend/src/components/Dashboard/DashboardEditor.tsx index 8845c1d..b72c5f9 100644 --- a/frontend/src/components/Dashboard/DashboardEditor.tsx +++ b/frontend/src/components/Dashboard/DashboardEditor.tsx @@ -484,7 +484,8 @@ export function DashboardEditor({ } const remove = useMutation({ - mutationFn: () => DashboardsService.deleteDashboard({ name: draft.name }), + mutationFn: () => + DashboardsService.deleteDashboard({ dashboardId: draft.name }), onSuccess: () => { queryClient.invalidateQueries({ queryKey: dashboardKeys.all }) navigate({ to: "/dashboards" }) diff --git a/frontend/src/components/Dashboard/queries.ts b/frontend/src/components/Dashboard/queries.ts index 82950b3..3b56570 100644 --- a/frontend/src/components/Dashboard/queries.ts +++ b/frontend/src/components/Dashboard/queries.ts @@ -28,7 +28,7 @@ export const dashboardsQueryOptions = () => ({ */ export const dashboardQueryOptions = (name: string, draft = false) => ({ queryKey: dashboardKeys.detail(name, draft), - queryFn: () => DashboardsService.readDashboard({ name, draft }), + queryFn: () => DashboardsService.readDashboard({ dashboardId: name, draft }), }) /** Every message any flow declares — what a widget can be pointed at. */ @@ -50,7 +50,7 @@ export function useSaveDashboard(name: string) { const queryClient = useQueryClient() return useMutation({ mutationFn: (body: DashboardDef_Input) => - DashboardsService.saveDashboard({ name, requestBody: body }), + DashboardsService.saveDashboard({ dashboardId: name, requestBody: body }), onSuccess: (saved) => { queryClient.setQueryData(dashboardKeys.detail(name, true), saved) queryClient.invalidateQueries({ queryKey: dashboardKeys.all }) @@ -64,7 +64,7 @@ export function usePublishDashboard(name: string) { return useMutation({ mutationFn: (version: number) => DashboardsService.publishDashboard({ - name, + dashboardId: name, requestBody: { version }, }), onSuccess: (published) => { @@ -79,7 +79,8 @@ export function usePublishDashboard(name: string) { export function useDiscardDashboardDraft(name: string) { const queryClient = useQueryClient() return useMutation({ - mutationFn: () => DashboardsService.discardDashboardDraft({ name }), + mutationFn: () => + DashboardsService.discardDashboardDraft({ dashboardId: name }), onSuccess: (published) => { queryClient.setQueryData(dashboardKeys.detail(name, true), published) queryClient.setQueryData(dashboardKeys.detail(name), published) diff --git a/frontend/src/components/Devices/DeviceDetail.tsx b/frontend/src/components/Devices/DeviceDetail.tsx index 8c66e6b..8cf8fbe 100644 --- a/frontend/src/components/Devices/DeviceDetail.tsx +++ b/frontend/src/components/Devices/DeviceDetail.tsx @@ -840,7 +840,7 @@ export function UseInFlow({ const flows = data?.data ?? [] const place = useMutation({ mutationFn: async (flowName: string) => { - const { definition } = await FlowsService.readFlow({ name: flowName }) + const { definition } = await FlowsService.readFlow({ flowId: flowName }) const nodes = definition.nodes ?? [] const existing = deviceNodeOf(nodes, device.id) const node = existing ?? { @@ -854,7 +854,7 @@ export function UseInFlow({ const placed = port ? withPort(node, port) : node if (placed !== existing) { await FlowsService.saveFlow({ - name: flowName, + flowId: flowName, requestBody: { ...definition, nodes: existing diff --git a/frontend/src/components/Flow/EdgeInspector.tsx b/frontend/src/components/Flow/EdgeInspector.tsx index 46e58b7..02a85f1 100644 --- a/frontend/src/components/Flow/EdgeInspector.tsx +++ b/frontend/src/components/Flow/EdgeInspector.tsx @@ -64,7 +64,7 @@ export function EdgeInspector({ const replay = useMutation({ mutationFn: (value: unknown) => FlowsService.triggerNode({ - name: flow, + flowId: flow, nodeId: edge?.sourceId ?? "", requestBody: { values: { [edge?.message ?? ""]: value } }, }), diff --git a/frontend/src/components/Flow/FlowEditor.tsx b/frontend/src/components/Flow/FlowEditor.tsx index b2a6e60..319cd1c 100644 --- a/frontend/src/components/Flow/FlowEditor.tsx +++ b/frontend/src/components/Flow/FlowEditor.tsx @@ -861,7 +861,7 @@ function FlowEditorInner({ const runMutation = useMutation({ mutationFn: (inputs: Record = {}) => - FlowsService.runFlow({ name: flowName, requestBody: { inputs } }), + FlowsService.runFlow({ flowId: flowName, requestBody: { inputs } }), onSuccess: (state) => { liveStore.setValues( Object.fromEntries( @@ -892,8 +892,8 @@ function FlowEditorInner({ const enableMutation = useMutation({ mutationFn: (next: boolean) => next - ? FlowsService.startFlow({ name: flowName }) - : FlowsService.stopFlow({ name: flowName }), + ? FlowsService.startFlow({ flowId: flowName }) + : FlowsService.stopFlow({ flowId: flowName }), onSuccess: (detail) => { queryClient.setQueryData(flowKeys.detail(flowName), detail) queryClient.invalidateQueries({ queryKey: flowKeys.all, exact: true }) @@ -904,14 +904,14 @@ function FlowEditorInner({ const pauseMutation = useMutation({ mutationFn: (next: boolean) => next - ? FlowsService.pauseFlow({ name: flowName }) - : FlowsService.resumeFlow({ name: flowName }), + ? FlowsService.pauseFlow({ flowId: flowName }) + : FlowsService.resumeFlow({ flowId: flowName }), // The engine answers with a flow_paused event, which is what the dock reads. onError: () => showErrorToast("The flow could not be paused."), }) const stepMutation = useMutation({ - mutationFn: () => FlowsService.stepFlow({ name: flowName }), + mutationFn: () => FlowsService.stepFlow({ flowId: flowName }), // Says which node ran, or that nothing was held back — both are answers. onSuccess: (result) => showSuccessToast(result.message), onError: () => showErrorToast("The flow could not be stepped."), @@ -919,7 +919,7 @@ function FlowEditorInner({ const deleteMutation = useMutation({ mutationFn: async (withRuns: boolean) => { - await FlowsService.deleteFlow({ name: flowName }) + await FlowsService.deleteFlow({ flowId: flowName }) // After the flow, which is refused while a run is live, so a refusal // leaves both standing. if (withRuns) @@ -947,7 +947,7 @@ function FlowEditorInner({ // draft, so the ports on the canvas have to be there first. await flush() return FlowsService.saveNodeSource({ - name: flowName, + flowId: flowName, nodeId, requestBody: { code }, drop, @@ -1227,7 +1227,7 @@ function FlowEditorInner({ .filter((node) => !node.source_ref && sourceTypes.has(node.type ?? "")) .map(async (node) => { const { code } = await FlowsService.readNodeSource({ - name: flowName, + flowId: flowName, nodeId: node.id, }) sources[node.id] = code diff --git a/frontend/src/components/Flow/FlowNode.tsx b/frontend/src/components/Flow/FlowNode.tsx index 8d4d64c..edd33ae 100644 --- a/frontend/src/components/Flow/FlowNode.tsx +++ b/frontend/src/components/Flow/FlowNode.tsx @@ -278,7 +278,7 @@ function FlowNodeComponent({ data, selected }: NodeProps) { // Best effort by nature: it may well have finished between // the render and the click, which is the outcome asked for. FlowsService.cancelNode({ - name: flow, + flowId: flow, nodeId: definition.id, }).catch(() => {}) }} diff --git a/frontend/src/components/Flow/NodePanel.tsx b/frontend/src/components/Flow/NodePanel.tsx index 8b4b895..30f5143 100644 --- a/frontend/src/components/Flow/NodePanel.tsx +++ b/frontend/src/components/Flow/NodePanel.tsx @@ -1647,7 +1647,7 @@ function SharingSection({ const share = useMutation({ mutationFn: (libName: string) => FlowsService.shareNode({ - name: flow, + flowId: flow, nodeId: node.id, requestBody: { lib_name: libName }, }), @@ -1657,7 +1657,8 @@ function SharingSection({ }) const unshare = useMutation({ - mutationFn: () => FlowsService.unshareNode({ name: flow, nodeId: node.id }), + mutationFn: () => + FlowsService.unshareNode({ flowId: flow, nodeId: node.id }), onSuccess: done, onError: () => showErrorToast("The node could not be unshared."), }) @@ -1749,13 +1750,13 @@ function TrainingSection({ const queryKey = ["wren-training", flow, node.id] const { data: training } = useQuery({ queryKey, - queryFn: () => FlowsService.readTraining({ name: flow, nodeId: node.id }), + queryFn: () => FlowsService.readTraining({ flowId: flow, nodeId: node.id }), // Training takes minutes, so only a run still going is read again. refetchInterval: (query) => query.state.data && isLive(query.state.data.status) ? 5_000 : false, }) const train = useMutation({ - mutationFn: () => FlowsService.trainNode({ name: flow, nodeId: node.id }), + mutationFn: () => FlowsService.trainNode({ flowId: flow, nodeId: node.id }), onMutate: () => setProblem(null), onSuccess: () => queryClient.invalidateQueries({ queryKey }), onError: handleError.bind(setProblem), diff --git a/frontend/src/components/Flow/queries.ts b/frontend/src/components/Flow/queries.ts index 8c82c78..d87dda9 100644 --- a/frontend/src/components/Flow/queries.ts +++ b/frontend/src/components/Flow/queries.ts @@ -36,7 +36,7 @@ export const flowsQueryOptions = () => ({ export const flowQueryOptions = (name: string) => ({ queryKey: flowKeys.detail(name), - queryFn: () => FlowsService.readFlow({ name }), + queryFn: () => FlowsService.readFlow({ flowId: name }), }) /** Names of the stored secrets, for pointing a credential parameter at one. */ @@ -71,13 +71,13 @@ export const nodeTypesQueryOptions = () => ({ export const nodeSourceQueryOptions = (name: string, nodeId: string) => ({ queryKey: flowKeys.source(name, nodeId), - queryFn: () => FlowsService.readNodeSource({ name, nodeId }), + queryFn: () => FlowsService.readNodeSource({ flowId: name, nodeId }), }) /** The recent values of one message, for the panel's sparkline. */ export const messageHistoryQueryOptions = (name: string, message: string) => ({ queryKey: flowKeys.history(name, message), - queryFn: () => FlowsService.readMessageHistory({ name, message }), + queryFn: () => FlowsService.readMessageHistory({ flowId: name, message }), }) /** Other flows change rarely; suggestions do not need them fresh to the second. */ @@ -134,7 +134,7 @@ export function usePublish(name: string) { const queryClient = useQueryClient() return useMutation({ mutationFn: (version: number) => - FlowsService.publishFlow({ name, requestBody: { version } }), + FlowsService.publishFlow({ flowId: name, requestBody: { version } }), onSuccess: (detail) => { queryClient.setQueryData(flowKeys.detail(name), detail) queryClient.invalidateQueries({ queryKey: flowKeys.all, exact: true }) @@ -146,7 +146,7 @@ export function usePublish(name: string) { export function useDiscardDraft(name: string) { const queryClient = useQueryClient() return useMutation({ - mutationFn: () => FlowsService.discardDraft({ name }), + mutationFn: () => FlowsService.discardDraft({ flowId: name }), onSuccess: (detail) => { queryClient.setQueryData(flowKeys.detail(name), detail) queryClient.invalidateQueries({ queryKey: flowKeys.all, exact: true }) @@ -189,7 +189,7 @@ export function useAutosave(name: string): { const mutation = useMutation({ mutationFn: (definition: FlowDef_Input) => FlowsService.saveFlow({ - name, + flowId: name, // The ref holds the freshest version this client has seen; before the // first save that is the one the document was loaded with. Sending // `undefined` would drop the field and let the server read the default, @@ -254,7 +254,7 @@ export function useAutosave(name: string): { }) return } - const current = await FlowsService.readFlow({ name }) + const current = await FlowsService.readFlow({ flowId: name }) version.current = current.definition.version ?? null const definition = pending.current ?? sent.current pending.current = null diff --git a/frontend/src/components/Health/FlowTable.tsx b/frontend/src/components/Health/FlowTable.tsx index adfe979..88640ca 100644 --- a/frontend/src/components/Health/FlowTable.tsx +++ b/frontend/src/components/Health/FlowTable.tsx @@ -132,8 +132,8 @@ function Row({ const toggle = useMutation({ mutationFn: (next: boolean) => next - ? FlowsService.startFlow({ name: flow.name }) - : FlowsService.stopFlow({ name: flow.name }), + ? FlowsService.startFlow({ flowId: flow.name }) + : FlowsService.stopFlow({ flowId: flow.name }), onSuccess: () => { queryClient.invalidateQueries({ queryKey: flowKeys.all }) queryClient.invalidateQueries({ queryKey: flowKeys.detail(flow.name) }) diff --git a/frontend/src/components/Runs/OpenInDashboard.tsx b/frontend/src/components/Runs/OpenInDashboard.tsx index 976558c..db899a2 100644 --- a/frontend/src/components/Runs/OpenInDashboard.tsx +++ b/frontend/src/components/Runs/OpenInDashboard.tsx @@ -59,7 +59,7 @@ export function OpenInDashboard({ const generate = useMutation({ mutationFn: (name: string) => - DashboardsService.generateResultsDashboard({ flow: name }), + DashboardsService.generateResultsDashboard({ flowId: name }), onSuccess: (made) => { client.invalidateQueries({ queryKey: dashboardKeys.all }) show(made.name) diff --git a/frontend/src/routes/_layout/dashboards/index.tsx b/frontend/src/routes/_layout/dashboards/index.tsx index f45c134..51a2e35 100644 --- a/frontend/src/routes/_layout/dashboards/index.tsx +++ b/frontend/src/routes/_layout/dashboards/index.tsx @@ -46,7 +46,7 @@ function Dashboards() { const create = useMutation({ mutationFn: (dashboard: string) => - DashboardsService.createDashboard({ name: dashboard }), + DashboardsService.createDashboard({ requestBody: { name: dashboard } }), onSuccess: (created) => { queryClient.invalidateQueries({ queryKey: dashboardKeys.all }) setDialogOpen(false) @@ -64,7 +64,7 @@ function Dashboards() { const publishAll = usePublishAll( (dashboard) => DashboardsService.publishDashboard({ - name: dashboard, + dashboardId: dashboard, requestBody: { version: data?.data.find((d) => d.name === dashboard)?.version ?? 1, }, @@ -88,13 +88,15 @@ function Dashboards() { const selection = useSelection(dashboards.map((dashboard) => dashboard.name)) const remove = useDeleteSelected( - (dashboard) => DashboardsService.deleteDashboard({ name: dashboard }), + (dashboard) => + DashboardsService.deleteDashboard({ dashboardId: dashboard }), "dashboard", () => queryClient.invalidateQueries({ queryKey: dashboardKeys.all }), ) const duplicate = useDuplicateSelected( - (dashboard) => DashboardsService.duplicateDashboard({ name: dashboard }), + (dashboard) => + DashboardsService.duplicateDashboard({ dashboardId: dashboard }), "dashboard", () => { selection.clear() diff --git a/frontend/src/routes/_layout/flows/index.tsx b/frontend/src/routes/_layout/flows/index.tsx index 306146b..1a764a5 100644 --- a/frontend/src/routes/_layout/flows/index.tsx +++ b/frontend/src/routes/_layout/flows/index.tsx @@ -50,7 +50,7 @@ function Flows() { const create = useMutation({ mutationFn: (flow: string) => FlowsService.saveFlow({ - name: flow, + flowId: flow, requestBody: { name: flow, nodes: [], inputs: [] }, }), onSuccess: (_saved, flow) => { @@ -66,7 +66,7 @@ function Flows() { const publishAll = usePublishAll( (flow) => FlowsService.publishFlow({ - name: flow, + flowId: flow, requestBody: { version: data?.data.find((f) => f.name === flow)?.version ?? 1, }, @@ -90,7 +90,7 @@ function Flows() { const selection = useSelection(flows.map((flow) => flow.name)) const remove = useDeleteSelected( async (flow) => { - await FlowsService.deleteFlow({ name: flow }) + await FlowsService.deleteFlow({ flowId: flow }) if (withRuns) await RunsService.deleteRuns({ flow, history: true }) }, "flow", @@ -98,7 +98,7 @@ function Flows() { ) const duplicate = useDuplicateSelected( - (flow) => FlowsService.duplicateFlow({ name: flow }), + (flow) => FlowsService.duplicateFlow({ flowId: flow }), "flow", () => { selection.clear()