diff --git a/backend/fluksio/main.py b/backend/fluksio/main.py index cf1cde5..842bfe2 100644 --- a/backend/fluksio/main.py +++ b/backend/fluksio/main.py @@ -67,7 +67,7 @@ from fluksio.flow.resources import ( from fluksio.flow.runs import RUN_STATE_TTL, RunService, sweep_artifacts from fluksio.flow.secrets import init_secrets from fluksio.flow.state import MemoryState, RedisState, StateBackend -from fluksio.flow.store import FlowStore, IdentityChange, InvalidName +from fluksio.flow.store import FlowStore, IdentityChange, InvalidName, RecordedKey from fluksio.flow.tombstones import Tombstones from fluksio.flow.watchdog import LoopWatchdog from fluksio.flow.workers import PythonWorkerPool @@ -673,8 +673,10 @@ async def invalid_name(request: Request, exc: InvalidName) -> JSONResponse: # n @app.exception_handler(IdentityChange) async def identity_change(request: Request, exc: IdentityChange) -> JSONResponse: # noqa: ARG001 - """A save that would change an object's id or key, which no save may.""" - return JSONResponse(status_code=409, content={"detail": str(exc)}) + """A save that would change an object's id or key, which no save may. A + restore refused for its key names the one its history is filed under.""" + key = {"key": exc.key} if isinstance(exc, RecordedKey) else {} + return JSONResponse(status_code=409, content={"detail": str(exc), **key}) @app.exception_handler(DeviceUnavailable) diff --git a/backend/tests/api/routes/test_flow_ids.py b/backend/tests/api/routes/test_flow_ids.py index 651eb5c..a3b3050 100644 --- a/backend/tests/api/routes/test_flow_ids.py +++ b/backend/tests/api/routes/test_flow_ids.py @@ -391,7 +391,7 @@ def test_a_pinned_restore_takes_its_history_back( headers=superuser_token_headers, json={**plain, "name": f"{key}_b", "id": pin}, ) - assert other.status_code == 409 + assert other.status_code == 409 and other.json()["key"] == key restored = client.post( f"{FLOWS}/", headers=superuser_token_headers, json={**plain, "id": pin} ) diff --git a/docs/code/api.md b/docs/code/api.md index 24fe495..aa819e7 100644 --- a/docs/code/api.md +++ b/docs/code/api.md @@ -60,6 +60,9 @@ a token. Password recovery uses `POST /password-recovery/{email}` and | `POST` | `/flows/{id}/discard-draft` | discard unpublished edits | | `POST` | `/flows/{id}/duplicate` | copy into an unpublished `_copy` with an id of its own; code nodes need a superuser | | `DELETE` | `/flows/{id}` | delete; refused while a run is queued or running | +| `POST` | `/flows/{id}/rename/plan` | what renaming the key to `{"new_name": ...}` would change; superuser only | +| `POST` | `/flows/{id}/rename` | rename the key as planned; superuser only | +| `DELETE` | `/flows/{id}/rename/{request_id}` | call off a rename still waiting for work to finish | | `GET` | `/flows/node-types` | list types and parameter schemas | | `GET` | `/flows/graph` | read the graph shown on Home | @@ -74,7 +77,10 @@ answers 201 with the new flow. The server issues an opaque id such as and restores do. A pinned id has to be free and cannot be a path of this API like `graph`. If runs or metrics of a deleted flow are still filed under it, the new flow takes that history back and has to use the key it was recorded -under. Creating a flow with code nodes needs a superuser, as saving one does. +under. Another key is refused with 409, and `key` in the answer names the +recorded one. To bring it back under another key, restore it under the +recorded key and then rename it. Creating a flow with code nodes needs a +superuser, as saving one does. A key that still has values or queued work from an earlier flow is refused with 409 until that is gone. @@ -92,6 +98,35 @@ lists `flow_ids` in the order of its `flows`, a device binding has `flow_id` and a search entry has `id`. Open things by these ids, not by the part of a `.` before the dot. +### Renaming a key + +A superuser renames a flow's key in two steps. `POST /flows/{id}/rename/plan` +with `{"new_name": "warmth"}` changes nothing. It lists the flows and +dashboards the rename would rewrite, counts the history, live state, queued +work and alert channels that would move, and gives warnings and `refusals`, +`allowed`, the flow's `name` and `version` and a `fingerprint`. +`POST /flows/{id}/rename` then sends the plan's `name` as `expected_name`, its +`version` as `expected_version`, `new_name` and `fingerprint`, plus a +`request_id` of your own of up to 64 characters. It answers with the new versions of the rewritten documents. Sending the same +`request_id` again gets the same answer. + +The whole engine pauses for a flow rename. The request first waits for running +work to finish, however long that takes, and `GET /observability/summary` shows +the rename under `rename`, with what it waits for in `busy`. Until it starts +changing things, `DELETE /flows/{id}/rename/{request_id}` or closing the +connection calls it off, and the request answers 409 `aborted`. Meanwhile other +changes get 503 with code `renaming` and `Retry-After`, and so do the hooks of +the flows being rewritten. Reads keep working. + +A refusal answers 409 with its `code` and the fresh `plan`, and so does +`plan_changed`, when something the rename would rewrite changed since the plan. +409 `stale` means the flow itself changed. A key that is no valid key answers +422. + +`POST /dashboards/{id}/rename/plan` and `POST /dashboards/{id}/rename` work the +same way for a dashboard. Only its own documents change, and the engine keeps +running. + ### Node source ```sh @@ -238,6 +273,7 @@ reclaiming space sooner. | `PUT` | `/dashboards/{id}` | save a draft of an existing dashboard | | `POST` | `/dashboards/{id}/publish`, `/dashboards/{id}/discard` | publish or discard | | `POST` | `/dashboards/{id}/duplicate` | copy into an unpublished `_copy` with an id of its own | +| `POST` | `/dashboards/{id}/rename/plan`, `/dashboards/{id}/rename` | plan or rename the key, see [Renaming a key](#renaming-a-key); superuser only | | `GET` | `/devices/` | list devices, with `online`, `last_seen`, `report` and `reported_at` | | `POST`, `GET`, `PUT`, `DELETE` | `/devices/{id}` | create, read, save or delete one; a save sends the `version` it read and gets 409 when someone saved first. A read adds `io` and `bindings`, its ports as the device nodes of published flows wire them | | `POST` | `/devices/pair` | start device pairing, optionally with `{name, board, firmware}`; answers `code`, `secret` and the two-character `check` the device shows beside its code; unauthenticated, five a minute per address (the portal limits its own), the oldest of 50 waiting requests from the same side gives way | @@ -317,6 +353,10 @@ messages are named, and by id in `flow_id`, which is empty for the engine as a whole. `dashboard_changed` names the dashboard by id, or nothing when a device's own wiring moved. +A key rename sends `rename_waiting` with `busy` while it waits for work to +finish, and `renamed` with `kind`, `id`, `old` and `new` once it is done. A +client then reads again what it shows, at the same ids. + A client asks for the bytes of the media it shows with `{"type": "media", "names": [...]}`, which replaces what it asked for before, for at most 8 messages, the last 8 it names when it names more. Each frame held in memory then arrives as a binary @@ -398,7 +438,7 @@ offer and candidates are not queued behind frames. | `409` | version conflict, duplicate, or resource still in use | | `413` | artifact upload exceeds the configured limit | | `422` | invalid parameters, ports, or bindings | -| `503` | subsystem unavailable | +| `503` | subsystem unavailable, or a key rename in progress, retry after `Retry-After` | For a version conflict, read `current_version`, reload, merge your edits, and save again. Other `409` responses describe their own conflict. diff --git a/docs/code/cli.md b/docs/code/cli.md index 3652281..89ad14e 100644 --- a/docs/code/cli.md +++ b/docs/code/cli.md @@ -221,7 +221,9 @@ flow reports `unchanged`. Sync finds a flow by its declared key. The first sync creates it, and the engine gives it an id, which sync prints beside the key. Another engine gives the same flow another id, so the key is what your code and every instance -share. See the +share. The editor does not rename the key of a synced flow. Changing the key +in your code makes the next sync create a second flow under the new key, with +an empty history, and leaves the old one as it is. See the [data science guide](../getting-started/data-science.md) for a complete example. ### `fluksio run` diff --git a/docs/concepts/flows.md b/docs/concepts/flows.md index 04b374b..cc4a524 100644 --- a/docs/concepts/flows.md +++ b/docs/concepts/flows.md @@ -44,6 +44,26 @@ def process(temperature): The canvas shows cross-flow messages as labelled endpoints. +### Ids and keys + +A flow has two names. Its **key**, such as `house`, is the namespace of its +messages, so it is what other flows, dashboards and your scripts write. Its +**id** says which flow it is and never changes. The editor's address, webhook +URLs (`/hooks//...`) and the API's paths use the id. A flow from +before ids existed has its key as its id, and a new one gets an opaque id such +as `f_k3q7m2xa`. Dashboards have an id and a key in the same way. + +A superuser can rename a flow's key. Everything named under it moves along, +so `house.temperature` becomes `home.temperature` in every flow and dashboard +bound to it, together with its history, live values and queued work. What uses +the id keeps working. Whatever names the old key from outside, such as a call +to `/messages/house.temperature` or a script, has to be changed by hand. The +whole engine pauses while a flow's key is renamed, see +[Renaming the key](../interface/flow-editor.md#renaming-the-key). + +Deleting a flow frees its key. Its runs stay under the key it last had, and a +new flow that takes the key starts with an empty history. + ## Node types and payloads Function nodes run your Python code. Built-in nodes handle MQTT, HTTP, diff --git a/docs/concepts/runs.md b/docs/concepts/runs.md index 4b474d6..351e814 100644 --- a/docs/concepts/runs.md +++ b/docs/concepts/runs.md @@ -216,7 +216,7 @@ such as `metric.final_metrics.train_loss`. A row names the flow by its id in `flow` and by its key in `flow_name`. History stays with the id, so a flow keeps its runs whatever its key is later, and a new flow that takes the key of a deleted one starts without them. Metric names keep the key, as in -`train.loss`. +`train.loss`, and a key rename moves them to the new one. Python returns the same rows directly. `flow` takes the flow's id or key, as everywhere in the client: diff --git a/docs/reference/node-types.md b/docs/reference/node-types.md index f093157..5716f29 100644 --- a/docs/reference/node-types.md +++ b/docs/reference/node-types.md @@ -63,7 +63,9 @@ The ports pick the direction: outputs only make a **listener**, inputs a **sender**. A listener is a **webhook**: the engine mounts a route at `/hooks/{flow id}/{url}/{secret}` while the flow runs. The flow's id is in the URL rather than its key, so the URL stays the same when the key changes, and -for a flow made before ids existed it reads as the key. A sender requests `url` +for a flow made before ids existed it reads as the key. While a rename rewrites +the flow, the hook answers 503 with `Retry-After`, and a served socket closes +with code 1013, try again later. A sender requests `url` and publishes a JSON object response on its outputs, one key per output port. Keys no port declares are dropped, and a response that is not an object publishes nothing.