Rename orchestration, splice and routes: a Renamer runs one rename at a time, closing the gate, taking the controller lock and holding the engine, each waiting without a limit and saying what it waits for (summary rename status, rename_waiting); DELETE on the pending request or a closed connection calls it off until the journal begins (409 aborted); then journal, steps and audit, retried in the background on failure (503 completing); splice_renamed rebuilds the affected flows under the held lock keeping stopped, paused and parked work, history limits and device wiring refreshed before activation; AlertManager.rename and EventBus.rename/settle follow the names; flow and dashboard rename routes answer by code and repeat stored results; renamed, dashboard_changed and pipeline_rebuilt after release; fluksio serve reads flow files without building a FlowStore

This commit is contained in:
2026-11-02 22:14:44 +01:00
parent 2d65118d82
commit ba4aae390d
16 changed files with 2174 additions and 22 deletions
+583 -10
View File
@@ -6,7 +6,9 @@ live state and queued work, so nothing may write through the API meanwhile.
a rename would change and what refuses it, and `rewrite_documents` is its
document part. The `Journal` records each step before the next, so a rename a
crash interrupted is undone or finished on the next start (`recover_documents`,
`recover_runtime`). ``docs/private/id_refac_rename.md`` has the whole protocol.
`recover_runtime`). `Renamer` runs one: the gate, the engine holds, the
journal and its steps, and the engine put back together under the new names.
``docs/private/id_refac_rename.md`` has the whole protocol.
"""
from __future__ import annotations
@@ -20,8 +22,8 @@ import re
import threading
import time
from collections import Counter
from collections.abc import Callable, Iterable, Iterator
from dataclasses import dataclass
from collections.abc import Awaitable, Callable, Iterable, Iterator
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Literal, cast
@@ -34,8 +36,13 @@ from starlette.types import Scope
from fluksio.core.config import settings
from fluksio.core.db import engine as db_engine
from fluksio.flow.alerts import AlertsConfig
from fluksio.flow.controller import HOOK_PREFIX, FlowController
from fluksio.flow.alerts import AlertManager, AlertsConfig
from fluksio.flow.controller import (
HOOK_PREFIX,
REBUILD_WAIT,
FlowController,
RebuildBusy,
)
from fluksio.flow.dashboards import (
DASHBOARD_BINDINGS,
DASHBOARD_DIR,
@@ -44,10 +51,12 @@ from fluksio.flow.dashboards import (
results_name,
)
from fluksio.flow.events import EventBus
from fluksio.flow.executor import wait_out
from fluksio.flow.executor import HOLD_POLL_S, ExecutionService, wait_out
from fluksio.flow.identity import _replace, write_documents
from fluksio.flow.messages import flow_of
from fluksio.flow.metrics import MetricsCollector
from fluksio.flow.namespace import FLOW_FIELDS, NAME, OUTPUTS, Rename, names, rebind
from fluksio.flow.pipeline import Pipeline
from fluksio.flow.queue import WorkQueue
from fluksio.flow.schemas import FlowDef, _validate_name
from fluksio.flow.state import MemoryState, StateBackend
@@ -73,13 +82,13 @@ _READS = frozenset({"GET", "HEAD", "OPTIONS"})
#: and are held there), a browser's push subscription, the portal, workers
#: attaching, artifact bytes (which are named by digest, and which a node on a
#: remote worker writes while the rename waits for it), the socket ticket and
#: the rename routes, which keep their own order.
#: the rename routes (plan, rename and its cancel), which keep their own order.
_EXEMPT = re.compile(
r"/(?:login|password-recovery|password-recovery-html-content|reset-password"
r"|oauth|users|ai|cloud|workers|artifacts)(?:/|$)"
r"|/alerts/webpush(?:/|$)"
r"|/flows/ws-ticket$"
r"|/(?:flows|dashboards)/[^/]+/rename(?:/plan)?$"
r"|/(?:flows|dashboards)/[^/]+/rename(?:/[^/]+)?$"
)
@@ -353,6 +362,38 @@ class RenamePlanRequest(BaseModel):
new_name: str
class RenameRequest(BaseModel):
"""A rename as its plan described it. Refused when the object or the
plan moved since (409 ``stale`` or ``plan_changed``)."""
#: The client's own id for this request, which a retry repeats.
request_id: str = Field(min_length=1, max_length=64)
expected_name: str
#: The working copy's version the plan was made against.
expected_version: int
new_name: str
fingerprint: str
class RenameStatus(BaseModel):
"""A rename under way, for whoever is waiting on it or was turned away."""
op: str
kind: Kind
id: str
old: str
new: str
#: ``draining`` while it waits for work to finish, when it can still be
#: called off, then the journal's phase: ``prepared`` while the
#: documents are written and ``forward`` while the rest moves.
phase: str
#: What it is waiting for: API requests (``POST /api/v1/...``), nodes
#: and messages running outside the queue, and nodes whose cascades run.
busy: list[str] = []
#: Why the last attempt at a step failed, while it is retried.
error: str = ""
class RenameRefusal(BaseModel):
"""Why a rename cannot go ahead, with the ids of what is in the way."""
@@ -856,13 +897,15 @@ def read_journal() -> Journal | None:
class RenameConflict(Exception):
"""A rename request that cannot go ahead as asked, with the status and
code it is answered with."""
code it is answered with, and what else the answer carries: the ``op``
in the way, or the fresh ``plan`` a refusal comes from."""
def __init__(self, status: int, code: str, detail: str) -> None:
def __init__(self, status: int, code: str, detail: str, **extra: Any) -> None:
super().__init__(detail)
self.status = status
self.code = code
self.detail = detail
self.extra = extra
def repeated(
@@ -1247,3 +1290,533 @@ async def retry_until_done(attempt: Callable[[], None], events: EventBus) -> Non
)
await asyncio.sleep(wait)
wait = min(wait * 2, longest)
# -----------------------------------------------------------------------------
# Running one
# -----------------------------------------------------------------------------
#: Refusals of a key that is no key at all, answered 422 rather than 409.
_MALFORMED = frozenset({"invalid_name", "reserved_shape"})
def _aborted() -> RenameConflict:
return RenameConflict(409, "aborted", "The rename was called off; nothing changed")
@dataclass
class _Held:
"""What a rename holds so far, so whatever stops it lets go of exactly
that, last taken first."""
gate: bool = False
lock: bool = False
pipeline: Pipeline | None = None
execution: ExecutionService | None = None
queue: WorkQueue | None = None
#: Keys of the flows taken down, which start again as they were if the
#: rename does not go ahead.
torn_down: list[str] = field(default_factory=list)
class Renamer:
"""Runs renames, one at a time, in the order section 2.3 of the protocol
gives.
A flow's rename closes the gate, takes the controller lock and holds the
engine. Each of those waits for the work in front of it without a limit,
says what it waits for (`pending`, and a ``rename_waiting`` event) and
can be called off meanwhile, by `cancel` or by the caller going away.
From the journal's first write on it only goes forward: a step that fails
keeps everything held and is tried again in the background. A dashboard's
rename is its documents and the audit, with nothing held.
"""
def __init__(
self,
controller: FlowController,
dashboards: DashboardStore,
gate: RenameGate,
events: EventBus,
alerts: AlertManager | None = None,
metrics: MetricsCollector | None = None,
) -> None:
self.controller = controller
self.dashboards = dashboards
self.gate = gate
self.events = events
self.alerts = alerts
self.metrics = metrics
#: The rename under way, None when there is none.
self.pending: RenameStatus | None = None
self._abort = threading.Event()
#: A rename being finished in the background once a step failed.
self._resume: asyncio.Task[None] | None = None
async def rename(
self,
kind: Kind,
obj_id: str,
request: RenameRequest,
actor: str = "",
disconnected: Callable[[], Awaitable[bool]] | None = None,
) -> RenameResult:
"""Rename the flow or dashboard *obj_id* as *request* asks, or answer
it with the result it got before.
*disconnected* says when the caller has gone, which calls the rename
off as `cancel` does, for as long as that is still possible.
:raises RenameConflict: For every answer but the result.
:raises FlowNotFound: For a flow that does not exist.
:raises DashboardNotFound: For a dashboard that does not exist.
"""
if self.pending is not None:
mine = self.pending.op == request.request_id
raise RenameConflict(
503,
"completing" if mine else "rename_in_progress",
f"The rename of '{self.pending.old}' to '{self.pending.new}' "
"is under way",
op=self.pending.op,
)
status = self.pending = RenameStatus(
op=request.request_id,
kind=kind,
id=obj_id,
old=request.expected_name,
new=request.new_name,
phase="draining",
)
abort = self._abort = threading.Event()
task = asyncio.create_task(self._run(status, request, actor, abort))
# Read here, or by nobody once the caller has gone.
task.add_done_callback(lambda done: done.cancelled() or done.exception())
watcher = (
asyncio.create_task(self._watch(disconnected, abort))
if disconnected is not None
else None
)
try:
# Shielded: a caller going away calls the rename off while it
# still can be, and never stops it halfway after.
return await asyncio.shield(task)
except asyncio.CancelledError:
abort.set()
raise
finally:
if watcher is not None:
watcher.cancel()
def cancel(self, obj_id: str, op: str) -> None:
"""Call off the rename *op* of *obj_id* while it still waits for the
engine. It answers 409 ``aborted`` once everything is as it was.
:raises KeyError: When no such rename is under way.
:raises RenameConflict: 409 ``too_late`` once it changes things.
"""
status = self.pending
if status is None or (status.id, status.op) != (obj_id, op):
raise KeyError(op)
if status.phase != "draining":
raise RenameConflict(
409, "too_late", "The rename has begun changing things and finishes"
)
self._abort.set()
@staticmethod
async def _watch(
disconnected: Callable[[], Awaitable[bool]], abort: threading.Event
) -> None:
while not abort.is_set():
if await disconnected():
abort.set()
return
await asyncio.sleep(HOLD_POLL_S)
def _report(self, busy: list[str]) -> None:
"""What the rename waits for, each time that changes. From a thread."""
status = self.pending
if status is None:
return
self.events.publish(
{
"type": "rename_waiting",
"op": status.op,
"kind": status.kind,
"id": status.id,
"busy": busy,
"ts": time.time(),
}
)
status.busy = busy
async def _run(
self,
status: RenameStatus,
request: RenameRequest,
actor: str,
abort: threading.Event,
) -> RenameResult:
try:
if status.kind == "dashboard":
return await self._rename_dashboard(status, request, actor)
checked = await asyncio.to_thread(self._checked, "flow", status.id, request)
if isinstance(checked, RenameResult):
return checked
return await self._rename_flow(status, checked, request, actor, abort)
finally:
# Kept while its journal is not final: it is being finished, or
# a restart has to.
if self.pending is status and status.phase in ("draining", "rolled_back"):
self.pending = None
def _checked(
self, kind: Kind, obj_id: str, request: RenameRequest
) -> RenamePlan | RenameResult:
"""The result a request asked again got, or the plan it goes ahead
on, once that is what the request was planned against. Blocking.
:raises RenameConflict: 409 ``stale`` or ``plan_changed``, or the
plan's first refusal: 422 for a key that is no key, else 409.
"""
current: FlowDef | DashboardDef = (
self.controller.store.read_flow(obj_id, draft=True)
if kind == "flow"
else self.dashboards.read(obj_id, draft=True)
)
stored = repeated(
request.request_id,
current.name,
current.version,
request.expected_name,
request.expected_version,
)
if stored is not None:
return stored
plan = plan_rename(
self.controller,
self.dashboards,
kind,
obj_id,
request.new_name,
self.alerts.config if self.alerts is not None else AlertsConfig(),
)
if plan.refusals:
first = plan.refusals[0]
raise RenameConflict(
422 if first.code in _MALFORMED else 409,
first.code,
first.detail,
plan=plan.model_dump(mode="json"),
)
if plan.fingerprint != request.fingerprint:
raise RenameConflict(
409,
"plan_changed",
"What the rename changes has moved since it was planned",
plan=plan.model_dump(mode="json"),
)
return plan
async def _rename_flow(
self,
status: RenameStatus,
plan: RenamePlan,
request: RenameRequest,
actor: str,
abort: threading.Event,
) -> RenameResult:
controller, op = self.controller, request.request_id
rename = Rename(plan.name, plan.new_name)
held = _Held()
try:
await self._hold(held, op, plan, abort)
# Again, now that nothing can move underneath it.
plan = cast(
RenamePlan,
await asyncio.to_thread(self._checked, "flow", plan.id, request),
)
if abort.is_set():
raise _aborted()
# Built ones only: a flow that is only a draft runs nothing.
rebuilt = [
doc.id
for doc in plan.documents
if doc.kind == "flow" and doc.id in controller._flow_keys
]
for flow_id in rebuilt:
key = controller._flow_keys[flow_id]
held.torn_down.append(key)
await controller._teardown(key)
if controller.execution is not None:
await asyncio.to_thread(controller.execution.queue.hold)
held.queue = controller.execution.queue
# Whatever the old names published is recorded under them.
await self.events.settle()
if self.metrics is not None:
await self.metrics.flush_now()
rewrites = await asyncio.to_thread(
rewrite_documents,
controller.store,
self.dashboards,
"flow",
plan.id,
rename,
)
if abort.is_set():
raise _aborted()
# Synchronously after the last look at abort: `cancel` is refused
# from here, and the rename is finished rather than called off.
status.phase = "prepared"
try:
journal = await asyncio.to_thread(
lambda: begin(
op,
"flow",
plan.id,
rename,
rewrites,
actor=actor,
fingerprint=plan.fingerprint,
durable=not isinstance(controller.state, MemoryState),
rebuilt=rebuilt,
)
)
except BaseException:
status.phase = "draining"
raise
except BaseException:
await self._release(held)
raise
try:
await asyncio.to_thread(apply_documents, journal, controller.store)
except Exception as exc:
logger.exception("Could not write the renamed documents; undoing them")
try:
await asyncio.to_thread(roll_back, journal, controller.store)
status.phase = "rolled_back"
finally:
await self._release(held)
raise RenameConflict(
500, "failed", f"The documents could not be written: {exc}", op=op
) from exc
status.phase = "forward"
# Never started again as they were: the splice builds them anew.
held.torn_down = []
try:
await asyncio.to_thread(self._finish_steps, journal, status)
except Exception as exc:
self.events.publish(
{
"type": "engine_degraded",
"detail": f"A rename could not finish: {exc}",
}
)
self._resume = asyncio.create_task(
self._finish_later(journal, held, status)
)
raise RenameConflict(
503,
"completing",
f"The rename could not finish yet and is being retried: {exc}",
op=op,
) from None
return await self._complete(journal, held, status)
async def _hold(
self, held: _Held, op: str, plan: RenamePlan, abort: threading.Event
) -> None:
"""The gate, the controller lock, the pipeline and the execution
service, each once what runs in front of it has finished. What it got
is in *held*.
:raises RenameConflict: 409 ``aborted`` once called off, 503 ``busy``
when a rebuild keeps the controller for longer than one may.
"""
controller = self.controller
affected = [doc.id for doc in plan.documents if doc.kind == "flow"]
held.gate = await asyncio.to_thread(
self.gate.close, op, affected, abort, self._report
)
if not held.gate:
raise _aborted()
try:
await controller._acquire(REBUILD_WAIT)
except RebuildBusy as exc:
raise RenameConflict(503, "busy", str(exc), op=op) from None
held.lock = True
pipeline, execution = controller.pipeline, controller.execution
if pipeline is not None:
if not await asyncio.to_thread(pipeline.hold, abort, self._report):
raise _aborted()
held.pipeline = pipeline
if execution is not None:
if not await asyncio.to_thread(execution.hold, abort, self._report):
raise _aborted()
held.execution = execution
if self.pending is not None:
self.pending.busy = []
def _finish_steps(self, journal: Journal, status: RenameStatus) -> None:
"""Every step after the documents, the audit and the journal's end.
Blocking, and tried again until it succeeds."""
execution = self.controller.execution
try:
run_steps(
journal,
self.controller.state,
execution.queue if execution is not None else None,
)
finish(journal)
except Exception as exc:
status.error = journal.error or f"{type(exc).__name__}: {exc}"
raise
status.error = ""
async def _finish_later(
self, journal: Journal, held: _Held, status: RenameStatus
) -> None:
"""Decision 5: held, and tried again however long it takes."""
await retry_until_done(lambda: self._finish_steps(journal, status), self.events)
await self._complete(journal, held, status)
async def _complete(
self, journal: Journal, held: _Held, status: RenameStatus
) -> RenameResult:
"""The engine under the new names, then everything let go of and
everyone told. Its journal is done already: what is left here lives
in memory, which a restart builds afresh anyway."""
rename = journal.rename
if held.execution is not None:
held.execution.rename_reclaimed(rename)
self.events.rename(rename)
if self.alerts is not None:
self.alerts.rename(rename)
try:
await self.controller.splice_renamed(rename, journal.affected.rebuilt)
except Exception as exc:
# What a failed `reload_flow` leaves: those flows stopped, until
# they are published again.
logger.exception("Could not rebuild the renamed flows")
self.events.publish(
{
"type": "engine_degraded",
"detail": f"The renamed flows could not be rebuilt: {exc}",
}
)
await self._release(held)
if self.pending is status:
self.pending = None
self._announce(journal)
return cast(RenameResult, journal.result)
async def _rename_dashboard(
self, status: RenameStatus, request: RenameRequest, actor: str
) -> RenameResult:
done = await asyncio.to_thread(
self._dashboard_documents, status, request, actor
)
if isinstance(done, RenameResult):
return done
await retry_until_done(lambda: self._finish_steps(done, status), self.events)
if self.pending is status:
self.pending = None
self._announce(done)
return cast(RenameResult, done.result)
def _dashboard_documents(
self, status: RenameStatus, request: RenameRequest, actor: str
) -> Journal | RenameResult:
"""A dashboard's documents, renamed and committed. Blocking.
Under the dashboard store's lock, which every dashboard save takes
before the flow store's, so none lands between the recheck and the
write. Nothing else names a dashboard's key, so nothing is held.
"""
store = self.controller.store
with self.dashboards._lock:
checked = self._checked("dashboard", status.id, request)
if isinstance(checked, RenameResult):
return checked
rename = Rename(checked.name, checked.new_name)
rewrites = rewrite_documents(
store, self.dashboards, "dashboard", status.id, rename
)
journal = begin(
request.request_id,
"dashboard",
status.id,
rename,
rewrites,
actor=actor,
fingerprint=checked.fingerprint,
)
status.phase = "prepared"
try:
apply_documents(journal, store)
except Exception as exc:
roll_back(journal, store)
status.phase = "rolled_back"
raise RenameConflict(
500,
"failed",
f"The documents could not be written: {exc}",
op=request.request_id,
) from exc
status.phase = "forward"
return journal
async def _release(self, held: _Held) -> None:
"""Let go of what *held* says, last taken first. Flows taken down for
a rename that does not go ahead start again as they were."""
if held.queue is not None:
try:
await asyncio.to_thread(held.queue.release)
except Exception:
# Left in the held list, which the next start releases.
logger.exception("Could not release the work held for a rename")
for key in held.torn_down:
await self.controller._activate(key)
if held.execution is not None:
held.execution.release()
if held.pipeline is not None:
held.pipeline.release()
if held.lock:
self.controller._lock.release()
if held.gate:
self.gate.open()
def _announce(self, journal: Journal) -> None:
"""Tell the clients, once everything is let go of: editors and panels
refetch, and the canvas takes the pipeline as it is now."""
now = time.time()
self.events.publish(
{
"type": "renamed",
"kind": journal.kind,
"id": journal.id,
"old": journal.old,
"new": journal.new,
"op": journal.op,
"ts": now,
}
)
for dash_id in journal.affected.dashboards:
self.events.publish(
{"type": "dashboard_changed", "dashboard": dash_id, "ts": now}
)
if journal.kind != "flow":
return
# What the devices' ports are wired to moved with the flow.
self.events.publish({"type": "dashboard_changed", "dashboard": "", "ts": now})
controller = self.controller
self.events.publish(
{
"type": "pipeline_rebuilt",
"issues": [issue.model_dump() for issue in controller.issues],
"nodes": [status.model_dump() for status in controller.node_statuses()],
"paused": controller.paused_flows(),
}
)