Rename orchestration, splice and routes: a Renamer runs one rename at a time, closing the gate, taking the controller lock and holding the engine, each waiting without a limit and saying what it waits for (summary rename status, rename_waiting); DELETE on the pending request or a closed connection calls it off until the journal begins (409 aborted); then journal, steps and audit, retried in the background on failure (503 completing); splice_renamed rebuilds the affected flows under the held lock keeping stopped, paused and parked work, history limits and device wiring refreshed before activation; AlertManager.rename and EventBus.rename/settle follow the names; flow and dashboard rename routes answer by code and repeat stored results; renamed, dashboard_changed and pipeline_rebuilt after release; fluksio serve reads flow files without building a FlowStore

This commit is contained in:
2026-11-02 22:14:44 +01:00
parent 2d65118d82
commit ba4aae390d
16 changed files with 2174 additions and 22 deletions
+11
View File
@@ -21,6 +21,7 @@ from fluksio.flow.controller import FlowController
from fluksio.flow.dashboards import DashboardDef, DashboardStore
from fluksio.flow.devices import DeviceDef, DeviceStore, DeviceUnavailable, presence
from fluksio.flow.events import event_bus
from fluksio.flow.rename import Renamer
from fluksio.flow.workers import PythonWorkerPool
from fluksio.models import OAuthClient, TokenPayload, User, WorkerToken
@@ -583,6 +584,16 @@ def get_device_store(request: Request) -> DeviceStore:
DeviceStoreDep = Annotated[DeviceStore, Depends(get_device_store)]
def get_renamer(request: Request) -> Renamer:
renamer: Renamer | None = getattr(request.app.state, "renamer", None)
if renamer is None:
raise HTTPException(status_code=503, detail="The flow engine is not running")
return renamer
RenamerDep = Annotated[Renamer, Depends(get_renamer)]
def get_worker_pool(request: Request) -> PythonWorkerPool:
pool: PythonWorkerPool | None = getattr(request.app.state, "worker_pool", None)
if pool is None:
+35 -2
View File
@@ -7,7 +7,7 @@ label unique among dashboards and what ``GET /dashboards?name=`` looks up.
import time
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.concurrency import run_in_threadpool
from pydantic import BaseModel
@@ -15,6 +15,7 @@ from fluksio.api.deps import (
CurrentUser,
DashboardStoreDep,
FlowControllerDep,
RenamerDep,
check_pin,
get_current_active_superuser,
get_current_user,
@@ -30,7 +31,13 @@ from fluksio.flow.dashboards import (
results_dashboard,
)
from fluksio.flow.events import event_bus
from fluksio.flow.rename import RenamePlan, RenamePlanRequest, plan_rename
from fluksio.flow.rename import (
RenamePlan,
RenamePlanRequest,
RenameRequest,
RenameResult,
plan_rename,
)
from fluksio.flow.store import FlowNotFound, IdTaken, StaleVersion
from fluksio.models import Message
@@ -279,3 +286,29 @@ async def plan_dashboard_rename(
)
except DashboardNotFound:
raise _missing(dashboard_id)
@router.post(
"/{dashboard_id}/rename",
response_model=RenameResult,
dependencies=[Depends(get_current_active_superuser)],
)
async def rename_dashboard(
dashboard_id: str,
body: RenameRequest,
request: Request,
renamer: RenamerDep,
user: CurrentUser,
) -> Any:
"""Rename this dashboard's key, as its plan described it.
Its two documents and nothing else: devices list it by id, and nothing
binds to it by key. The same ``request_id`` again gets the same answer;
refusals are as for a flow.
"""
try:
return await renamer.rename(
"dashboard", dashboard_id, body, user.email, request.is_disconnected
)
except DashboardNotFound:
raise _missing(dashboard_id)
+62 -1
View File
@@ -35,6 +35,7 @@ from fluksio.api.deps import (
DashboardStoreDep,
FlowControllerDep,
HiddenDep,
RenamerDep,
SessionDep,
TokenDep,
check_pin,
@@ -69,7 +70,13 @@ from fluksio.flow.events import event_bus
from fluksio.flow.messages import MEDIA_FAMILIES, DType, flow_of, qualify
from fluksio.flow.nodes.wren import TRAIN_FLOW, WrenNode, pack_meta
from fluksio.flow.pipeline import EngineHeld, ValidationIssue
from fluksio.flow.rename import RenamePlan, RenamePlanRequest, plan_rename
from fluksio.flow.rename import (
RenamePlan,
RenamePlanRequest,
RenameRequest,
RenameResult,
plan_rename,
)
from fluksio.flow.runs import RunRejected
from fluksio.flow.schemas import (
NAME_PATTERN,
@@ -776,6 +783,60 @@ async def plan_flow_rename(
raise _missing(flow_id)
@router.post(
"/{flow_id}/rename",
response_model=RenameResult,
dependencies=[Depends(get_current_active_superuser)],
)
async def rename_flow(
flow_id: str,
body: RenameRequest,
request: Request,
renamer: RenamerDep,
user: CurrentUser,
) -> Any:
"""Rename this flow's key, as its plan described it.
Everything named under the key moves: every flow and dashboard binding
one of its messages, its history, live state and queued work. The engine
waits meanwhile, for as long as the work running when it was asked takes
to finish. Until then it can be called off, with ``DELETE`` on the
request or by closing the connection, and nothing changes (409
``aborted``); ``GET /observability/summary`` says what it waits for.
The same ``request_id`` again gets the same answer. A rename that
could not finish yet answers 503 ``completing`` and is finished in the
background. Refusals are the plan's codes (409, 422 for a key that is no
key), ``stale`` and ``plan_changed`` (409).
"""
try:
return await renamer.rename(
"flow", flow_id, body, user.email, request.is_disconnected
)
except FlowNotFound:
raise _missing(flow_id)
@router.delete(
"/{flow_id}/rename/{request_id}",
response_model=Message,
dependencies=[Depends(get_current_active_superuser)],
)
async def cancel_flow_rename(flow_id: str, request_id: str, renamer: RenamerDep) -> Any:
"""Call off a rename still waiting for the engine. It answers 409
``aborted`` once everything is as it was, and one that has begun to
change things refuses this with 409 ``too_late``."""
# Async, so on the event loop, where the rename decides when it is past
# calling off.
try:
renamer.cancel(flow_id, request_id)
except KeyError:
raise HTTPException(
status_code=404, detail=f"No rename '{request_id}' of '{flow_id}' waits"
)
return Message(message=f"Calling off rename '{request_id}'")
# -----------------------------------------------------------------------------
# Node source
# -----------------------------------------------------------------------------
@@ -28,6 +28,7 @@ from fluksio.api.deps import (
from fluksio.api.routes.runs import elapsed_ms
from fluksio.core.config import settings
from fluksio.flow.controller import NodeStatus
from fluksio.flow.rename import RenameStatus
from fluksio.flow.tombstones import names_of, shown_nodes, stored_node
from fluksio.models import EngineEvent, FlowRun, Message, MetricBucket
@@ -65,6 +66,9 @@ class HealthSummary(BaseModel):
#: route the client knows and the engine does not answers a flat 404. This
#: is what turns that into a sentence. Absent means older than this field.
version: str = ""
#: A key rename under way: what it waits for while it can still be
#: called off, and why a step failed while it is retried.
rename: RenameStatus | None = None
class SeriesPoint(BaseModel):
@@ -285,6 +289,7 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
f"{', '.join(sorted(e.id for e in failed_once))}"
)
renamer = getattr(request.app.state, "renamer", None)
return HealthSummary(
status="degraded" if problems else "ok",
problems=problems,
@@ -322,6 +327,7 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
else {"ewma": 0.0, "max_60s": 0.0}
),
version=__version__,
rename=renamer.pending if renamer is not None else None,
)
+11 -5
View File
@@ -99,18 +99,24 @@ def _mention_undeclared_cards() -> None:
if settings.FLOW_GPUS or machine_gpus():
return
from fluksio.flow.runs import required_resources
from fluksio.flow.store import FlowStore
from fluksio.flow.schemas import FlowDef
store = FlowStore(settings.FLOWS_DIR)
# Read as files, not through a `FlowStore`: building one can commit the
# store, and this runs before the engine has settled an interrupted
# rename and given every document its id.
asking = []
for name in store.list_flows():
for path in sorted(settings.FLOWS_DIR.glob("*/flow.json")):
# `_devices/flow.json` is a device, not a flow.
if path.parent.name.startswith("_"):
continue
try:
needs = required_resources(store.read_flow(name))
flow = FlowDef.model_validate_json(path.read_text())
needs = required_resources(flow)
except Exception:
# A flow that will not parse is the engine's to complain about.
continue
if (needs or {}).get("gpus"):
asking.append(name)
asking.append(flow.name)
if not asking:
return
named = ", ".join(sorted(asking)[:3]) + (" …" if len(asking) > 3 else "")
+23 -1
View File
@@ -17,7 +17,7 @@ import logging
import time
from collections import deque
from collections.abc import Callable
from typing import Any, Literal, Protocol
from typing import TYPE_CHECKING, Any, Literal, Protocol
import httpx
from pydantic import BaseModel, Field
@@ -25,6 +25,9 @@ from pydantic import BaseModel, Field
from fluksio.flow.events import EventBus
from fluksio.flow.secrets import resolve_params
if TYPE_CHECKING:
from fluksio.flow.namespace import Rename
logger = logging.getLogger(__name__)
# What is worth waking someone for. Everything else on the bus is traffic.
@@ -271,6 +274,25 @@ class AlertManager:
return
await self._deliver(event)
def rename(self, rename: Rename) -> None:
"""Follow a key rename: dashboard channels publish to the moved
message, as `alerts.json` now says, and what was sent, held back or
muted is kept under the new names with its timestamps, so nothing is
sent again. A connection waiting out its grace under an old name is
let go: its node reports again once it runs under the new one."""
config = self.config.model_copy(deep=True)
for channel in config.channels:
message = channel.config.get("message")
if channel.kind == "dashboard" and isinstance(message, str):
channel.config["message"] = rename.binding(message)
self.config = config
self._last_sent = {rename.dedup_key(k): v for k, v in self._last_sent.items()}
self._suppressed = {rename.dedup_key(k): v for k, v in self._suppressed.items()}
self._health_flips = {rename.name(k): v for k, v in self._health_flips.items()}
self._flapping = {rename.name(k): v for k, v in self._flapping.items()}
for node in [node for node in self._down if rename.name(node) != node]:
self._cancel_down(node)
def _cancel_down(self, node: str) -> None:
task = self._down.pop(node, None)
if task is not None:
+70 -1
View File
@@ -39,7 +39,7 @@ from fluksio.flow.demand import demand
from fluksio.flow.events import EventBus
from fluksio.flow.executor import ExecutionService
from fluksio.flow.messages import MessageSpec, flow_of, qualify
from fluksio.flow.namespace import touches
from fluksio.flow.namespace import Rename, touches
from fluksio.flow.nodes import (
RESERVED_SETTINGS,
AssistantNode,
@@ -913,6 +913,75 @@ class FlowController:
)
self._rewired()
async def splice_renamed(self, rename: Rename, rebuilt: list[str]) -> None:
"""Put the flows a key rename rewrote back into the graph, built from
their new documents and named as they are now.
Under the controller lock, which the rename holds, with the engine
held and the flows *rebuilt* (ids) torn down before their documents
changed. Not `reload_flow`, which would take the lock again, nor
`reload`, which clears every pause. Stopped flows stay stopped, which
is stored by id, pauses carry over under the new keys and parked work
stays parked. A rebuilt flow's quarantine is gone with its teardown,
as on a publish. What the documents feed is refreshed before anything
starts: history limits, device wiring and what devices are asked to
send.
"""
pipeline = self.pipeline
if pipeline is None:
return
paused = pipeline.paused_flows()
builds = await run_in_threadpool(
lambda: [(flow_id, self._build_one(flow_id)) for flow_id in rebuilt]
)
# By the name each node has now, so a failure carries over.
previous = {rename.name(node_id): e for node_id, e in self.loaded.items()}
loaded = dict(self.loaded)
keys = []
for flow_id, built in builds:
if built is None:
continue
flow, nodes, entries, initial_values, enabled = built
old, key = self._flow_keys.get(flow_id, flow.name), flow.name
if old != key:
pipeline.replace_flow(old, [])
pipeline.replace_flow(key, nodes, initial_values)
self._flow_keys[flow_id] = key
for node_id, entry in entries.items():
before = previous.get(node_id)
if before is not None and before.last_error:
entry.inherit_failure(before)
loaded = {
node_id: entry for node_id, entry in loaded.items() if entry.flow != old
} | entries
self._flow_inputs.pop(old, None)
self._flow_inputs[key] = _declared_inputs(flow)[0]
self.disabled = (self.disabled - {old}) | (set() if enabled else {key})
self.batch = (self.batch - {old}) | (
{key} if flow.mode == "batch" else set()
)
keys.append(key)
self.loaded = loaded
pipeline.set_disabled(self.disabled)
# The replace cleared the pause of every flow it touched.
for was in map(rename.flow, paused):
if not pipeline.is_paused(was):
pipeline.pause(was)
self.issues = _collect_issues(
self.loaded, pipeline, self._all_flow_inputs(), self.devices
)
if self.dashboards is not None:
self.set_history_limits(
await run_in_threadpool(self.dashboards.history_requirements)
)
if self.devices is not None:
# Taken note of without telling the devices: the rename does,
# once it has let go of the engine.
await run_in_threadpool(demand.refresh)
self._device_wiring = await run_in_threadpool(self.devices.wiring)
for key in keys:
await self._activate(key)
async def recheck(self, wait: float | None = FLOW_REBUILD_WAIT) -> None:
"""Validate the built graph again without rebuilding anything.
+29 -1
View File
@@ -15,7 +15,10 @@ import threading
from collections import deque
from collections.abc import AsyncIterator, Callable
from contextlib import asynccontextmanager
from typing import Any
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from fluksio.flow.namespace import Rename
logger = logging.getLogger(__name__)
@@ -24,6 +27,8 @@ DISPATCH_BUDGET = 1024
LOG_HISTORY = 400
#: How many of a message's last artifact digests are remembered.
RECENT_DIGESTS = 8
#: How many turns of the loop `settle` gives the dispatch at most.
SETTLE_TURNS = 100
def _critical(event: dict[str, Any]) -> bool:
@@ -211,6 +216,29 @@ class EventBus:
if self._loop is not None:
self._loop.call_soon(self._dispatch)
async def settle(self) -> None:
"""Wait until what was published so far has reached every subscriber.
For a rename, before any key changes: an event a node published under
the old key is then recorded under it. Bounded, since a busy bus may
never be empty.
"""
for _ in range(SETTLE_TURNS):
with self._pending_lock:
if not self._flush_scheduled:
return
await asyncio.sleep(0)
def rename(self, rename: Rename) -> None:
"""Keep emit counts and recent digests under the names a key rename
gave their nodes and messages, so a client reconnecting or a device
fetching a recent frame finds them there. Rebound, not changed in
place: publishers on other threads write to them."""
self.emits = {rename.name(node): n for node, n in list(self.emits.items())}
self.digests = {
rename.name(name): recent for name, recent in list(self.digests.items())
}
def dropped(self, queue: asyncio.Queue[dict[str, Any]]) -> bool:
"""Whether this subscriber lost events since it last asked.
+583 -10
View File
@@ -6,7 +6,9 @@ live state and queued work, so nothing may write through the API meanwhile.
a rename would change and what refuses it, and `rewrite_documents` is its
document part. The `Journal` records each step before the next, so a rename a
crash interrupted is undone or finished on the next start (`recover_documents`,
`recover_runtime`). ``docs/private/id_refac_rename.md`` has the whole protocol.
`recover_runtime`). `Renamer` runs one: the gate, the engine holds, the
journal and its steps, and the engine put back together under the new names.
``docs/private/id_refac_rename.md`` has the whole protocol.
"""
from __future__ import annotations
@@ -20,8 +22,8 @@ import re
import threading
import time
from collections import Counter
from collections.abc import Callable, Iterable, Iterator
from dataclasses import dataclass
from collections.abc import Awaitable, Callable, Iterable, Iterator
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Literal, cast
@@ -34,8 +36,13 @@ from starlette.types import Scope
from fluksio.core.config import settings
from fluksio.core.db import engine as db_engine
from fluksio.flow.alerts import AlertsConfig
from fluksio.flow.controller import HOOK_PREFIX, FlowController
from fluksio.flow.alerts import AlertManager, AlertsConfig
from fluksio.flow.controller import (
HOOK_PREFIX,
REBUILD_WAIT,
FlowController,
RebuildBusy,
)
from fluksio.flow.dashboards import (
DASHBOARD_BINDINGS,
DASHBOARD_DIR,
@@ -44,10 +51,12 @@ from fluksio.flow.dashboards import (
results_name,
)
from fluksio.flow.events import EventBus
from fluksio.flow.executor import wait_out
from fluksio.flow.executor import HOLD_POLL_S, ExecutionService, wait_out
from fluksio.flow.identity import _replace, write_documents
from fluksio.flow.messages import flow_of
from fluksio.flow.metrics import MetricsCollector
from fluksio.flow.namespace import FLOW_FIELDS, NAME, OUTPUTS, Rename, names, rebind
from fluksio.flow.pipeline import Pipeline
from fluksio.flow.queue import WorkQueue
from fluksio.flow.schemas import FlowDef, _validate_name
from fluksio.flow.state import MemoryState, StateBackend
@@ -73,13 +82,13 @@ _READS = frozenset({"GET", "HEAD", "OPTIONS"})
#: and are held there), a browser's push subscription, the portal, workers
#: attaching, artifact bytes (which are named by digest, and which a node on a
#: remote worker writes while the rename waits for it), the socket ticket and
#: the rename routes, which keep their own order.
#: the rename routes (plan, rename and its cancel), which keep their own order.
_EXEMPT = re.compile(
r"/(?:login|password-recovery|password-recovery-html-content|reset-password"
r"|oauth|users|ai|cloud|workers|artifacts)(?:/|$)"
r"|/alerts/webpush(?:/|$)"
r"|/flows/ws-ticket$"
r"|/(?:flows|dashboards)/[^/]+/rename(?:/plan)?$"
r"|/(?:flows|dashboards)/[^/]+/rename(?:/[^/]+)?$"
)
@@ -353,6 +362,38 @@ class RenamePlanRequest(BaseModel):
new_name: str
class RenameRequest(BaseModel):
"""A rename as its plan described it. Refused when the object or the
plan moved since (409 ``stale`` or ``plan_changed``)."""
#: The client's own id for this request, which a retry repeats.
request_id: str = Field(min_length=1, max_length=64)
expected_name: str
#: The working copy's version the plan was made against.
expected_version: int
new_name: str
fingerprint: str
class RenameStatus(BaseModel):
"""A rename under way, for whoever is waiting on it or was turned away."""
op: str
kind: Kind
id: str
old: str
new: str
#: ``draining`` while it waits for work to finish, when it can still be
#: called off, then the journal's phase: ``prepared`` while the
#: documents are written and ``forward`` while the rest moves.
phase: str
#: What it is waiting for: API requests (``POST /api/v1/...``), nodes
#: and messages running outside the queue, and nodes whose cascades run.
busy: list[str] = []
#: Why the last attempt at a step failed, while it is retried.
error: str = ""
class RenameRefusal(BaseModel):
"""Why a rename cannot go ahead, with the ids of what is in the way."""
@@ -856,13 +897,15 @@ def read_journal() -> Journal | None:
class RenameConflict(Exception):
"""A rename request that cannot go ahead as asked, with the status and
code it is answered with."""
code it is answered with, and what else the answer carries: the ``op``
in the way, or the fresh ``plan`` a refusal comes from."""
def __init__(self, status: int, code: str, detail: str) -> None:
def __init__(self, status: int, code: str, detail: str, **extra: Any) -> None:
super().__init__(detail)
self.status = status
self.code = code
self.detail = detail
self.extra = extra
def repeated(
@@ -1247,3 +1290,533 @@ async def retry_until_done(attempt: Callable[[], None], events: EventBus) -> Non
)
await asyncio.sleep(wait)
wait = min(wait * 2, longest)
# -----------------------------------------------------------------------------
# Running one
# -----------------------------------------------------------------------------
#: Refusals of a key that is no key at all, answered 422 rather than 409.
_MALFORMED = frozenset({"invalid_name", "reserved_shape"})
def _aborted() -> RenameConflict:
return RenameConflict(409, "aborted", "The rename was called off; nothing changed")
@dataclass
class _Held:
"""What a rename holds so far, so whatever stops it lets go of exactly
that, last taken first."""
gate: bool = False
lock: bool = False
pipeline: Pipeline | None = None
execution: ExecutionService | None = None
queue: WorkQueue | None = None
#: Keys of the flows taken down, which start again as they were if the
#: rename does not go ahead.
torn_down: list[str] = field(default_factory=list)
class Renamer:
"""Runs renames, one at a time, in the order section 2.3 of the protocol
gives.
A flow's rename closes the gate, takes the controller lock and holds the
engine. Each of those waits for the work in front of it without a limit,
says what it waits for (`pending`, and a ``rename_waiting`` event) and
can be called off meanwhile, by `cancel` or by the caller going away.
From the journal's first write on it only goes forward: a step that fails
keeps everything held and is tried again in the background. A dashboard's
rename is its documents and the audit, with nothing held.
"""
def __init__(
self,
controller: FlowController,
dashboards: DashboardStore,
gate: RenameGate,
events: EventBus,
alerts: AlertManager | None = None,
metrics: MetricsCollector | None = None,
) -> None:
self.controller = controller
self.dashboards = dashboards
self.gate = gate
self.events = events
self.alerts = alerts
self.metrics = metrics
#: The rename under way, None when there is none.
self.pending: RenameStatus | None = None
self._abort = threading.Event()
#: A rename being finished in the background once a step failed.
self._resume: asyncio.Task[None] | None = None
async def rename(
self,
kind: Kind,
obj_id: str,
request: RenameRequest,
actor: str = "",
disconnected: Callable[[], Awaitable[bool]] | None = None,
) -> RenameResult:
"""Rename the flow or dashboard *obj_id* as *request* asks, or answer
it with the result it got before.
*disconnected* says when the caller has gone, which calls the rename
off as `cancel` does, for as long as that is still possible.
:raises RenameConflict: For every answer but the result.
:raises FlowNotFound: For a flow that does not exist.
:raises DashboardNotFound: For a dashboard that does not exist.
"""
if self.pending is not None:
mine = self.pending.op == request.request_id
raise RenameConflict(
503,
"completing" if mine else "rename_in_progress",
f"The rename of '{self.pending.old}' to '{self.pending.new}' "
"is under way",
op=self.pending.op,
)
status = self.pending = RenameStatus(
op=request.request_id,
kind=kind,
id=obj_id,
old=request.expected_name,
new=request.new_name,
phase="draining",
)
abort = self._abort = threading.Event()
task = asyncio.create_task(self._run(status, request, actor, abort))
# Read here, or by nobody once the caller has gone.
task.add_done_callback(lambda done: done.cancelled() or done.exception())
watcher = (
asyncio.create_task(self._watch(disconnected, abort))
if disconnected is not None
else None
)
try:
# Shielded: a caller going away calls the rename off while it
# still can be, and never stops it halfway after.
return await asyncio.shield(task)
except asyncio.CancelledError:
abort.set()
raise
finally:
if watcher is not None:
watcher.cancel()
def cancel(self, obj_id: str, op: str) -> None:
"""Call off the rename *op* of *obj_id* while it still waits for the
engine. It answers 409 ``aborted`` once everything is as it was.
:raises KeyError: When no such rename is under way.
:raises RenameConflict: 409 ``too_late`` once it changes things.
"""
status = self.pending
if status is None or (status.id, status.op) != (obj_id, op):
raise KeyError(op)
if status.phase != "draining":
raise RenameConflict(
409, "too_late", "The rename has begun changing things and finishes"
)
self._abort.set()
@staticmethod
async def _watch(
disconnected: Callable[[], Awaitable[bool]], abort: threading.Event
) -> None:
while not abort.is_set():
if await disconnected():
abort.set()
return
await asyncio.sleep(HOLD_POLL_S)
def _report(self, busy: list[str]) -> None:
"""What the rename waits for, each time that changes. From a thread."""
status = self.pending
if status is None:
return
self.events.publish(
{
"type": "rename_waiting",
"op": status.op,
"kind": status.kind,
"id": status.id,
"busy": busy,
"ts": time.time(),
}
)
status.busy = busy
async def _run(
self,
status: RenameStatus,
request: RenameRequest,
actor: str,
abort: threading.Event,
) -> RenameResult:
try:
if status.kind == "dashboard":
return await self._rename_dashboard(status, request, actor)
checked = await asyncio.to_thread(self._checked, "flow", status.id, request)
if isinstance(checked, RenameResult):
return checked
return await self._rename_flow(status, checked, request, actor, abort)
finally:
# Kept while its journal is not final: it is being finished, or
# a restart has to.
if self.pending is status and status.phase in ("draining", "rolled_back"):
self.pending = None
def _checked(
self, kind: Kind, obj_id: str, request: RenameRequest
) -> RenamePlan | RenameResult:
"""The result a request asked again got, or the plan it goes ahead
on, once that is what the request was planned against. Blocking.
:raises RenameConflict: 409 ``stale`` or ``plan_changed``, or the
plan's first refusal: 422 for a key that is no key, else 409.
"""
current: FlowDef | DashboardDef = (
self.controller.store.read_flow(obj_id, draft=True)
if kind == "flow"
else self.dashboards.read(obj_id, draft=True)
)
stored = repeated(
request.request_id,
current.name,
current.version,
request.expected_name,
request.expected_version,
)
if stored is not None:
return stored
plan = plan_rename(
self.controller,
self.dashboards,
kind,
obj_id,
request.new_name,
self.alerts.config if self.alerts is not None else AlertsConfig(),
)
if plan.refusals:
first = plan.refusals[0]
raise RenameConflict(
422 if first.code in _MALFORMED else 409,
first.code,
first.detail,
plan=plan.model_dump(mode="json"),
)
if plan.fingerprint != request.fingerprint:
raise RenameConflict(
409,
"plan_changed",
"What the rename changes has moved since it was planned",
plan=plan.model_dump(mode="json"),
)
return plan
async def _rename_flow(
self,
status: RenameStatus,
plan: RenamePlan,
request: RenameRequest,
actor: str,
abort: threading.Event,
) -> RenameResult:
controller, op = self.controller, request.request_id
rename = Rename(plan.name, plan.new_name)
held = _Held()
try:
await self._hold(held, op, plan, abort)
# Again, now that nothing can move underneath it.
plan = cast(
RenamePlan,
await asyncio.to_thread(self._checked, "flow", plan.id, request),
)
if abort.is_set():
raise _aborted()
# Built ones only: a flow that is only a draft runs nothing.
rebuilt = [
doc.id
for doc in plan.documents
if doc.kind == "flow" and doc.id in controller._flow_keys
]
for flow_id in rebuilt:
key = controller._flow_keys[flow_id]
held.torn_down.append(key)
await controller._teardown(key)
if controller.execution is not None:
await asyncio.to_thread(controller.execution.queue.hold)
held.queue = controller.execution.queue
# Whatever the old names published is recorded under them.
await self.events.settle()
if self.metrics is not None:
await self.metrics.flush_now()
rewrites = await asyncio.to_thread(
rewrite_documents,
controller.store,
self.dashboards,
"flow",
plan.id,
rename,
)
if abort.is_set():
raise _aborted()
# Synchronously after the last look at abort: `cancel` is refused
# from here, and the rename is finished rather than called off.
status.phase = "prepared"
try:
journal = await asyncio.to_thread(
lambda: begin(
op,
"flow",
plan.id,
rename,
rewrites,
actor=actor,
fingerprint=plan.fingerprint,
durable=not isinstance(controller.state, MemoryState),
rebuilt=rebuilt,
)
)
except BaseException:
status.phase = "draining"
raise
except BaseException:
await self._release(held)
raise
try:
await asyncio.to_thread(apply_documents, journal, controller.store)
except Exception as exc:
logger.exception("Could not write the renamed documents; undoing them")
try:
await asyncio.to_thread(roll_back, journal, controller.store)
status.phase = "rolled_back"
finally:
await self._release(held)
raise RenameConflict(
500, "failed", f"The documents could not be written: {exc}", op=op
) from exc
status.phase = "forward"
# Never started again as they were: the splice builds them anew.
held.torn_down = []
try:
await asyncio.to_thread(self._finish_steps, journal, status)
except Exception as exc:
self.events.publish(
{
"type": "engine_degraded",
"detail": f"A rename could not finish: {exc}",
}
)
self._resume = asyncio.create_task(
self._finish_later(journal, held, status)
)
raise RenameConflict(
503,
"completing",
f"The rename could not finish yet and is being retried: {exc}",
op=op,
) from None
return await self._complete(journal, held, status)
async def _hold(
self, held: _Held, op: str, plan: RenamePlan, abort: threading.Event
) -> None:
"""The gate, the controller lock, the pipeline and the execution
service, each once what runs in front of it has finished. What it got
is in *held*.
:raises RenameConflict: 409 ``aborted`` once called off, 503 ``busy``
when a rebuild keeps the controller for longer than one may.
"""
controller = self.controller
affected = [doc.id for doc in plan.documents if doc.kind == "flow"]
held.gate = await asyncio.to_thread(
self.gate.close, op, affected, abort, self._report
)
if not held.gate:
raise _aborted()
try:
await controller._acquire(REBUILD_WAIT)
except RebuildBusy as exc:
raise RenameConflict(503, "busy", str(exc), op=op) from None
held.lock = True
pipeline, execution = controller.pipeline, controller.execution
if pipeline is not None:
if not await asyncio.to_thread(pipeline.hold, abort, self._report):
raise _aborted()
held.pipeline = pipeline
if execution is not None:
if not await asyncio.to_thread(execution.hold, abort, self._report):
raise _aborted()
held.execution = execution
if self.pending is not None:
self.pending.busy = []
def _finish_steps(self, journal: Journal, status: RenameStatus) -> None:
"""Every step after the documents, the audit and the journal's end.
Blocking, and tried again until it succeeds."""
execution = self.controller.execution
try:
run_steps(
journal,
self.controller.state,
execution.queue if execution is not None else None,
)
finish(journal)
except Exception as exc:
status.error = journal.error or f"{type(exc).__name__}: {exc}"
raise
status.error = ""
async def _finish_later(
self, journal: Journal, held: _Held, status: RenameStatus
) -> None:
"""Decision 5: held, and tried again however long it takes."""
await retry_until_done(lambda: self._finish_steps(journal, status), self.events)
await self._complete(journal, held, status)
async def _complete(
self, journal: Journal, held: _Held, status: RenameStatus
) -> RenameResult:
"""The engine under the new names, then everything let go of and
everyone told. Its journal is done already: what is left here lives
in memory, which a restart builds afresh anyway."""
rename = journal.rename
if held.execution is not None:
held.execution.rename_reclaimed(rename)
self.events.rename(rename)
if self.alerts is not None:
self.alerts.rename(rename)
try:
await self.controller.splice_renamed(rename, journal.affected.rebuilt)
except Exception as exc:
# What a failed `reload_flow` leaves: those flows stopped, until
# they are published again.
logger.exception("Could not rebuild the renamed flows")
self.events.publish(
{
"type": "engine_degraded",
"detail": f"The renamed flows could not be rebuilt: {exc}",
}
)
await self._release(held)
if self.pending is status:
self.pending = None
self._announce(journal)
return cast(RenameResult, journal.result)
async def _rename_dashboard(
self, status: RenameStatus, request: RenameRequest, actor: str
) -> RenameResult:
done = await asyncio.to_thread(
self._dashboard_documents, status, request, actor
)
if isinstance(done, RenameResult):
return done
await retry_until_done(lambda: self._finish_steps(done, status), self.events)
if self.pending is status:
self.pending = None
self._announce(done)
return cast(RenameResult, done.result)
def _dashboard_documents(
self, status: RenameStatus, request: RenameRequest, actor: str
) -> Journal | RenameResult:
"""A dashboard's documents, renamed and committed. Blocking.
Under the dashboard store's lock, which every dashboard save takes
before the flow store's, so none lands between the recheck and the
write. Nothing else names a dashboard's key, so nothing is held.
"""
store = self.controller.store
with self.dashboards._lock:
checked = self._checked("dashboard", status.id, request)
if isinstance(checked, RenameResult):
return checked
rename = Rename(checked.name, checked.new_name)
rewrites = rewrite_documents(
store, self.dashboards, "dashboard", status.id, rename
)
journal = begin(
request.request_id,
"dashboard",
status.id,
rename,
rewrites,
actor=actor,
fingerprint=checked.fingerprint,
)
status.phase = "prepared"
try:
apply_documents(journal, store)
except Exception as exc:
roll_back(journal, store)
status.phase = "rolled_back"
raise RenameConflict(
500,
"failed",
f"The documents could not be written: {exc}",
op=request.request_id,
) from exc
status.phase = "forward"
return journal
async def _release(self, held: _Held) -> None:
"""Let go of what *held* says, last taken first. Flows taken down for
a rename that does not go ahead start again as they were."""
if held.queue is not None:
try:
await asyncio.to_thread(held.queue.release)
except Exception:
# Left in the held list, which the next start releases.
logger.exception("Could not release the work held for a rename")
for key in held.torn_down:
await self.controller._activate(key)
if held.execution is not None:
held.execution.release()
if held.pipeline is not None:
held.pipeline.release()
if held.lock:
self.controller._lock.release()
if held.gate:
self.gate.open()
def _announce(self, journal: Journal) -> None:
"""Tell the clients, once everything is let go of: editors and panels
refetch, and the canvas takes the pipeline as it is now."""
now = time.time()
self.events.publish(
{
"type": "renamed",
"kind": journal.kind,
"id": journal.id,
"old": journal.old,
"new": journal.new,
"op": journal.op,
"ts": now,
}
)
for dash_id in journal.affected.dashboards:
self.events.publish(
{"type": "dashboard_changed", "dashboard": dash_id, "ts": now}
)
if journal.kind != "flow":
return
# What the devices' ports are wired to moved with the flow.
self.events.publish({"type": "dashboard_changed", "dashboard": "", "ts": now})
controller = self.controller
self.events.publish(
{
"type": "pipeline_rebuilt",
"issues": [issue.model_dump() for issue in controller.issues],
"nodes": [status.model_dump() for status in controller.node_statuses()],
"paused": controller.paused_flows(),
}
)
+22
View File
@@ -50,7 +50,9 @@ from fluksio.flow.provision import load_provisioners
from fluksio.flow.queue import MemoryWorkQueue, RedisWorkQueue, WorkQueue
from fluksio.flow.remote import RemoteWorkerHub
from fluksio.flow.rename import (
RenameConflict,
RenameGate,
Renamer,
recover_documents,
recover_runtime,
retry_until_done,
@@ -464,6 +466,14 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
# On the app so a superuser can ask it to flush now.
app.state.metrics_collector = MetricsCollector(event_bus)
_background(app.state.metrics_collector.run(), "metrics-collector")
app.state.renamer = Renamer(
controller,
dashboards,
rename_gate,
event_bus,
alerts=alerts,
metrics=app.state.metrics_collector,
)
_background(_sweep_artifacts(artifacts, controller), "artifact-gc")
if volatile is not None:
_background(_trim_volatile(volatile), "artifact-ring")
@@ -688,6 +698,18 @@ async def engine_held(request: Request, exc: EngineHeld) -> JSONResponse: # noq
)
@app.exception_handler(RenameConflict)
async def rename_conflict(request: Request, exc: RenameConflict) -> JSONResponse: # noqa: ARG001
"""A rename answered with a code a client acts on, as the gate's 503 is:
a refusal or a stale request (409), a key that is no key (422), or one
finishing (503, which is worth retrying)."""
return JSONResponse(
status_code=exc.status,
content={"detail": exc.detail, "code": exc.code, **exc.extra},
headers={"Retry-After": "5"} if exc.status == 503 else None,
)
# Tagged because the operation-id builder reads the first tag; the route
# itself stays out of the schema.
@app.get(