Rename orchestration, splice and routes: a Renamer runs one rename at a time, closing the gate, taking the controller lock and holding the engine, each waiting without a limit and saying what it waits for (summary rename status, rename_waiting); DELETE on the pending request or a closed connection calls it off until the journal begins (409 aborted); then journal, steps and audit, retried in the background on failure (503 completing); splice_renamed rebuilds the affected flows under the held lock keeping stopped, paused and parked work, history limits and device wiring refreshed before activation; AlertManager.rename and EventBus.rename/settle follow the names; flow and dashboard rename routes answer by code and repeat stored results; renamed, dashboard_changed and pipeline_rebuilt after release; fluksio serve reads flow files without building a FlowStore
This commit is contained in:
@@ -21,6 +21,7 @@ from fluksio.flow.controller import FlowController
|
||||
from fluksio.flow.dashboards import DashboardDef, DashboardStore
|
||||
from fluksio.flow.devices import DeviceDef, DeviceStore, DeviceUnavailable, presence
|
||||
from fluksio.flow.events import event_bus
|
||||
from fluksio.flow.rename import Renamer
|
||||
from fluksio.flow.workers import PythonWorkerPool
|
||||
from fluksio.models import OAuthClient, TokenPayload, User, WorkerToken
|
||||
|
||||
@@ -583,6 +584,16 @@ def get_device_store(request: Request) -> DeviceStore:
|
||||
DeviceStoreDep = Annotated[DeviceStore, Depends(get_device_store)]
|
||||
|
||||
|
||||
def get_renamer(request: Request) -> Renamer:
|
||||
renamer: Renamer | None = getattr(request.app.state, "renamer", None)
|
||||
if renamer is None:
|
||||
raise HTTPException(status_code=503, detail="The flow engine is not running")
|
||||
return renamer
|
||||
|
||||
|
||||
RenamerDep = Annotated[Renamer, Depends(get_renamer)]
|
||||
|
||||
|
||||
def get_worker_pool(request: Request) -> PythonWorkerPool:
|
||||
pool: PythonWorkerPool | None = getattr(request.app.state, "worker_pool", None)
|
||||
if pool is None:
|
||||
|
||||
@@ -7,7 +7,7 @@ label unique among dashboards and what ``GET /dashboards?name=`` looks up.
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from pydantic import BaseModel
|
||||
|
||||
@@ -15,6 +15,7 @@ from fluksio.api.deps import (
|
||||
CurrentUser,
|
||||
DashboardStoreDep,
|
||||
FlowControllerDep,
|
||||
RenamerDep,
|
||||
check_pin,
|
||||
get_current_active_superuser,
|
||||
get_current_user,
|
||||
@@ -30,7 +31,13 @@ from fluksio.flow.dashboards import (
|
||||
results_dashboard,
|
||||
)
|
||||
from fluksio.flow.events import event_bus
|
||||
from fluksio.flow.rename import RenamePlan, RenamePlanRequest, plan_rename
|
||||
from fluksio.flow.rename import (
|
||||
RenamePlan,
|
||||
RenamePlanRequest,
|
||||
RenameRequest,
|
||||
RenameResult,
|
||||
plan_rename,
|
||||
)
|
||||
from fluksio.flow.store import FlowNotFound, IdTaken, StaleVersion
|
||||
from fluksio.models import Message
|
||||
|
||||
@@ -279,3 +286,29 @@ async def plan_dashboard_rename(
|
||||
)
|
||||
except DashboardNotFound:
|
||||
raise _missing(dashboard_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{dashboard_id}/rename",
|
||||
response_model=RenameResult,
|
||||
dependencies=[Depends(get_current_active_superuser)],
|
||||
)
|
||||
async def rename_dashboard(
|
||||
dashboard_id: str,
|
||||
body: RenameRequest,
|
||||
request: Request,
|
||||
renamer: RenamerDep,
|
||||
user: CurrentUser,
|
||||
) -> Any:
|
||||
"""Rename this dashboard's key, as its plan described it.
|
||||
|
||||
Its two documents and nothing else: devices list it by id, and nothing
|
||||
binds to it by key. The same ``request_id`` again gets the same answer;
|
||||
refusals are as for a flow.
|
||||
"""
|
||||
try:
|
||||
return await renamer.rename(
|
||||
"dashboard", dashboard_id, body, user.email, request.is_disconnected
|
||||
)
|
||||
except DashboardNotFound:
|
||||
raise _missing(dashboard_id)
|
||||
|
||||
@@ -35,6 +35,7 @@ from fluksio.api.deps import (
|
||||
DashboardStoreDep,
|
||||
FlowControllerDep,
|
||||
HiddenDep,
|
||||
RenamerDep,
|
||||
SessionDep,
|
||||
TokenDep,
|
||||
check_pin,
|
||||
@@ -69,7 +70,13 @@ from fluksio.flow.events import event_bus
|
||||
from fluksio.flow.messages import MEDIA_FAMILIES, DType, flow_of, qualify
|
||||
from fluksio.flow.nodes.wren import TRAIN_FLOW, WrenNode, pack_meta
|
||||
from fluksio.flow.pipeline import EngineHeld, ValidationIssue
|
||||
from fluksio.flow.rename import RenamePlan, RenamePlanRequest, plan_rename
|
||||
from fluksio.flow.rename import (
|
||||
RenamePlan,
|
||||
RenamePlanRequest,
|
||||
RenameRequest,
|
||||
RenameResult,
|
||||
plan_rename,
|
||||
)
|
||||
from fluksio.flow.runs import RunRejected
|
||||
from fluksio.flow.schemas import (
|
||||
NAME_PATTERN,
|
||||
@@ -776,6 +783,60 @@ async def plan_flow_rename(
|
||||
raise _missing(flow_id)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{flow_id}/rename",
|
||||
response_model=RenameResult,
|
||||
dependencies=[Depends(get_current_active_superuser)],
|
||||
)
|
||||
async def rename_flow(
|
||||
flow_id: str,
|
||||
body: RenameRequest,
|
||||
request: Request,
|
||||
renamer: RenamerDep,
|
||||
user: CurrentUser,
|
||||
) -> Any:
|
||||
"""Rename this flow's key, as its plan described it.
|
||||
|
||||
Everything named under the key moves: every flow and dashboard binding
|
||||
one of its messages, its history, live state and queued work. The engine
|
||||
waits meanwhile, for as long as the work running when it was asked takes
|
||||
to finish. Until then it can be called off, with ``DELETE`` on the
|
||||
request or by closing the connection, and nothing changes (409
|
||||
``aborted``); ``GET /observability/summary`` says what it waits for.
|
||||
|
||||
The same ``request_id`` again gets the same answer. A rename that
|
||||
could not finish yet answers 503 ``completing`` and is finished in the
|
||||
background. Refusals are the plan's codes (409, 422 for a key that is no
|
||||
key), ``stale`` and ``plan_changed`` (409).
|
||||
"""
|
||||
try:
|
||||
return await renamer.rename(
|
||||
"flow", flow_id, body, user.email, request.is_disconnected
|
||||
)
|
||||
except FlowNotFound:
|
||||
raise _missing(flow_id)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/{flow_id}/rename/{request_id}",
|
||||
response_model=Message,
|
||||
dependencies=[Depends(get_current_active_superuser)],
|
||||
)
|
||||
async def cancel_flow_rename(flow_id: str, request_id: str, renamer: RenamerDep) -> Any:
|
||||
"""Call off a rename still waiting for the engine. It answers 409
|
||||
``aborted`` once everything is as it was, and one that has begun to
|
||||
change things refuses this with 409 ``too_late``."""
|
||||
# Async, so on the event loop, where the rename decides when it is past
|
||||
# calling off.
|
||||
try:
|
||||
renamer.cancel(flow_id, request_id)
|
||||
except KeyError:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"No rename '{request_id}' of '{flow_id}' waits"
|
||||
)
|
||||
return Message(message=f"Calling off rename '{request_id}'")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Node source
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
@@ -28,6 +28,7 @@ from fluksio.api.deps import (
|
||||
from fluksio.api.routes.runs import elapsed_ms
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.flow.controller import NodeStatus
|
||||
from fluksio.flow.rename import RenameStatus
|
||||
from fluksio.flow.tombstones import names_of, shown_nodes, stored_node
|
||||
from fluksio.models import EngineEvent, FlowRun, Message, MetricBucket
|
||||
|
||||
@@ -65,6 +66,9 @@ class HealthSummary(BaseModel):
|
||||
#: route the client knows and the engine does not answers a flat 404. This
|
||||
#: is what turns that into a sentence. Absent means older than this field.
|
||||
version: str = ""
|
||||
#: A key rename under way: what it waits for while it can still be
|
||||
#: called off, and why a step failed while it is retried.
|
||||
rename: RenameStatus | None = None
|
||||
|
||||
|
||||
class SeriesPoint(BaseModel):
|
||||
@@ -285,6 +289,7 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
|
||||
f"{', '.join(sorted(e.id for e in failed_once))}"
|
||||
)
|
||||
|
||||
renamer = getattr(request.app.state, "renamer", None)
|
||||
return HealthSummary(
|
||||
status="degraded" if problems else "ok",
|
||||
problems=problems,
|
||||
@@ -322,6 +327,7 @@ async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
|
||||
else {"ewma": 0.0, "max_60s": 0.0}
|
||||
),
|
||||
version=__version__,
|
||||
rename=renamer.pending if renamer is not None else None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
+11
-5
@@ -99,18 +99,24 @@ def _mention_undeclared_cards() -> None:
|
||||
if settings.FLOW_GPUS or machine_gpus():
|
||||
return
|
||||
from fluksio.flow.runs import required_resources
|
||||
from fluksio.flow.store import FlowStore
|
||||
from fluksio.flow.schemas import FlowDef
|
||||
|
||||
store = FlowStore(settings.FLOWS_DIR)
|
||||
# Read as files, not through a `FlowStore`: building one can commit the
|
||||
# store, and this runs before the engine has settled an interrupted
|
||||
# rename and given every document its id.
|
||||
asking = []
|
||||
for name in store.list_flows():
|
||||
for path in sorted(settings.FLOWS_DIR.glob("*/flow.json")):
|
||||
# `_devices/flow.json` is a device, not a flow.
|
||||
if path.parent.name.startswith("_"):
|
||||
continue
|
||||
try:
|
||||
needs = required_resources(store.read_flow(name))
|
||||
flow = FlowDef.model_validate_json(path.read_text())
|
||||
needs = required_resources(flow)
|
||||
except Exception:
|
||||
# A flow that will not parse is the engine's to complain about.
|
||||
continue
|
||||
if (needs or {}).get("gpus"):
|
||||
asking.append(name)
|
||||
asking.append(flow.name)
|
||||
if not asking:
|
||||
return
|
||||
named = ", ".join(sorted(asking)[:3]) + (" …" if len(asking) > 3 else "")
|
||||
|
||||
@@ -17,7 +17,7 @@ import logging
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Literal, Protocol
|
||||
from typing import TYPE_CHECKING, Any, Literal, Protocol
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, Field
|
||||
@@ -25,6 +25,9 @@ from pydantic import BaseModel, Field
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.secrets import resolve_params
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fluksio.flow.namespace import Rename
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# What is worth waking someone for. Everything else on the bus is traffic.
|
||||
@@ -271,6 +274,25 @@ class AlertManager:
|
||||
return
|
||||
await self._deliver(event)
|
||||
|
||||
def rename(self, rename: Rename) -> None:
|
||||
"""Follow a key rename: dashboard channels publish to the moved
|
||||
message, as `alerts.json` now says, and what was sent, held back or
|
||||
muted is kept under the new names with its timestamps, so nothing is
|
||||
sent again. A connection waiting out its grace under an old name is
|
||||
let go: its node reports again once it runs under the new one."""
|
||||
config = self.config.model_copy(deep=True)
|
||||
for channel in config.channels:
|
||||
message = channel.config.get("message")
|
||||
if channel.kind == "dashboard" and isinstance(message, str):
|
||||
channel.config["message"] = rename.binding(message)
|
||||
self.config = config
|
||||
self._last_sent = {rename.dedup_key(k): v for k, v in self._last_sent.items()}
|
||||
self._suppressed = {rename.dedup_key(k): v for k, v in self._suppressed.items()}
|
||||
self._health_flips = {rename.name(k): v for k, v in self._health_flips.items()}
|
||||
self._flapping = {rename.name(k): v for k, v in self._flapping.items()}
|
||||
for node in [node for node in self._down if rename.name(node) != node]:
|
||||
self._cancel_down(node)
|
||||
|
||||
def _cancel_down(self, node: str) -> None:
|
||||
task = self._down.pop(node, None)
|
||||
if task is not None:
|
||||
|
||||
@@ -39,7 +39,7 @@ from fluksio.flow.demand import demand
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.executor import ExecutionService
|
||||
from fluksio.flow.messages import MessageSpec, flow_of, qualify
|
||||
from fluksio.flow.namespace import touches
|
||||
from fluksio.flow.namespace import Rename, touches
|
||||
from fluksio.flow.nodes import (
|
||||
RESERVED_SETTINGS,
|
||||
AssistantNode,
|
||||
@@ -913,6 +913,75 @@ class FlowController:
|
||||
)
|
||||
self._rewired()
|
||||
|
||||
async def splice_renamed(self, rename: Rename, rebuilt: list[str]) -> None:
|
||||
"""Put the flows a key rename rewrote back into the graph, built from
|
||||
their new documents and named as they are now.
|
||||
|
||||
Under the controller lock, which the rename holds, with the engine
|
||||
held and the flows *rebuilt* (ids) torn down before their documents
|
||||
changed. Not `reload_flow`, which would take the lock again, nor
|
||||
`reload`, which clears every pause. Stopped flows stay stopped, which
|
||||
is stored by id, pauses carry over under the new keys and parked work
|
||||
stays parked. A rebuilt flow's quarantine is gone with its teardown,
|
||||
as on a publish. What the documents feed is refreshed before anything
|
||||
starts: history limits, device wiring and what devices are asked to
|
||||
send.
|
||||
"""
|
||||
pipeline = self.pipeline
|
||||
if pipeline is None:
|
||||
return
|
||||
paused = pipeline.paused_flows()
|
||||
builds = await run_in_threadpool(
|
||||
lambda: [(flow_id, self._build_one(flow_id)) for flow_id in rebuilt]
|
||||
)
|
||||
# By the name each node has now, so a failure carries over.
|
||||
previous = {rename.name(node_id): e for node_id, e in self.loaded.items()}
|
||||
loaded = dict(self.loaded)
|
||||
keys = []
|
||||
for flow_id, built in builds:
|
||||
if built is None:
|
||||
continue
|
||||
flow, nodes, entries, initial_values, enabled = built
|
||||
old, key = self._flow_keys.get(flow_id, flow.name), flow.name
|
||||
if old != key:
|
||||
pipeline.replace_flow(old, [])
|
||||
pipeline.replace_flow(key, nodes, initial_values)
|
||||
self._flow_keys[flow_id] = key
|
||||
for node_id, entry in entries.items():
|
||||
before = previous.get(node_id)
|
||||
if before is not None and before.last_error:
|
||||
entry.inherit_failure(before)
|
||||
loaded = {
|
||||
node_id: entry for node_id, entry in loaded.items() if entry.flow != old
|
||||
} | entries
|
||||
self._flow_inputs.pop(old, None)
|
||||
self._flow_inputs[key] = _declared_inputs(flow)[0]
|
||||
self.disabled = (self.disabled - {old}) | (set() if enabled else {key})
|
||||
self.batch = (self.batch - {old}) | (
|
||||
{key} if flow.mode == "batch" else set()
|
||||
)
|
||||
keys.append(key)
|
||||
self.loaded = loaded
|
||||
pipeline.set_disabled(self.disabled)
|
||||
# The replace cleared the pause of every flow it touched.
|
||||
for was in map(rename.flow, paused):
|
||||
if not pipeline.is_paused(was):
|
||||
pipeline.pause(was)
|
||||
self.issues = _collect_issues(
|
||||
self.loaded, pipeline, self._all_flow_inputs(), self.devices
|
||||
)
|
||||
if self.dashboards is not None:
|
||||
self.set_history_limits(
|
||||
await run_in_threadpool(self.dashboards.history_requirements)
|
||||
)
|
||||
if self.devices is not None:
|
||||
# Taken note of without telling the devices: the rename does,
|
||||
# once it has let go of the engine.
|
||||
await run_in_threadpool(demand.refresh)
|
||||
self._device_wiring = await run_in_threadpool(self.devices.wiring)
|
||||
for key in keys:
|
||||
await self._activate(key)
|
||||
|
||||
async def recheck(self, wait: float | None = FLOW_REBUILD_WAIT) -> None:
|
||||
"""Validate the built graph again without rebuilding anything.
|
||||
|
||||
|
||||
@@ -15,7 +15,10 @@ import threading
|
||||
from collections import deque
|
||||
from collections.abc import AsyncIterator, Callable
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fluksio.flow.namespace import Rename
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -24,6 +27,8 @@ DISPATCH_BUDGET = 1024
|
||||
LOG_HISTORY = 400
|
||||
#: How many of a message's last artifact digests are remembered.
|
||||
RECENT_DIGESTS = 8
|
||||
#: How many turns of the loop `settle` gives the dispatch at most.
|
||||
SETTLE_TURNS = 100
|
||||
|
||||
|
||||
def _critical(event: dict[str, Any]) -> bool:
|
||||
@@ -211,6 +216,29 @@ class EventBus:
|
||||
if self._loop is not None:
|
||||
self._loop.call_soon(self._dispatch)
|
||||
|
||||
async def settle(self) -> None:
|
||||
"""Wait until what was published so far has reached every subscriber.
|
||||
|
||||
For a rename, before any key changes: an event a node published under
|
||||
the old key is then recorded under it. Bounded, since a busy bus may
|
||||
never be empty.
|
||||
"""
|
||||
for _ in range(SETTLE_TURNS):
|
||||
with self._pending_lock:
|
||||
if not self._flush_scheduled:
|
||||
return
|
||||
await asyncio.sleep(0)
|
||||
|
||||
def rename(self, rename: Rename) -> None:
|
||||
"""Keep emit counts and recent digests under the names a key rename
|
||||
gave their nodes and messages, so a client reconnecting or a device
|
||||
fetching a recent frame finds them there. Rebound, not changed in
|
||||
place: publishers on other threads write to them."""
|
||||
self.emits = {rename.name(node): n for node, n in list(self.emits.items())}
|
||||
self.digests = {
|
||||
rename.name(name): recent for name, recent in list(self.digests.items())
|
||||
}
|
||||
|
||||
def dropped(self, queue: asyncio.Queue[dict[str, Any]]) -> bool:
|
||||
"""Whether this subscriber lost events since it last asked.
|
||||
|
||||
|
||||
+583
-10
@@ -6,7 +6,9 @@ live state and queued work, so nothing may write through the API meanwhile.
|
||||
a rename would change and what refuses it, and `rewrite_documents` is its
|
||||
document part. The `Journal` records each step before the next, so a rename a
|
||||
crash interrupted is undone or finished on the next start (`recover_documents`,
|
||||
`recover_runtime`). ``docs/private/id_refac_rename.md`` has the whole protocol.
|
||||
`recover_runtime`). `Renamer` runs one: the gate, the engine holds, the
|
||||
journal and its steps, and the engine put back together under the new names.
|
||||
``docs/private/id_refac_rename.md`` has the whole protocol.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -20,8 +22,8 @@ import re
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Awaitable, Callable, Iterable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, cast
|
||||
@@ -34,8 +36,13 @@ from starlette.types import Scope
|
||||
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.core.db import engine as db_engine
|
||||
from fluksio.flow.alerts import AlertsConfig
|
||||
from fluksio.flow.controller import HOOK_PREFIX, FlowController
|
||||
from fluksio.flow.alerts import AlertManager, AlertsConfig
|
||||
from fluksio.flow.controller import (
|
||||
HOOK_PREFIX,
|
||||
REBUILD_WAIT,
|
||||
FlowController,
|
||||
RebuildBusy,
|
||||
)
|
||||
from fluksio.flow.dashboards import (
|
||||
DASHBOARD_BINDINGS,
|
||||
DASHBOARD_DIR,
|
||||
@@ -44,10 +51,12 @@ from fluksio.flow.dashboards import (
|
||||
results_name,
|
||||
)
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.executor import wait_out
|
||||
from fluksio.flow.executor import HOLD_POLL_S, ExecutionService, wait_out
|
||||
from fluksio.flow.identity import _replace, write_documents
|
||||
from fluksio.flow.messages import flow_of
|
||||
from fluksio.flow.metrics import MetricsCollector
|
||||
from fluksio.flow.namespace import FLOW_FIELDS, NAME, OUTPUTS, Rename, names, rebind
|
||||
from fluksio.flow.pipeline import Pipeline
|
||||
from fluksio.flow.queue import WorkQueue
|
||||
from fluksio.flow.schemas import FlowDef, _validate_name
|
||||
from fluksio.flow.state import MemoryState, StateBackend
|
||||
@@ -73,13 +82,13 @@ _READS = frozenset({"GET", "HEAD", "OPTIONS"})
|
||||
#: and are held there), a browser's push subscription, the portal, workers
|
||||
#: attaching, artifact bytes (which are named by digest, and which a node on a
|
||||
#: remote worker writes while the rename waits for it), the socket ticket and
|
||||
#: the rename routes, which keep their own order.
|
||||
#: the rename routes (plan, rename and its cancel), which keep their own order.
|
||||
_EXEMPT = re.compile(
|
||||
r"/(?:login|password-recovery|password-recovery-html-content|reset-password"
|
||||
r"|oauth|users|ai|cloud|workers|artifacts)(?:/|$)"
|
||||
r"|/alerts/webpush(?:/|$)"
|
||||
r"|/flows/ws-ticket$"
|
||||
r"|/(?:flows|dashboards)/[^/]+/rename(?:/plan)?$"
|
||||
r"|/(?:flows|dashboards)/[^/]+/rename(?:/[^/]+)?$"
|
||||
)
|
||||
|
||||
|
||||
@@ -353,6 +362,38 @@ class RenamePlanRequest(BaseModel):
|
||||
new_name: str
|
||||
|
||||
|
||||
class RenameRequest(BaseModel):
|
||||
"""A rename as its plan described it. Refused when the object or the
|
||||
plan moved since (409 ``stale`` or ``plan_changed``)."""
|
||||
|
||||
#: The client's own id for this request, which a retry repeats.
|
||||
request_id: str = Field(min_length=1, max_length=64)
|
||||
expected_name: str
|
||||
#: The working copy's version the plan was made against.
|
||||
expected_version: int
|
||||
new_name: str
|
||||
fingerprint: str
|
||||
|
||||
|
||||
class RenameStatus(BaseModel):
|
||||
"""A rename under way, for whoever is waiting on it or was turned away."""
|
||||
|
||||
op: str
|
||||
kind: Kind
|
||||
id: str
|
||||
old: str
|
||||
new: str
|
||||
#: ``draining`` while it waits for work to finish, when it can still be
|
||||
#: called off, then the journal's phase: ``prepared`` while the
|
||||
#: documents are written and ``forward`` while the rest moves.
|
||||
phase: str
|
||||
#: What it is waiting for: API requests (``POST /api/v1/...``), nodes
|
||||
#: and messages running outside the queue, and nodes whose cascades run.
|
||||
busy: list[str] = []
|
||||
#: Why the last attempt at a step failed, while it is retried.
|
||||
error: str = ""
|
||||
|
||||
|
||||
class RenameRefusal(BaseModel):
|
||||
"""Why a rename cannot go ahead, with the ids of what is in the way."""
|
||||
|
||||
@@ -856,13 +897,15 @@ def read_journal() -> Journal | None:
|
||||
|
||||
class RenameConflict(Exception):
|
||||
"""A rename request that cannot go ahead as asked, with the status and
|
||||
code it is answered with."""
|
||||
code it is answered with, and what else the answer carries: the ``op``
|
||||
in the way, or the fresh ``plan`` a refusal comes from."""
|
||||
|
||||
def __init__(self, status: int, code: str, detail: str) -> None:
|
||||
def __init__(self, status: int, code: str, detail: str, **extra: Any) -> None:
|
||||
super().__init__(detail)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.detail = detail
|
||||
self.extra = extra
|
||||
|
||||
|
||||
def repeated(
|
||||
@@ -1247,3 +1290,533 @@ async def retry_until_done(attempt: Callable[[], None], events: EventBus) -> Non
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
wait = min(wait * 2, longest)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Running one
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
#: Refusals of a key that is no key at all, answered 422 rather than 409.
|
||||
_MALFORMED = frozenset({"invalid_name", "reserved_shape"})
|
||||
|
||||
|
||||
def _aborted() -> RenameConflict:
|
||||
return RenameConflict(409, "aborted", "The rename was called off; nothing changed")
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Held:
|
||||
"""What a rename holds so far, so whatever stops it lets go of exactly
|
||||
that, last taken first."""
|
||||
|
||||
gate: bool = False
|
||||
lock: bool = False
|
||||
pipeline: Pipeline | None = None
|
||||
execution: ExecutionService | None = None
|
||||
queue: WorkQueue | None = None
|
||||
#: Keys of the flows taken down, which start again as they were if the
|
||||
#: rename does not go ahead.
|
||||
torn_down: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class Renamer:
|
||||
"""Runs renames, one at a time, in the order section 2.3 of the protocol
|
||||
gives.
|
||||
|
||||
A flow's rename closes the gate, takes the controller lock and holds the
|
||||
engine. Each of those waits for the work in front of it without a limit,
|
||||
says what it waits for (`pending`, and a ``rename_waiting`` event) and
|
||||
can be called off meanwhile, by `cancel` or by the caller going away.
|
||||
From the journal's first write on it only goes forward: a step that fails
|
||||
keeps everything held and is tried again in the background. A dashboard's
|
||||
rename is its documents and the audit, with nothing held.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
controller: FlowController,
|
||||
dashboards: DashboardStore,
|
||||
gate: RenameGate,
|
||||
events: EventBus,
|
||||
alerts: AlertManager | None = None,
|
||||
metrics: MetricsCollector | None = None,
|
||||
) -> None:
|
||||
self.controller = controller
|
||||
self.dashboards = dashboards
|
||||
self.gate = gate
|
||||
self.events = events
|
||||
self.alerts = alerts
|
||||
self.metrics = metrics
|
||||
#: The rename under way, None when there is none.
|
||||
self.pending: RenameStatus | None = None
|
||||
self._abort = threading.Event()
|
||||
#: A rename being finished in the background once a step failed.
|
||||
self._resume: asyncio.Task[None] | None = None
|
||||
|
||||
async def rename(
|
||||
self,
|
||||
kind: Kind,
|
||||
obj_id: str,
|
||||
request: RenameRequest,
|
||||
actor: str = "",
|
||||
disconnected: Callable[[], Awaitable[bool]] | None = None,
|
||||
) -> RenameResult:
|
||||
"""Rename the flow or dashboard *obj_id* as *request* asks, or answer
|
||||
it with the result it got before.
|
||||
|
||||
*disconnected* says when the caller has gone, which calls the rename
|
||||
off as `cancel` does, for as long as that is still possible.
|
||||
|
||||
:raises RenameConflict: For every answer but the result.
|
||||
:raises FlowNotFound: For a flow that does not exist.
|
||||
:raises DashboardNotFound: For a dashboard that does not exist.
|
||||
"""
|
||||
if self.pending is not None:
|
||||
mine = self.pending.op == request.request_id
|
||||
raise RenameConflict(
|
||||
503,
|
||||
"completing" if mine else "rename_in_progress",
|
||||
f"The rename of '{self.pending.old}' to '{self.pending.new}' "
|
||||
"is under way",
|
||||
op=self.pending.op,
|
||||
)
|
||||
status = self.pending = RenameStatus(
|
||||
op=request.request_id,
|
||||
kind=kind,
|
||||
id=obj_id,
|
||||
old=request.expected_name,
|
||||
new=request.new_name,
|
||||
phase="draining",
|
||||
)
|
||||
abort = self._abort = threading.Event()
|
||||
task = asyncio.create_task(self._run(status, request, actor, abort))
|
||||
# Read here, or by nobody once the caller has gone.
|
||||
task.add_done_callback(lambda done: done.cancelled() or done.exception())
|
||||
watcher = (
|
||||
asyncio.create_task(self._watch(disconnected, abort))
|
||||
if disconnected is not None
|
||||
else None
|
||||
)
|
||||
try:
|
||||
# Shielded: a caller going away calls the rename off while it
|
||||
# still can be, and never stops it halfway after.
|
||||
return await asyncio.shield(task)
|
||||
except asyncio.CancelledError:
|
||||
abort.set()
|
||||
raise
|
||||
finally:
|
||||
if watcher is not None:
|
||||
watcher.cancel()
|
||||
|
||||
def cancel(self, obj_id: str, op: str) -> None:
|
||||
"""Call off the rename *op* of *obj_id* while it still waits for the
|
||||
engine. It answers 409 ``aborted`` once everything is as it was.
|
||||
|
||||
:raises KeyError: When no such rename is under way.
|
||||
:raises RenameConflict: 409 ``too_late`` once it changes things.
|
||||
"""
|
||||
status = self.pending
|
||||
if status is None or (status.id, status.op) != (obj_id, op):
|
||||
raise KeyError(op)
|
||||
if status.phase != "draining":
|
||||
raise RenameConflict(
|
||||
409, "too_late", "The rename has begun changing things and finishes"
|
||||
)
|
||||
self._abort.set()
|
||||
|
||||
@staticmethod
|
||||
async def _watch(
|
||||
disconnected: Callable[[], Awaitable[bool]], abort: threading.Event
|
||||
) -> None:
|
||||
while not abort.is_set():
|
||||
if await disconnected():
|
||||
abort.set()
|
||||
return
|
||||
await asyncio.sleep(HOLD_POLL_S)
|
||||
|
||||
def _report(self, busy: list[str]) -> None:
|
||||
"""What the rename waits for, each time that changes. From a thread."""
|
||||
status = self.pending
|
||||
if status is None:
|
||||
return
|
||||
self.events.publish(
|
||||
{
|
||||
"type": "rename_waiting",
|
||||
"op": status.op,
|
||||
"kind": status.kind,
|
||||
"id": status.id,
|
||||
"busy": busy,
|
||||
"ts": time.time(),
|
||||
}
|
||||
)
|
||||
status.busy = busy
|
||||
|
||||
async def _run(
|
||||
self,
|
||||
status: RenameStatus,
|
||||
request: RenameRequest,
|
||||
actor: str,
|
||||
abort: threading.Event,
|
||||
) -> RenameResult:
|
||||
try:
|
||||
if status.kind == "dashboard":
|
||||
return await self._rename_dashboard(status, request, actor)
|
||||
checked = await asyncio.to_thread(self._checked, "flow", status.id, request)
|
||||
if isinstance(checked, RenameResult):
|
||||
return checked
|
||||
return await self._rename_flow(status, checked, request, actor, abort)
|
||||
finally:
|
||||
# Kept while its journal is not final: it is being finished, or
|
||||
# a restart has to.
|
||||
if self.pending is status and status.phase in ("draining", "rolled_back"):
|
||||
self.pending = None
|
||||
|
||||
def _checked(
|
||||
self, kind: Kind, obj_id: str, request: RenameRequest
|
||||
) -> RenamePlan | RenameResult:
|
||||
"""The result a request asked again got, or the plan it goes ahead
|
||||
on, once that is what the request was planned against. Blocking.
|
||||
|
||||
:raises RenameConflict: 409 ``stale`` or ``plan_changed``, or the
|
||||
plan's first refusal: 422 for a key that is no key, else 409.
|
||||
"""
|
||||
current: FlowDef | DashboardDef = (
|
||||
self.controller.store.read_flow(obj_id, draft=True)
|
||||
if kind == "flow"
|
||||
else self.dashboards.read(obj_id, draft=True)
|
||||
)
|
||||
stored = repeated(
|
||||
request.request_id,
|
||||
current.name,
|
||||
current.version,
|
||||
request.expected_name,
|
||||
request.expected_version,
|
||||
)
|
||||
if stored is not None:
|
||||
return stored
|
||||
plan = plan_rename(
|
||||
self.controller,
|
||||
self.dashboards,
|
||||
kind,
|
||||
obj_id,
|
||||
request.new_name,
|
||||
self.alerts.config if self.alerts is not None else AlertsConfig(),
|
||||
)
|
||||
if plan.refusals:
|
||||
first = plan.refusals[0]
|
||||
raise RenameConflict(
|
||||
422 if first.code in _MALFORMED else 409,
|
||||
first.code,
|
||||
first.detail,
|
||||
plan=plan.model_dump(mode="json"),
|
||||
)
|
||||
if plan.fingerprint != request.fingerprint:
|
||||
raise RenameConflict(
|
||||
409,
|
||||
"plan_changed",
|
||||
"What the rename changes has moved since it was planned",
|
||||
plan=plan.model_dump(mode="json"),
|
||||
)
|
||||
return plan
|
||||
|
||||
async def _rename_flow(
|
||||
self,
|
||||
status: RenameStatus,
|
||||
plan: RenamePlan,
|
||||
request: RenameRequest,
|
||||
actor: str,
|
||||
abort: threading.Event,
|
||||
) -> RenameResult:
|
||||
controller, op = self.controller, request.request_id
|
||||
rename = Rename(plan.name, plan.new_name)
|
||||
held = _Held()
|
||||
try:
|
||||
await self._hold(held, op, plan, abort)
|
||||
# Again, now that nothing can move underneath it.
|
||||
plan = cast(
|
||||
RenamePlan,
|
||||
await asyncio.to_thread(self._checked, "flow", plan.id, request),
|
||||
)
|
||||
if abort.is_set():
|
||||
raise _aborted()
|
||||
# Built ones only: a flow that is only a draft runs nothing.
|
||||
rebuilt = [
|
||||
doc.id
|
||||
for doc in plan.documents
|
||||
if doc.kind == "flow" and doc.id in controller._flow_keys
|
||||
]
|
||||
for flow_id in rebuilt:
|
||||
key = controller._flow_keys[flow_id]
|
||||
held.torn_down.append(key)
|
||||
await controller._teardown(key)
|
||||
if controller.execution is not None:
|
||||
await asyncio.to_thread(controller.execution.queue.hold)
|
||||
held.queue = controller.execution.queue
|
||||
# Whatever the old names published is recorded under them.
|
||||
await self.events.settle()
|
||||
if self.metrics is not None:
|
||||
await self.metrics.flush_now()
|
||||
rewrites = await asyncio.to_thread(
|
||||
rewrite_documents,
|
||||
controller.store,
|
||||
self.dashboards,
|
||||
"flow",
|
||||
plan.id,
|
||||
rename,
|
||||
)
|
||||
if abort.is_set():
|
||||
raise _aborted()
|
||||
# Synchronously after the last look at abort: `cancel` is refused
|
||||
# from here, and the rename is finished rather than called off.
|
||||
status.phase = "prepared"
|
||||
try:
|
||||
journal = await asyncio.to_thread(
|
||||
lambda: begin(
|
||||
op,
|
||||
"flow",
|
||||
plan.id,
|
||||
rename,
|
||||
rewrites,
|
||||
actor=actor,
|
||||
fingerprint=plan.fingerprint,
|
||||
durable=not isinstance(controller.state, MemoryState),
|
||||
rebuilt=rebuilt,
|
||||
)
|
||||
)
|
||||
except BaseException:
|
||||
status.phase = "draining"
|
||||
raise
|
||||
except BaseException:
|
||||
await self._release(held)
|
||||
raise
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(apply_documents, journal, controller.store)
|
||||
except Exception as exc:
|
||||
logger.exception("Could not write the renamed documents; undoing them")
|
||||
try:
|
||||
await asyncio.to_thread(roll_back, journal, controller.store)
|
||||
status.phase = "rolled_back"
|
||||
finally:
|
||||
await self._release(held)
|
||||
raise RenameConflict(
|
||||
500, "failed", f"The documents could not be written: {exc}", op=op
|
||||
) from exc
|
||||
status.phase = "forward"
|
||||
# Never started again as they were: the splice builds them anew.
|
||||
held.torn_down = []
|
||||
try:
|
||||
await asyncio.to_thread(self._finish_steps, journal, status)
|
||||
except Exception as exc:
|
||||
self.events.publish(
|
||||
{
|
||||
"type": "engine_degraded",
|
||||
"detail": f"A rename could not finish: {exc}",
|
||||
}
|
||||
)
|
||||
self._resume = asyncio.create_task(
|
||||
self._finish_later(journal, held, status)
|
||||
)
|
||||
raise RenameConflict(
|
||||
503,
|
||||
"completing",
|
||||
f"The rename could not finish yet and is being retried: {exc}",
|
||||
op=op,
|
||||
) from None
|
||||
return await self._complete(journal, held, status)
|
||||
|
||||
async def _hold(
|
||||
self, held: _Held, op: str, plan: RenamePlan, abort: threading.Event
|
||||
) -> None:
|
||||
"""The gate, the controller lock, the pipeline and the execution
|
||||
service, each once what runs in front of it has finished. What it got
|
||||
is in *held*.
|
||||
|
||||
:raises RenameConflict: 409 ``aborted`` once called off, 503 ``busy``
|
||||
when a rebuild keeps the controller for longer than one may.
|
||||
"""
|
||||
controller = self.controller
|
||||
affected = [doc.id for doc in plan.documents if doc.kind == "flow"]
|
||||
held.gate = await asyncio.to_thread(
|
||||
self.gate.close, op, affected, abort, self._report
|
||||
)
|
||||
if not held.gate:
|
||||
raise _aborted()
|
||||
try:
|
||||
await controller._acquire(REBUILD_WAIT)
|
||||
except RebuildBusy as exc:
|
||||
raise RenameConflict(503, "busy", str(exc), op=op) from None
|
||||
held.lock = True
|
||||
pipeline, execution = controller.pipeline, controller.execution
|
||||
if pipeline is not None:
|
||||
if not await asyncio.to_thread(pipeline.hold, abort, self._report):
|
||||
raise _aborted()
|
||||
held.pipeline = pipeline
|
||||
if execution is not None:
|
||||
if not await asyncio.to_thread(execution.hold, abort, self._report):
|
||||
raise _aborted()
|
||||
held.execution = execution
|
||||
if self.pending is not None:
|
||||
self.pending.busy = []
|
||||
|
||||
def _finish_steps(self, journal: Journal, status: RenameStatus) -> None:
|
||||
"""Every step after the documents, the audit and the journal's end.
|
||||
Blocking, and tried again until it succeeds."""
|
||||
execution = self.controller.execution
|
||||
try:
|
||||
run_steps(
|
||||
journal,
|
||||
self.controller.state,
|
||||
execution.queue if execution is not None else None,
|
||||
)
|
||||
finish(journal)
|
||||
except Exception as exc:
|
||||
status.error = journal.error or f"{type(exc).__name__}: {exc}"
|
||||
raise
|
||||
status.error = ""
|
||||
|
||||
async def _finish_later(
|
||||
self, journal: Journal, held: _Held, status: RenameStatus
|
||||
) -> None:
|
||||
"""Decision 5: held, and tried again however long it takes."""
|
||||
await retry_until_done(lambda: self._finish_steps(journal, status), self.events)
|
||||
await self._complete(journal, held, status)
|
||||
|
||||
async def _complete(
|
||||
self, journal: Journal, held: _Held, status: RenameStatus
|
||||
) -> RenameResult:
|
||||
"""The engine under the new names, then everything let go of and
|
||||
everyone told. Its journal is done already: what is left here lives
|
||||
in memory, which a restart builds afresh anyway."""
|
||||
rename = journal.rename
|
||||
if held.execution is not None:
|
||||
held.execution.rename_reclaimed(rename)
|
||||
self.events.rename(rename)
|
||||
if self.alerts is not None:
|
||||
self.alerts.rename(rename)
|
||||
try:
|
||||
await self.controller.splice_renamed(rename, journal.affected.rebuilt)
|
||||
except Exception as exc:
|
||||
# What a failed `reload_flow` leaves: those flows stopped, until
|
||||
# they are published again.
|
||||
logger.exception("Could not rebuild the renamed flows")
|
||||
self.events.publish(
|
||||
{
|
||||
"type": "engine_degraded",
|
||||
"detail": f"The renamed flows could not be rebuilt: {exc}",
|
||||
}
|
||||
)
|
||||
await self._release(held)
|
||||
if self.pending is status:
|
||||
self.pending = None
|
||||
self._announce(journal)
|
||||
return cast(RenameResult, journal.result)
|
||||
|
||||
async def _rename_dashboard(
|
||||
self, status: RenameStatus, request: RenameRequest, actor: str
|
||||
) -> RenameResult:
|
||||
done = await asyncio.to_thread(
|
||||
self._dashboard_documents, status, request, actor
|
||||
)
|
||||
if isinstance(done, RenameResult):
|
||||
return done
|
||||
await retry_until_done(lambda: self._finish_steps(done, status), self.events)
|
||||
if self.pending is status:
|
||||
self.pending = None
|
||||
self._announce(done)
|
||||
return cast(RenameResult, done.result)
|
||||
|
||||
def _dashboard_documents(
|
||||
self, status: RenameStatus, request: RenameRequest, actor: str
|
||||
) -> Journal | RenameResult:
|
||||
"""A dashboard's documents, renamed and committed. Blocking.
|
||||
|
||||
Under the dashboard store's lock, which every dashboard save takes
|
||||
before the flow store's, so none lands between the recheck and the
|
||||
write. Nothing else names a dashboard's key, so nothing is held.
|
||||
"""
|
||||
store = self.controller.store
|
||||
with self.dashboards._lock:
|
||||
checked = self._checked("dashboard", status.id, request)
|
||||
if isinstance(checked, RenameResult):
|
||||
return checked
|
||||
rename = Rename(checked.name, checked.new_name)
|
||||
rewrites = rewrite_documents(
|
||||
store, self.dashboards, "dashboard", status.id, rename
|
||||
)
|
||||
journal = begin(
|
||||
request.request_id,
|
||||
"dashboard",
|
||||
status.id,
|
||||
rename,
|
||||
rewrites,
|
||||
actor=actor,
|
||||
fingerprint=checked.fingerprint,
|
||||
)
|
||||
status.phase = "prepared"
|
||||
try:
|
||||
apply_documents(journal, store)
|
||||
except Exception as exc:
|
||||
roll_back(journal, store)
|
||||
status.phase = "rolled_back"
|
||||
raise RenameConflict(
|
||||
500,
|
||||
"failed",
|
||||
f"The documents could not be written: {exc}",
|
||||
op=request.request_id,
|
||||
) from exc
|
||||
status.phase = "forward"
|
||||
return journal
|
||||
|
||||
async def _release(self, held: _Held) -> None:
|
||||
"""Let go of what *held* says, last taken first. Flows taken down for
|
||||
a rename that does not go ahead start again as they were."""
|
||||
if held.queue is not None:
|
||||
try:
|
||||
await asyncio.to_thread(held.queue.release)
|
||||
except Exception:
|
||||
# Left in the held list, which the next start releases.
|
||||
logger.exception("Could not release the work held for a rename")
|
||||
for key in held.torn_down:
|
||||
await self.controller._activate(key)
|
||||
if held.execution is not None:
|
||||
held.execution.release()
|
||||
if held.pipeline is not None:
|
||||
held.pipeline.release()
|
||||
if held.lock:
|
||||
self.controller._lock.release()
|
||||
if held.gate:
|
||||
self.gate.open()
|
||||
|
||||
def _announce(self, journal: Journal) -> None:
|
||||
"""Tell the clients, once everything is let go of: editors and panels
|
||||
refetch, and the canvas takes the pipeline as it is now."""
|
||||
now = time.time()
|
||||
self.events.publish(
|
||||
{
|
||||
"type": "renamed",
|
||||
"kind": journal.kind,
|
||||
"id": journal.id,
|
||||
"old": journal.old,
|
||||
"new": journal.new,
|
||||
"op": journal.op,
|
||||
"ts": now,
|
||||
}
|
||||
)
|
||||
for dash_id in journal.affected.dashboards:
|
||||
self.events.publish(
|
||||
{"type": "dashboard_changed", "dashboard": dash_id, "ts": now}
|
||||
)
|
||||
if journal.kind != "flow":
|
||||
return
|
||||
# What the devices' ports are wired to moved with the flow.
|
||||
self.events.publish({"type": "dashboard_changed", "dashboard": "", "ts": now})
|
||||
controller = self.controller
|
||||
self.events.publish(
|
||||
{
|
||||
"type": "pipeline_rebuilt",
|
||||
"issues": [issue.model_dump() for issue in controller.issues],
|
||||
"nodes": [status.model_dump() for status in controller.node_statuses()],
|
||||
"paused": controller.paused_flows(),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -50,7 +50,9 @@ from fluksio.flow.provision import load_provisioners
|
||||
from fluksio.flow.queue import MemoryWorkQueue, RedisWorkQueue, WorkQueue
|
||||
from fluksio.flow.remote import RemoteWorkerHub
|
||||
from fluksio.flow.rename import (
|
||||
RenameConflict,
|
||||
RenameGate,
|
||||
Renamer,
|
||||
recover_documents,
|
||||
recover_runtime,
|
||||
retry_until_done,
|
||||
@@ -464,6 +466,14 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
# On the app so a superuser can ask it to flush now.
|
||||
app.state.metrics_collector = MetricsCollector(event_bus)
|
||||
_background(app.state.metrics_collector.run(), "metrics-collector")
|
||||
app.state.renamer = Renamer(
|
||||
controller,
|
||||
dashboards,
|
||||
rename_gate,
|
||||
event_bus,
|
||||
alerts=alerts,
|
||||
metrics=app.state.metrics_collector,
|
||||
)
|
||||
_background(_sweep_artifacts(artifacts, controller), "artifact-gc")
|
||||
if volatile is not None:
|
||||
_background(_trim_volatile(volatile), "artifact-ring")
|
||||
@@ -688,6 +698,18 @@ async def engine_held(request: Request, exc: EngineHeld) -> JSONResponse: # noq
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(RenameConflict)
|
||||
async def rename_conflict(request: Request, exc: RenameConflict) -> JSONResponse: # noqa: ARG001
|
||||
"""A rename answered with a code a client acts on, as the gate's 503 is:
|
||||
a refusal or a stale request (409), a key that is no key (422), or one
|
||||
finishing (503, which is worth retrying)."""
|
||||
return JSONResponse(
|
||||
status_code=exc.status,
|
||||
content={"detail": exc.detail, "code": exc.code, **exc.extra},
|
||||
headers={"Retry-After": "5"} if exc.status == 503 else None,
|
||||
)
|
||||
|
||||
|
||||
# Tagged because the operation-id builder reads the first tag; the route
|
||||
# itself stays out of the schema.
|
||||
@app.get(
|
||||
|
||||
Reference in New Issue
Block a user