Admission gate for a rename: RenameGate in flow/rename.py, installed by main.AdmissionGate inside CORS, turns mutating API requests away with 503 renaming and Retry-After while closed and drains those admitted without a limit, abortable; affected flows' hooks get 503 or socket close 1013, reads, sign-in, assistant, workers, artifacts and rename routes pass; the run cache misses while the gate is closed; trigger_node lets EngineHeld through instead of reporting a node failure

This commit is contained in:
2026-11-02 20:53:08 +01:00
parent cb709c19ac
commit 81bd7c2724
7 changed files with 404 additions and 2 deletions
+54
View File
@@ -49,6 +49,7 @@ from fluksio.flow.plugins import load_plugins
from fluksio.flow.provision import load_provisioners
from fluksio.flow.queue import MemoryWorkQueue, RedisWorkQueue, WorkQueue
from fluksio.flow.remote import RemoteWorkerHub
from fluksio.flow.rename import RenameGate
from fluksio.flow.resources import (
ResourceAccountant,
fair_share_env,
@@ -419,6 +420,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
state_factory=_run_state,
artifacts=artifacts,
parallel=settings.FLOW_MAX_RUNS,
renaming=lambda: rename_gate.closed,
)
app.state.run_service = run_service
# Said out loud because it is the only way to tell that a settings file was
@@ -536,6 +538,52 @@ class DocsForThisMachine:
await self.app(scope, receive, send)
class AdmissionGate:
"""Turn away what a rename must not see half done (`RenameGate`):
mutating API requests, and the hooks of the flows it renames."""
def __init__(self, app: ASGIApp, gate: RenameGate) -> None:
self.app = app
self.gate = gate
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] not in ("http", "websocket"):
await self.app(scope, receive, send)
return
allowed, counted = self.gate.admit(scope)
if not allowed:
await _try_later(self.gate.op, scope, receive, send)
return
try:
await self.app(scope, receive, send)
finally:
if counted:
self.gate.leave(counted)
async def _try_later(op: str, scope: Scope, receive: Receive, send: Send) -> None:
"""A 503 to retry, or a socket closed with 1013, try again later.
Accepted first: a socket closed before that is a refused handshake, and
the client never sees the code.
"""
if scope["type"] == "websocket":
await receive()
await send({"type": "websocket.accept"})
await send({"type": "websocket.close", "code": 1013})
return
response = JSONResponse(
status_code=503,
content={
"detail": "A rename is in progress; try again shortly",
"code": "renaming",
"op": op,
},
headers={"Retry-After": "5"},
)
await response(scope, receive, send)
def _is_loopback(host: str) -> bool:
try:
return ipaddress.ip_address(host).is_loopback
@@ -553,6 +601,12 @@ app = FastAPI(
lifespan=lifespan,
)
# Closed by a rename. Added first, so it sits inside the CORS layer and a
# browser can read the 503 it answers.
rename_gate = RenameGate()
app.state.rename_gate = rename_gate
app.add_middleware(AdmissionGate, gate=rename_gate)
# Set all CORS enabled origins
if settings.all_cors_origins:
app.add_middleware(