Admission gate for a rename: RenameGate in flow/rename.py, installed by main.AdmissionGate inside CORS, turns mutating API requests away with 503 renaming and Retry-After while closed and drains those admitted without a limit, abortable; affected flows' hooks get 503 or socket close 1013, reads, sign-in, assistant, workers, artifacts and rename routes pass; the run cache misses while the gate is closed; trigger_node lets EngineHeld through instead of reporting a node failure
This commit is contained in:
@@ -49,6 +49,7 @@ from fluksio.flow.plugins import load_plugins
|
||||
from fluksio.flow.provision import load_provisioners
|
||||
from fluksio.flow.queue import MemoryWorkQueue, RedisWorkQueue, WorkQueue
|
||||
from fluksio.flow.remote import RemoteWorkerHub
|
||||
from fluksio.flow.rename import RenameGate
|
||||
from fluksio.flow.resources import (
|
||||
ResourceAccountant,
|
||||
fair_share_env,
|
||||
@@ -419,6 +420,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
state_factory=_run_state,
|
||||
artifacts=artifacts,
|
||||
parallel=settings.FLOW_MAX_RUNS,
|
||||
renaming=lambda: rename_gate.closed,
|
||||
)
|
||||
app.state.run_service = run_service
|
||||
# Said out loud because it is the only way to tell that a settings file was
|
||||
@@ -536,6 +538,52 @@ class DocsForThisMachine:
|
||||
await self.app(scope, receive, send)
|
||||
|
||||
|
||||
class AdmissionGate:
|
||||
"""Turn away what a rename must not see half done (`RenameGate`):
|
||||
mutating API requests, and the hooks of the flows it renames."""
|
||||
|
||||
def __init__(self, app: ASGIApp, gate: RenameGate) -> None:
|
||||
self.app = app
|
||||
self.gate = gate
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] not in ("http", "websocket"):
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
allowed, counted = self.gate.admit(scope)
|
||||
if not allowed:
|
||||
await _try_later(self.gate.op, scope, receive, send)
|
||||
return
|
||||
try:
|
||||
await self.app(scope, receive, send)
|
||||
finally:
|
||||
if counted:
|
||||
self.gate.leave(counted)
|
||||
|
||||
|
||||
async def _try_later(op: str, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
"""A 503 to retry, or a socket closed with 1013, try again later.
|
||||
|
||||
Accepted first: a socket closed before that is a refused handshake, and
|
||||
the client never sees the code.
|
||||
"""
|
||||
if scope["type"] == "websocket":
|
||||
await receive()
|
||||
await send({"type": "websocket.accept"})
|
||||
await send({"type": "websocket.close", "code": 1013})
|
||||
return
|
||||
response = JSONResponse(
|
||||
status_code=503,
|
||||
content={
|
||||
"detail": "A rename is in progress; try again shortly",
|
||||
"code": "renaming",
|
||||
"op": op,
|
||||
},
|
||||
headers={"Retry-After": "5"},
|
||||
)
|
||||
await response(scope, receive, send)
|
||||
|
||||
|
||||
def _is_loopback(host: str) -> bool:
|
||||
try:
|
||||
return ipaddress.ip_address(host).is_loopback
|
||||
@@ -553,6 +601,12 @@ app = FastAPI(
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
# Closed by a rename. Added first, so it sits inside the CORS layer and a
|
||||
# browser can read the 503 it answers.
|
||||
rename_gate = RenameGate()
|
||||
app.state.rename_gate = rename_gate
|
||||
app.add_middleware(AdmissionGate, gate=rename_gate)
|
||||
|
||||
# Set all CORS enabled origins
|
||||
if settings.all_cors_origins:
|
||||
app.add_middleware(
|
||||
|
||||
Reference in New Issue
Block a user