Admission gate for a rename: RenameGate in flow/rename.py, installed by main.AdmissionGate inside CORS, turns mutating API requests away with 503 renaming and Retry-After while closed and drains those admitted without a limit, abortable; affected flows' hooks get 503 or socket close 1013, reads, sign-in, assistant, workers, artifacts and rename routes pass; the run cache misses while the gate is closed; trigger_node lets EngineHeld through instead of reporting a node failure

This commit is contained in:
2026-11-02 20:53:08 +01:00
parent cb709c19ac
commit 81bd7c2724
7 changed files with 404 additions and 2 deletions
+6
View File
@@ -65,6 +65,7 @@ from fluksio.flow.nodes import (
WrenNode,
)
from fluksio.flow.pipeline import (
EngineHeld,
NodeOutcome,
Pipeline,
RunCacheLookup,
@@ -1984,6 +1985,9 @@ class FlowController:
Returns what the node raised, or an empty string. A node failing by hand
is reported the way a queued run reports it — on the canvas and in the
metrics — rather than as a stack trace in the server log.
:raises EngineHeld: While a rename holds the engine, which is no
failure of the node's.
"""
node = self.get_node(node_id)
if node is None:
@@ -1993,6 +1997,8 @@ class FlowController:
node.trigger(values, durable=False)
else:
node.inject(values or {}, durable=False)
except EngineHeld:
raise
except Exception as exc:
if self.pipeline is None:
raise
+122
View File
@@ -0,0 +1,122 @@
"""Renaming a flow's key: the door that keeps the API out while it runs.
A rename moves everything named under a flow's key, in documents, history,
live state and queued work, so nothing may write through the API meanwhile.
`RenameGate` is that door. ``docs/private/id_refac_rename.md`` has the whole
protocol.
"""
from __future__ import annotations
import re
import threading
from collections import Counter
from collections.abc import Callable, Iterable
from starlette.types import Scope
from fluksio.core.config import settings
from fluksio.flow.controller import HOOK_PREFIX
from fluksio.flow.executor import wait_out
#: What reading looks like. Everything else under the API changes something.
_READS = frozenset({"GET", "HEAD", "OPTIONS"})
#: Mutations that cannot reach a flow's namespace, under the API prefix: signing
#: in and accounts, the assistant (whose tool calls come back through the API
#: and are held there), a browser's push subscription, the portal, workers
#: attaching, artifact bytes (which are named by digest, and which a node on a
#: remote worker writes while the rename waits for it), the socket ticket and
#: the rename routes, which keep their own order.
_EXEMPT = re.compile(
r"/(?:login|password-recovery|password-recovery-html-content|reset-password"
r"|oauth|users|ai|cloud|workers|artifacts)(?:/|$)"
r"|/alerts/webpush(?:/|$)"
r"|/flows/ws-ticket$"
r"|/(?:flows|dashboards)/[^/]+/rename(?:/plan)?$"
)
class RenameGate:
"""The door every request passes, which a rename closes.
Open, it counts the mutating API requests it lets in, for as long as each
takes. Closed, it turns new ones away and the rename waits for those
already in, so it starts from a namespace nothing is writing to through
the API. Hooks of the flows being renamed are turned away too, since their
nodes are torn down meanwhile. Reads, websockets and the exempt routes
pass either way, uncounted.
Thread-safe: requests pass on the event loop and `close` waits on a
thread, as the engine holds do.
"""
def __init__(self) -> None:
self._lock = threading.Condition()
self._admitted: Counter[str] = Counter()
self._closed = False
self._affected: frozenset[str] = frozenset()
#: The rename that closed it, for whoever is turned away.
self.op = ""
@property
def closed(self) -> bool:
with self._lock:
return self._closed
def admit(self, scope: Scope) -> tuple[bool, str]:
"""Whether *scope* may pass, and the name it is counted under until
`leave`, empty when it is not counted."""
path: str = scope["path"]
with self._lock:
if path.startswith(f"{HOOK_PREFIX}/"):
flow_id = path[len(HOOK_PREFIX) + 1 :].split("/", 1)[0]
return not (self._closed and flow_id in self._affected), ""
api = settings.API_V1_STR
if (
scope["type"] != "http"
or scope["method"] in _READS
or not path.startswith(f"{api}/")
or _EXEMPT.match(path, len(api))
):
return True, ""
if self._closed:
return False, ""
name = f"{scope['method']} {path}"
self._admitted[name] += 1
return True, name
def leave(self, name: str) -> None:
"""A request counted under *name* is done."""
with self._lock:
self._admitted -= Counter([name])
self._lock.notify_all()
def close(
self,
op: str,
affected: Iterable[str],
abort: threading.Event,
report: Callable[[list[str]], None] | None = None,
) -> bool:
"""Turn mutations and the hooks of the *affected* flow ids away, then
wait for the requests already in to finish. Blocking.
Without a limit: ``report`` hears the requests still running each time
they change, and setting ``abort`` gives up, opens the gate again and
answers False. True means closed until `open`.
"""
with self._lock:
self._closed = True
self._affected = frozenset(affected)
self.op = op
if wait_out(self._lock, lambda: sorted(self._admitted), abort, report):
return True
self.open()
return False
def open(self) -> None:
with self._lock:
self._closed = False
self._affected = frozenset()
self.op = ""
+10 -2
View File
@@ -813,6 +813,7 @@ class RunCache:
self,
artifacts: ArtifactStore | None = None,
key_of: Callable[[str], str] | None = None,
renaming: Callable[[], bool] | None = None,
) -> None:
self.artifacts = artifacts
#: The key a source run's outputs are named with, from its flow's id:
@@ -820,9 +821,13 @@ class RunCache:
#: one the id is taken for the key, which it is for a flow from before
#: ids.
self.key_of = key_of or (lambda flow_id: flow_id)
#: Whether a rename is under way, which may have moved some of a
#: source run's output names and not yet the others. Every lookup
#: misses meanwhile, which only costs running the node.
self.renaming = renaming or (lambda: False)
def lookup(self, key: str) -> CacheHit | None:
if not key:
if not key or self.renaming():
return None
try:
with Session(db_engine) as session:
@@ -874,12 +879,15 @@ class RunService:
state_factory: Callable[[str], StateBackend] | None = None,
parallel: int = MAX_PARALLEL,
artifacts: ArtifactStore | None = None,
renaming: Callable[[], bool] | None = None,
) -> None:
self.controller = controller
self.queue = queue
self._artifacts = artifacts
self._cache = RunCache(
artifacts, lambda flow_id: tombstones.name_of(flow_id, controller.store)
artifacts,
lambda flow_id: tombstones.name_of(flow_id, controller.store),
renaming,
)
# Without one, a run gets a private in-memory state — which is exactly
# the isolation it wants, minus surviving the process.
+54
View File
@@ -49,6 +49,7 @@ from fluksio.flow.plugins import load_plugins
from fluksio.flow.provision import load_provisioners
from fluksio.flow.queue import MemoryWorkQueue, RedisWorkQueue, WorkQueue
from fluksio.flow.remote import RemoteWorkerHub
from fluksio.flow.rename import RenameGate
from fluksio.flow.resources import (
ResourceAccountant,
fair_share_env,
@@ -419,6 +420,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
state_factory=_run_state,
artifacts=artifacts,
parallel=settings.FLOW_MAX_RUNS,
renaming=lambda: rename_gate.closed,
)
app.state.run_service = run_service
# Said out loud because it is the only way to tell that a settings file was
@@ -536,6 +538,52 @@ class DocsForThisMachine:
await self.app(scope, receive, send)
class AdmissionGate:
"""Turn away what a rename must not see half done (`RenameGate`):
mutating API requests, and the hooks of the flows it renames."""
def __init__(self, app: ASGIApp, gate: RenameGate) -> None:
self.app = app
self.gate = gate
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] not in ("http", "websocket"):
await self.app(scope, receive, send)
return
allowed, counted = self.gate.admit(scope)
if not allowed:
await _try_later(self.gate.op, scope, receive, send)
return
try:
await self.app(scope, receive, send)
finally:
if counted:
self.gate.leave(counted)
async def _try_later(op: str, scope: Scope, receive: Receive, send: Send) -> None:
"""A 503 to retry, or a socket closed with 1013, try again later.
Accepted first: a socket closed before that is a refused handshake, and
the client never sees the code.
"""
if scope["type"] == "websocket":
await receive()
await send({"type": "websocket.accept"})
await send({"type": "websocket.close", "code": 1013})
return
response = JSONResponse(
status_code=503,
content={
"detail": "A rename is in progress; try again shortly",
"code": "renaming",
"op": op,
},
headers={"Retry-After": "5"},
)
await response(scope, receive, send)
def _is_loopback(host: str) -> bool:
try:
return ipaddress.ip_address(host).is_loopback
@@ -553,6 +601,12 @@ app = FastAPI(
lifespan=lifespan,
)
# Closed by a rename. Added first, so it sits inside the CORS layer and a
# browser can read the 503 it answers.
rename_gate = RenameGate()
app.state.rename_gate = rename_gate
app.add_middleware(AdmissionGate, gate=rename_gate)
# Set all CORS enabled origins
if settings.all_cors_origins:
app.add_middleware(