Bound the pipeline teardown so a stuck node cannot wedge the controller
A node's stop() and a supervised task's cancellation are both waited on inside the rebuild lock, and neither had a deadline: an MQTT client whose broker never acknowledges the disconnect leaves aiomqtt's __aexit__ waiting forever, so reload() never returned and every start, stop or publish behind it hung until the container was restarted. Each node now gets five seconds to close and is abandoned after that, and cancel_all reports what is still running rather than waiting on it — it also no longer swallows a cancellation aimed at the caller, which used to make the lock holder unkillable. A rebuild asked for by a request gives up on the lock after fifteen seconds with RebuildBusy, answered as a 503. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01StpRc2C6au1WJ1EUU7fsfu
This commit is contained in:
+12
-2
@@ -4,7 +4,7 @@ from collections.abc import AsyncIterator
|
||||
from contextlib import AbstractAsyncContextManager, asynccontextmanager
|
||||
|
||||
import sentry_sdk
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.routing import APIRoute
|
||||
@@ -21,7 +21,7 @@ from fluksio.core.db import prepare
|
||||
from fluksio.flow import logs, modules
|
||||
from fluksio.flow.alerts import AlertManager
|
||||
from fluksio.flow.artifacts import ArtifactStore
|
||||
from fluksio.flow.controller import FlowController
|
||||
from fluksio.flow.controller import FlowController, RebuildBusy
|
||||
from fluksio.flow.dashboards import DashboardStore
|
||||
from fluksio.flow.events import event_bus
|
||||
from fluksio.flow.executor import ExecutionService
|
||||
@@ -234,6 +234,16 @@ if settings.all_cors_origins:
|
||||
app.include_router(api_router, prefix=settings.API_V1_STR)
|
||||
|
||||
|
||||
@app.exception_handler(RebuildBusy)
|
||||
async def rebuild_busy(request: Request, exc: RebuildBusy) -> JSONResponse: # noqa: ARG001
|
||||
"""Every route that deploys something answers a wedged rebuild the same way.
|
||||
|
||||
The request was fine and retrying it may well work, so this is the engine
|
||||
saying it is busy rather than the request having gone wrong.
|
||||
"""
|
||||
return JSONResponse(status_code=503, content={"detail": str(exc)})
|
||||
|
||||
|
||||
# Tagged because the operation-id builder reads the first tag; the route
|
||||
# itself stays out of the schema.
|
||||
@app.get(
|
||||
|
||||
Reference in New Issue
Block a user