Tell someone when the engine breaks
Everything that goes wrong already travelled the event bus, but the only subscriber was the editor's websocket — so a flow quarantined at three in the morning was invisible until someone opened the browser. An alert manager now watches the same bus and forwards failures to ntfy, email or a webhook. Most of what it does is decline to send: the same node failing every second is one alert with a count of what followed, a connection flapping up and down is muted until it settles, and nothing gets past ten notifications an hour. Verified against a live instance — six identical failures produced one alert carrying the real traceback message. Channels and rules are configured through the API, with a test send so a channel can be proven before anything depends on it.
This commit is contained in:
@@ -10,9 +10,11 @@ from fastapi.routing import APIRoute
|
||||
from starlette.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.api.main import api_router
|
||||
from app.api.routes.alerts import read_config as read_alerts_config
|
||||
from app.core import security
|
||||
from app.core.config import settings
|
||||
from app.flow import logs
|
||||
from app.flow.alerts import AlertManager
|
||||
from app.flow.controller import FlowController
|
||||
from app.flow.events import event_bus
|
||||
from app.flow.executor import ExecutionService
|
||||
@@ -65,6 +67,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
# Connectors register their node types before any flow is built with them.
|
||||
load_plugins()
|
||||
|
||||
alerts = AlertManager(event_bus, config=read_alerts_config())
|
||||
execution = ExecutionService(
|
||||
queue=_work_queue(),
|
||||
max_workers=settings.FLOW_MAX_WORKERS,
|
||||
@@ -77,11 +80,13 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
max_workers=settings.FLOW_MAX_WORKERS,
|
||||
fastapi_app=app,
|
||||
execution=execution,
|
||||
alerts=alerts,
|
||||
)
|
||||
app.state.flow_controller = controller
|
||||
watchdog = LoopWatchdog(event_bus)
|
||||
app.state.watchdog = watchdog
|
||||
watchdog_task = asyncio.create_task(watchdog.run(), name="loop-watchdog")
|
||||
alerts_task = asyncio.create_task(alerts.run(), name="alert-manager")
|
||||
await controller.start()
|
||||
try:
|
||||
# A mounted sub-app gets no lifespan of its own, so the MCP session
|
||||
@@ -90,6 +95,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
yield
|
||||
finally:
|
||||
watchdog_task.cancel()
|
||||
alerts_task.cancel()
|
||||
await controller.stop()
|
||||
close_shared_client()
|
||||
if settings.MCP_ENABLED:
|
||||
|
||||
Reference in New Issue
Block a user