Tell someone when the engine breaks

Everything that goes wrong already travelled the event bus, but the only
subscriber was the editor's websocket — so a flow quarantined at three in
the morning was invisible until someone opened the browser.

An alert manager now watches the same bus and forwards failures to ntfy,
email or a webhook. Most of what it does is decline to send: the same
node failing every second is one alert with a count of what followed, a
connection flapping up and down is muted until it settles, and nothing
gets past ten notifications an hour. Verified against a live instance —
six identical failures produced one alert carrying the real traceback
message.

Channels and rules are configured through the API, with a test send so a
channel can be proven before anything depends on it.
This commit is contained in:
root
2026-08-16 08:01:50 +02:00
parent 92ccd6c930
commit c0ac383bff
12 changed files with 824 additions and 3 deletions
+6
View File
@@ -10,9 +10,11 @@ from fastapi.routing import APIRoute
from starlette.middleware.cors import CORSMiddleware
from app.api.main import api_router
from app.api.routes.alerts import read_config as read_alerts_config
from app.core import security
from app.core.config import settings
from app.flow import logs
from app.flow.alerts import AlertManager
from app.flow.controller import FlowController
from app.flow.events import event_bus
from app.flow.executor import ExecutionService
@@ -65,6 +67,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
# Connectors register their node types before any flow is built with them.
load_plugins()
alerts = AlertManager(event_bus, config=read_alerts_config())
execution = ExecutionService(
queue=_work_queue(),
max_workers=settings.FLOW_MAX_WORKERS,
@@ -77,11 +80,13 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
max_workers=settings.FLOW_MAX_WORKERS,
fastapi_app=app,
execution=execution,
alerts=alerts,
)
app.state.flow_controller = controller
watchdog = LoopWatchdog(event_bus)
app.state.watchdog = watchdog
watchdog_task = asyncio.create_task(watchdog.run(), name="loop-watchdog")
alerts_task = asyncio.create_task(alerts.run(), name="alert-manager")
await controller.start()
try:
# A mounted sub-app gets no lifespan of its own, so the MCP session
@@ -90,6 +95,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
yield
finally:
watchdog_task.cancel()
alerts_task.cancel()
await controller.stop()
close_shared_client()
if settings.MCP_ENABLED: