Everything that goes wrong already travelled the event bus, but the only subscriber was the editor's websocket — so a flow quarantined at three in the morning was invisible until someone opened the browser. An alert manager now watches the same bus and forwards failures to ntfy, email or a webhook. Most of what it does is decline to send: the same node failing every second is one alert with a count of what followed, a connection flapping up and down is muted until it settles, and nothing gets past ten notifications an hour. Verified against a live instance — six identical failures produced one alert carrying the real traceback message. Channels and rules are configured through the API, with a test send so a channel can be proven before anything depends on it.
86 lines
2.8 KiB
Python
86 lines
2.8 KiB
Python
"""Alerting configuration: which failures go where."""
|
|
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException
|
|
from fastapi.concurrency import run_in_threadpool
|
|
|
|
from app.api.deps import FlowControllerDep, get_current_user
|
|
from app.core.config import settings
|
|
from app.flow.alerts import Alert, AlertsConfig
|
|
from app.models import Message
|
|
|
|
router = APIRouter(
|
|
prefix="/alerts", tags=["alerts"], dependencies=[Depends(get_current_user)]
|
|
)
|
|
|
|
|
|
def _path() -> Path:
|
|
return settings.ALERTS_FILE
|
|
|
|
|
|
def read_config() -> AlertsConfig:
|
|
"""The stored configuration, or an empty one. Blocking."""
|
|
path = _path()
|
|
if not path.exists():
|
|
return AlertsConfig()
|
|
try:
|
|
return AlertsConfig.model_validate_json(path.read_text())
|
|
except Exception:
|
|
# A hand-edited file that no longer parses must not stop the engine.
|
|
return AlertsConfig()
|
|
|
|
|
|
def write_config(config: AlertsConfig) -> None:
|
|
"""Blocking."""
|
|
path = _path()
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(config.model_dump_json(indent=2))
|
|
|
|
|
|
@router.get("/config", response_model=AlertsConfig)
|
|
async def read_alerts_config() -> Any:
|
|
"""What the engine alerts on, and where it sends it."""
|
|
return await run_in_threadpool(read_config)
|
|
|
|
|
|
@router.put("/config", response_model=AlertsConfig)
|
|
async def save_alerts_config(body: AlertsConfig, controller: FlowControllerDep) -> Any:
|
|
"""Replace the configuration. Takes effect immediately."""
|
|
known = {channel.name for channel in body.channels}
|
|
for rule in body.rules:
|
|
missing = [name for name in rule.channels if name not in known]
|
|
if missing:
|
|
raise HTTPException(
|
|
status_code=422,
|
|
detail=f"No channel named {', '.join(repr(m) for m in missing)}",
|
|
)
|
|
|
|
await run_in_threadpool(write_config, body)
|
|
if controller.alerts is not None:
|
|
controller.alerts.config = body
|
|
return body
|
|
|
|
|
|
@router.post("/test/{channel_name}", response_model=Message)
|
|
async def test_channel(channel_name: str, controller: FlowControllerDep) -> Any:
|
|
"""Send one alert, to prove the channel works before relying on it."""
|
|
if controller.alerts is None:
|
|
raise HTTPException(status_code=503, detail="Alerting is not running")
|
|
|
|
config = controller.alerts.config
|
|
channel = next((c for c in config.channels if c.name == channel_name), None)
|
|
if channel is None:
|
|
raise HTTPException(status_code=404, detail=f"No channel '{channel_name}'")
|
|
|
|
await controller.alerts.send(
|
|
channel,
|
|
Alert(
|
|
title="Fluksio test alert",
|
|
body="If you are reading this, the channel works.",
|
|
severity="warning",
|
|
),
|
|
)
|
|
return Message(message=f"Sent a test alert through '{channel_name}'")
|