stroblmeandClaude Opus 5
60d7ec81c0
A wheel whose top-level module is `app` collides with anything else in a user's venv, so the package that is about to be published takes the name it is published under. Only the Python package moves; the repo, the Docker WORKDIR and the compose project keep theirs. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
409 lines
14 KiB
Python
409 lines
14 KiB
Python
"""What the engine has been doing: health now, and history since.
|
|
|
|
The live state comes from the controller; everything older than the websocket's
|
|
memory comes from the rollups the metrics collector writes. Deliberately not
|
|
built on ``/utils/health/``: that endpoint answers 503 when something is wrong,
|
|
which the generated SDK turns into a thrown error — and a health page that
|
|
cannot render while the engine is degraded is the wrong way round.
|
|
"""
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any, Literal
|
|
|
|
from fastapi import APIRouter, Depends, Request
|
|
from fastapi.concurrency import run_in_threadpool
|
|
from pydantic import BaseModel
|
|
from sqlalchemy import ColumnElement, DateTime, Interval, cast, func, literal
|
|
from sqlalchemy import select as sa_select
|
|
from sqlmodel import col, select
|
|
|
|
from fluksio.api.deps import FlowControllerDep, SessionDep, get_current_user
|
|
from fluksio.core.config import settings
|
|
from fluksio.flow.controller import ADVISORY_ISSUES, NodeStatus
|
|
from fluksio.models import EngineEvent, FlowRun, MetricBucket
|
|
|
|
router = APIRouter(
|
|
prefix="/observability",
|
|
tags=["observability"],
|
|
dependencies=[Depends(get_current_user)],
|
|
)
|
|
|
|
#: How many slices a per-flow sparkline is folded into.
|
|
SPARK_SLICES = 60
|
|
|
|
#: The origin fixed-stride slots are aligned to, which is the alignment the
|
|
#: fold used to get from ``stamp - stamp % bucket_s``.
|
|
EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
|
|
|
|
|
|
class HealthSummary(BaseModel):
|
|
status: str
|
|
problems: list[str]
|
|
flows: dict[str, int]
|
|
nodes: dict[str, int]
|
|
queue: dict[str, Any]
|
|
loop_lag: dict[str, float]
|
|
|
|
|
|
class SeriesPoint(BaseModel):
|
|
ts: float
|
|
executions: int
|
|
errors: int
|
|
messages: int
|
|
avg_ms: float
|
|
max_ms: float
|
|
avg_lag_ms: float
|
|
|
|
|
|
class FlowRollup(BaseModel):
|
|
flow: str
|
|
executions: int
|
|
errors: int
|
|
messages: int
|
|
avg_ms: float
|
|
avg_lag_ms: float
|
|
spark: list[int]
|
|
last_error_ts: float | None = None
|
|
|
|
|
|
class RunRow(BaseModel):
|
|
id: str
|
|
flow: str
|
|
source: str
|
|
status: str
|
|
started_at: datetime
|
|
finished_at: datetime | None = None
|
|
nodes: int
|
|
errors: int
|
|
duration_ms: float
|
|
deliveries: int
|
|
|
|
|
|
class RunPage(BaseModel):
|
|
data: list[RunRow]
|
|
count: int
|
|
|
|
|
|
class EventRow(BaseModel):
|
|
id: int
|
|
ts: datetime
|
|
type: str
|
|
flow: str
|
|
node: str
|
|
detail: str
|
|
actor: str
|
|
|
|
|
|
class DeadLetter(BaseModel):
|
|
id: str
|
|
ts: float
|
|
flow: str
|
|
node: str
|
|
cause: str
|
|
reason: str
|
|
|
|
|
|
def _since(hours: int) -> datetime:
|
|
return datetime.now(timezone.utc) - timedelta(hours=hours)
|
|
|
|
|
|
def _window_hours(hours: int) -> int:
|
|
"""A window the rollups can answer for: an hour at least, retention at most.
|
|
|
|
Zero used to divide by nothing and answer 500, and nothing older than
|
|
retention exists, so a larger window is a scan that can only find less.
|
|
"""
|
|
return max(1, min(hours, settings.OBS_RETENTION_DAYS * 24))
|
|
|
|
|
|
def _aware(when: datetime) -> datetime:
|
|
"""A bound as the columns store it. A naive one is read as UTC."""
|
|
return when if when.tzinfo else when.replace(tzinfo=timezone.utc)
|
|
|
|
|
|
@router.get("/summary", response_model=HealthSummary)
|
|
async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
|
|
"""How the engine is doing right now. Always 200, degraded or not."""
|
|
watchdog = getattr(request.app.state, "watchdog", None)
|
|
problems: list[str] = []
|
|
if watchdog is not None and watchdog.degraded:
|
|
problems.append("event loop lagging")
|
|
|
|
queue = await run_in_threadpool(controller.queue_stats)
|
|
if queue.get("oldest_pending_s", 0) > 120:
|
|
problems.append("queue stalled")
|
|
if queue.get("error"):
|
|
problems.append("work queue unreachable")
|
|
|
|
names = await run_in_threadpool(controller.store.list_flows)
|
|
quarantined = controller.quarantined
|
|
paused = set(controller.paused_flows())
|
|
entries = list(controller.loaded.values())
|
|
errored = [e for e in entries if e.status is NodeStatus.ERROR]
|
|
if quarantined:
|
|
problems.append(f"{len(quarantined)} flow(s) quarantined")
|
|
if errored:
|
|
problems.append(f"{len(errored)} node(s) failed to load")
|
|
|
|
# What the canvas flags on a flow — a dependency loop, an input nothing
|
|
# feeds — stops that flow running just as surely as a node that will not
|
|
# load, and until now this screen was the one place it did not show.
|
|
# `node_error` is left out: those are the nodes already counted above.
|
|
invalid = sorted(
|
|
{
|
|
issue.flow
|
|
for issue in controller.issues
|
|
if issue.flow
|
|
and issue.code != "node_error"
|
|
and issue.code not in ADVISORY_ISSUES
|
|
}
|
|
)
|
|
if invalid:
|
|
problems.append(f"{len(invalid)} flow(s) cannot run: {', '.join(invalid)}")
|
|
|
|
return HealthSummary(
|
|
status="degraded" if problems else "ok",
|
|
problems=problems,
|
|
flows={
|
|
"total": len(names),
|
|
# What the engine will actually act on. Enabled is not enough:
|
|
# validation stops a flow as surely as quarantine does, and a
|
|
# paused one is holding its messages rather than running them.
|
|
"running": len(
|
|
[
|
|
n
|
|
for n in names
|
|
if controller.is_enabled(n)
|
|
and n not in quarantined
|
|
and n not in paused
|
|
and n not in invalid
|
|
]
|
|
),
|
|
"paused": len(paused),
|
|
"quarantined": len(quarantined),
|
|
"invalid": len(invalid),
|
|
},
|
|
nodes={"total": len(entries), "error": len(errored)},
|
|
queue=queue,
|
|
loop_lag=(
|
|
watchdog.snapshot()
|
|
if watchdog is not None
|
|
else {"ewma": 0.0, "max_60s": 0.0}
|
|
),
|
|
)
|
|
|
|
|
|
@router.get("/timeseries", response_model=list[SeriesPoint])
|
|
def read_timeseries(
|
|
session: SessionDep,
|
|
flow: str | None = None,
|
|
node: str | None = None,
|
|
hours: int = 24,
|
|
bucket_s: int = 60,
|
|
) -> Any:
|
|
"""Executions, errors and timings over time, summed across nodes."""
|
|
hours = _window_hours(hours)
|
|
# Postgres does the fold: a week of minute rows per node used to cross the
|
|
# wire on every poll, and only the slices need to. The casts are load
|
|
# bearing — date_bin() is overloaded on timestamp and timestamptz, and an
|
|
# untyped bind parameter leaves the call ambiguous.
|
|
stride = timedelta(seconds=max(60, bucket_s))
|
|
slot = func.date_bin(
|
|
cast(literal(stride), Interval),
|
|
col(MetricBucket.bucket),
|
|
cast(literal(EPOCH), DateTime(timezone=True)),
|
|
).label("slot")
|
|
statement = sa_select(
|
|
slot,
|
|
func.sum(col(MetricBucket.executions)).label("executions"),
|
|
func.sum(col(MetricBucket.errors)).label("errors"),
|
|
func.sum(col(MetricBucket.messages)).label("messages"),
|
|
func.sum(col(MetricBucket.duration_sum_ms)).label("duration_sum_ms"),
|
|
func.max(col(MetricBucket.duration_max_ms)).label("max_ms"),
|
|
func.sum(col(MetricBucket.lag_sum_ms)).label("lag_sum_ms"),
|
|
func.sum(col(MetricBucket.items)).label("items"),
|
|
).where(col(MetricBucket.bucket) >= _since(hours))
|
|
if flow:
|
|
statement = statement.where(col(MetricBucket.flow) == flow)
|
|
if node:
|
|
statement = statement.where(col(MetricBucket.node) == node)
|
|
|
|
return [
|
|
SeriesPoint(
|
|
ts=row.slot.timestamp(),
|
|
executions=int(row.executions),
|
|
errors=int(row.errors),
|
|
messages=int(row.messages),
|
|
avg_ms=round(row.duration_sum_ms / (row.executions or 1), 2),
|
|
max_ms=round(row.max_ms, 2),
|
|
avg_lag_ms=round(row.lag_sum_ms / (row.items or 1), 2),
|
|
)
|
|
for row in session.execute(statement.group_by(slot).order_by(slot))
|
|
]
|
|
|
|
|
|
@router.get("/flows", response_model=list[FlowRollup])
|
|
def read_flow_rollups(session: SessionDep, hours: int = 24) -> Any:
|
|
"""One row per flow, with a coarse trend of how much it ran."""
|
|
hours = _window_hours(hours)
|
|
since = _since(hours)
|
|
window = hours * 3600
|
|
start = since.timestamp()
|
|
|
|
# Binned to the sparkline slice rather than the minute, so a flow costs at
|
|
# most SPARK_SLICES rows however long the window is. The slice is the
|
|
# window over SPARK_SLICES, which for whole hours is whole minutes.
|
|
slot = func.date_bin(
|
|
cast(literal(timedelta(minutes=hours)), Interval),
|
|
col(MetricBucket.bucket),
|
|
cast(literal(since), DateTime(timezone=True)),
|
|
).label("slot")
|
|
statement = (
|
|
sa_select(
|
|
col(MetricBucket.flow),
|
|
slot,
|
|
func.sum(col(MetricBucket.executions)).label("executions"),
|
|
func.sum(col(MetricBucket.errors)).label("errors"),
|
|
func.sum(col(MetricBucket.messages)).label("messages"),
|
|
func.sum(col(MetricBucket.duration_sum_ms)).label("duration_sum_ms"),
|
|
func.sum(col(MetricBucket.lag_sum_ms)).label("lag_sum_ms"),
|
|
func.sum(col(MetricBucket.items)).label("items"),
|
|
)
|
|
.where(col(MetricBucket.bucket) >= since)
|
|
.group_by(col(MetricBucket.flow), slot)
|
|
)
|
|
|
|
rollups: dict[str, dict[str, Any]] = {}
|
|
for row in session.execute(statement):
|
|
entry = rollups.setdefault(
|
|
row.flow,
|
|
{
|
|
"executions": 0,
|
|
"errors": 0,
|
|
"messages": 0,
|
|
"duration_sum_ms": 0.0,
|
|
"lag_sum_ms": 0.0,
|
|
"items": 0,
|
|
"spark": [0] * SPARK_SLICES,
|
|
},
|
|
)
|
|
entry["executions"] += row.executions
|
|
entry["errors"] += row.errors
|
|
entry["messages"] += row.messages
|
|
entry["duration_sum_ms"] += row.duration_sum_ms
|
|
entry["lag_sum_ms"] += row.lag_sum_ms
|
|
entry["items"] += row.items
|
|
# A slot sits a whole number of slices from `since`, so this rounds
|
|
# rather than truncates: a float a hair short would lose a slice. The
|
|
# clamp holds the bucket landing exactly on the far edge in range.
|
|
index = min(
|
|
SPARK_SLICES - 1,
|
|
max(0, round((row.slot.timestamp() - start) / window * SPARK_SLICES)),
|
|
)
|
|
entry["spark"][index] += row.executions
|
|
|
|
# `.all()` first: a Result has `keys()`, so dict() would read it as a
|
|
# mapping and subscript it.
|
|
last_errors = dict(
|
|
session.exec(
|
|
select(col(EngineEvent.flow), func.max(col(EngineEvent.ts)))
|
|
.where(col(EngineEvent.type) == "node_error", col(EngineEvent.ts) >= since)
|
|
.group_by(col(EngineEvent.flow))
|
|
).all()
|
|
)
|
|
|
|
return [
|
|
FlowRollup(
|
|
flow=flow,
|
|
executions=entry["executions"],
|
|
errors=entry["errors"],
|
|
messages=entry["messages"],
|
|
avg_ms=round(entry["duration_sum_ms"] / (entry["executions"] or 1), 2),
|
|
avg_lag_ms=round(entry["lag_sum_ms"] / (entry["items"] or 1), 2),
|
|
spark=entry["spark"],
|
|
last_error_ts=(
|
|
last_errors[flow].timestamp() if flow in last_errors else None
|
|
),
|
|
)
|
|
for flow, entry in sorted(rollups.items())
|
|
]
|
|
|
|
|
|
@router.get("/runs", response_model=RunPage)
|
|
def read_runs(
|
|
session: SessionDep,
|
|
flow: str | None = None,
|
|
status: str | None = None,
|
|
since: datetime | None = None,
|
|
until: datetime | None = None,
|
|
limit: int = 50,
|
|
) -> Any:
|
|
"""Recent cascades, newest first, and how many there were in total.
|
|
|
|
``since`` is inclusive and ``until`` exclusive, so a window of one minute
|
|
holds exactly the runs of the minute bucket the charts are drawn from.
|
|
"""
|
|
filters: list[ColumnElement[bool]] = []
|
|
if flow:
|
|
filters.append(col(FlowRun.flow) == flow)
|
|
if status:
|
|
filters.append(col(FlowRun.status) == status)
|
|
if since:
|
|
filters.append(col(FlowRun.started_at) >= _aware(since))
|
|
if until:
|
|
filters.append(col(FlowRun.started_at) < _aware(until))
|
|
|
|
capped = min(limit, 200)
|
|
statement = select(FlowRun).where(*filters).order_by(col(FlowRun.started_at).desc())
|
|
rows = list(session.exec(statement.limit(capped)))
|
|
# A short page is its own total. The lists poll their whole range every
|
|
# thirty seconds, and counting on each of those would hand back what
|
|
# binning the metrics just saved — for a number that only ever says
|
|
# "there is more here than fits".
|
|
count = (
|
|
len(rows)
|
|
if len(rows) < capped
|
|
else int(
|
|
session.exec(
|
|
select(func.count()).select_from(FlowRun).where(*filters)
|
|
).one()
|
|
)
|
|
)
|
|
return {"data": rows, "count": count}
|
|
|
|
|
|
@router.get("/events", response_model=list[EventRow])
|
|
def read_events(
|
|
session: SessionDep,
|
|
kind: Literal["failure", "audit"] = "failure",
|
|
flow: str | None = None,
|
|
since: datetime | None = None,
|
|
until: datetime | None = None,
|
|
limit: int = 100,
|
|
) -> Any:
|
|
"""What went wrong, or who changed what. Newest first.
|
|
|
|
``since`` is inclusive and ``until`` exclusive, the same window ``/runs``
|
|
takes, so a list can cover the span the charts beside it are drawn from.
|
|
"""
|
|
statement = select(EngineEvent).order_by(col(EngineEvent.ts).desc())
|
|
if kind == "audit":
|
|
statement = statement.where(col(EngineEvent.type) == "audit")
|
|
else:
|
|
statement = statement.where(col(EngineEvent.type) != "audit")
|
|
if flow:
|
|
statement = statement.where(col(EngineEvent.flow) == flow)
|
|
if since:
|
|
statement = statement.where(col(EngineEvent.ts) >= _aware(since))
|
|
if until:
|
|
statement = statement.where(col(EngineEvent.ts) < _aware(until))
|
|
return list(session.exec(statement.limit(min(limit, 500))))
|
|
|
|
|
|
@router.get("/dead-letter", response_model=list[DeadLetter])
|
|
async def read_dead_letters(controller: FlowControllerDep, limit: int = 50) -> Any:
|
|
"""Work the engine gave up on, which nothing else surfaces."""
|
|
if controller.execution is None:
|
|
return []
|
|
return await run_in_threadpool(controller.execution.queue.dead_letters, limit)
|