Rename the import package app to fluksio
A wheel whose top-level module is `app` collides with anything else in a user's venv, so the package that is about to be published takes the name it is published under. Only the Python package moves; the repo, the Docker WORKDIR and the compose project keep theirs. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,408 @@
|
||||
"""What the engine has been doing: health now, and history since.
|
||||
|
||||
The live state comes from the controller; everything older than the websocket's
|
||||
memory comes from the rollups the metrics collector writes. Deliberately not
|
||||
built on ``/utils/health/``: that endpoint answers 503 when something is wrong,
|
||||
which the generated SDK turns into a thrown error — and a health page that
|
||||
cannot render while the engine is degraded is the wrong way round.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import ColumnElement, DateTime, Interval, cast, func, literal
|
||||
from sqlalchemy import select as sa_select
|
||||
from sqlmodel import col, select
|
||||
|
||||
from fluksio.api.deps import FlowControllerDep, SessionDep, get_current_user
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.flow.controller import ADVISORY_ISSUES, NodeStatus
|
||||
from fluksio.models import EngineEvent, FlowRun, MetricBucket
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/observability",
|
||||
tags=["observability"],
|
||||
dependencies=[Depends(get_current_user)],
|
||||
)
|
||||
|
||||
#: How many slices a per-flow sparkline is folded into.
|
||||
SPARK_SLICES = 60
|
||||
|
||||
#: The origin fixed-stride slots are aligned to, which is the alignment the
|
||||
#: fold used to get from ``stamp - stamp % bucket_s``.
|
||||
EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
class HealthSummary(BaseModel):
|
||||
status: str
|
||||
problems: list[str]
|
||||
flows: dict[str, int]
|
||||
nodes: dict[str, int]
|
||||
queue: dict[str, Any]
|
||||
loop_lag: dict[str, float]
|
||||
|
||||
|
||||
class SeriesPoint(BaseModel):
|
||||
ts: float
|
||||
executions: int
|
||||
errors: int
|
||||
messages: int
|
||||
avg_ms: float
|
||||
max_ms: float
|
||||
avg_lag_ms: float
|
||||
|
||||
|
||||
class FlowRollup(BaseModel):
|
||||
flow: str
|
||||
executions: int
|
||||
errors: int
|
||||
messages: int
|
||||
avg_ms: float
|
||||
avg_lag_ms: float
|
||||
spark: list[int]
|
||||
last_error_ts: float | None = None
|
||||
|
||||
|
||||
class RunRow(BaseModel):
|
||||
id: str
|
||||
flow: str
|
||||
source: str
|
||||
status: str
|
||||
started_at: datetime
|
||||
finished_at: datetime | None = None
|
||||
nodes: int
|
||||
errors: int
|
||||
duration_ms: float
|
||||
deliveries: int
|
||||
|
||||
|
||||
class RunPage(BaseModel):
|
||||
data: list[RunRow]
|
||||
count: int
|
||||
|
||||
|
||||
class EventRow(BaseModel):
|
||||
id: int
|
||||
ts: datetime
|
||||
type: str
|
||||
flow: str
|
||||
node: str
|
||||
detail: str
|
||||
actor: str
|
||||
|
||||
|
||||
class DeadLetter(BaseModel):
|
||||
id: str
|
||||
ts: float
|
||||
flow: str
|
||||
node: str
|
||||
cause: str
|
||||
reason: str
|
||||
|
||||
|
||||
def _since(hours: int) -> datetime:
|
||||
return datetime.now(timezone.utc) - timedelta(hours=hours)
|
||||
|
||||
|
||||
def _window_hours(hours: int) -> int:
|
||||
"""A window the rollups can answer for: an hour at least, retention at most.
|
||||
|
||||
Zero used to divide by nothing and answer 500, and nothing older than
|
||||
retention exists, so a larger window is a scan that can only find less.
|
||||
"""
|
||||
return max(1, min(hours, settings.OBS_RETENTION_DAYS * 24))
|
||||
|
||||
|
||||
def _aware(when: datetime) -> datetime:
|
||||
"""A bound as the columns store it. A naive one is read as UTC."""
|
||||
return when if when.tzinfo else when.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
@router.get("/summary", response_model=HealthSummary)
|
||||
async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
|
||||
"""How the engine is doing right now. Always 200, degraded or not."""
|
||||
watchdog = getattr(request.app.state, "watchdog", None)
|
||||
problems: list[str] = []
|
||||
if watchdog is not None and watchdog.degraded:
|
||||
problems.append("event loop lagging")
|
||||
|
||||
queue = await run_in_threadpool(controller.queue_stats)
|
||||
if queue.get("oldest_pending_s", 0) > 120:
|
||||
problems.append("queue stalled")
|
||||
if queue.get("error"):
|
||||
problems.append("work queue unreachable")
|
||||
|
||||
names = await run_in_threadpool(controller.store.list_flows)
|
||||
quarantined = controller.quarantined
|
||||
paused = set(controller.paused_flows())
|
||||
entries = list(controller.loaded.values())
|
||||
errored = [e for e in entries if e.status is NodeStatus.ERROR]
|
||||
if quarantined:
|
||||
problems.append(f"{len(quarantined)} flow(s) quarantined")
|
||||
if errored:
|
||||
problems.append(f"{len(errored)} node(s) failed to load")
|
||||
|
||||
# What the canvas flags on a flow — a dependency loop, an input nothing
|
||||
# feeds — stops that flow running just as surely as a node that will not
|
||||
# load, and until now this screen was the one place it did not show.
|
||||
# `node_error` is left out: those are the nodes already counted above.
|
||||
invalid = sorted(
|
||||
{
|
||||
issue.flow
|
||||
for issue in controller.issues
|
||||
if issue.flow
|
||||
and issue.code != "node_error"
|
||||
and issue.code not in ADVISORY_ISSUES
|
||||
}
|
||||
)
|
||||
if invalid:
|
||||
problems.append(f"{len(invalid)} flow(s) cannot run: {', '.join(invalid)}")
|
||||
|
||||
return HealthSummary(
|
||||
status="degraded" if problems else "ok",
|
||||
problems=problems,
|
||||
flows={
|
||||
"total": len(names),
|
||||
# What the engine will actually act on. Enabled is not enough:
|
||||
# validation stops a flow as surely as quarantine does, and a
|
||||
# paused one is holding its messages rather than running them.
|
||||
"running": len(
|
||||
[
|
||||
n
|
||||
for n in names
|
||||
if controller.is_enabled(n)
|
||||
and n not in quarantined
|
||||
and n not in paused
|
||||
and n not in invalid
|
||||
]
|
||||
),
|
||||
"paused": len(paused),
|
||||
"quarantined": len(quarantined),
|
||||
"invalid": len(invalid),
|
||||
},
|
||||
nodes={"total": len(entries), "error": len(errored)},
|
||||
queue=queue,
|
||||
loop_lag=(
|
||||
watchdog.snapshot()
|
||||
if watchdog is not None
|
||||
else {"ewma": 0.0, "max_60s": 0.0}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/timeseries", response_model=list[SeriesPoint])
|
||||
def read_timeseries(
|
||||
session: SessionDep,
|
||||
flow: str | None = None,
|
||||
node: str | None = None,
|
||||
hours: int = 24,
|
||||
bucket_s: int = 60,
|
||||
) -> Any:
|
||||
"""Executions, errors and timings over time, summed across nodes."""
|
||||
hours = _window_hours(hours)
|
||||
# Postgres does the fold: a week of minute rows per node used to cross the
|
||||
# wire on every poll, and only the slices need to. The casts are load
|
||||
# bearing — date_bin() is overloaded on timestamp and timestamptz, and an
|
||||
# untyped bind parameter leaves the call ambiguous.
|
||||
stride = timedelta(seconds=max(60, bucket_s))
|
||||
slot = func.date_bin(
|
||||
cast(literal(stride), Interval),
|
||||
col(MetricBucket.bucket),
|
||||
cast(literal(EPOCH), DateTime(timezone=True)),
|
||||
).label("slot")
|
||||
statement = sa_select(
|
||||
slot,
|
||||
func.sum(col(MetricBucket.executions)).label("executions"),
|
||||
func.sum(col(MetricBucket.errors)).label("errors"),
|
||||
func.sum(col(MetricBucket.messages)).label("messages"),
|
||||
func.sum(col(MetricBucket.duration_sum_ms)).label("duration_sum_ms"),
|
||||
func.max(col(MetricBucket.duration_max_ms)).label("max_ms"),
|
||||
func.sum(col(MetricBucket.lag_sum_ms)).label("lag_sum_ms"),
|
||||
func.sum(col(MetricBucket.items)).label("items"),
|
||||
).where(col(MetricBucket.bucket) >= _since(hours))
|
||||
if flow:
|
||||
statement = statement.where(col(MetricBucket.flow) == flow)
|
||||
if node:
|
||||
statement = statement.where(col(MetricBucket.node) == node)
|
||||
|
||||
return [
|
||||
SeriesPoint(
|
||||
ts=row.slot.timestamp(),
|
||||
executions=int(row.executions),
|
||||
errors=int(row.errors),
|
||||
messages=int(row.messages),
|
||||
avg_ms=round(row.duration_sum_ms / (row.executions or 1), 2),
|
||||
max_ms=round(row.max_ms, 2),
|
||||
avg_lag_ms=round(row.lag_sum_ms / (row.items or 1), 2),
|
||||
)
|
||||
for row in session.execute(statement.group_by(slot).order_by(slot))
|
||||
]
|
||||
|
||||
|
||||
@router.get("/flows", response_model=list[FlowRollup])
|
||||
def read_flow_rollups(session: SessionDep, hours: int = 24) -> Any:
|
||||
"""One row per flow, with a coarse trend of how much it ran."""
|
||||
hours = _window_hours(hours)
|
||||
since = _since(hours)
|
||||
window = hours * 3600
|
||||
start = since.timestamp()
|
||||
|
||||
# Binned to the sparkline slice rather than the minute, so a flow costs at
|
||||
# most SPARK_SLICES rows however long the window is. The slice is the
|
||||
# window over SPARK_SLICES, which for whole hours is whole minutes.
|
||||
slot = func.date_bin(
|
||||
cast(literal(timedelta(minutes=hours)), Interval),
|
||||
col(MetricBucket.bucket),
|
||||
cast(literal(since), DateTime(timezone=True)),
|
||||
).label("slot")
|
||||
statement = (
|
||||
sa_select(
|
||||
col(MetricBucket.flow),
|
||||
slot,
|
||||
func.sum(col(MetricBucket.executions)).label("executions"),
|
||||
func.sum(col(MetricBucket.errors)).label("errors"),
|
||||
func.sum(col(MetricBucket.messages)).label("messages"),
|
||||
func.sum(col(MetricBucket.duration_sum_ms)).label("duration_sum_ms"),
|
||||
func.sum(col(MetricBucket.lag_sum_ms)).label("lag_sum_ms"),
|
||||
func.sum(col(MetricBucket.items)).label("items"),
|
||||
)
|
||||
.where(col(MetricBucket.bucket) >= since)
|
||||
.group_by(col(MetricBucket.flow), slot)
|
||||
)
|
||||
|
||||
rollups: dict[str, dict[str, Any]] = {}
|
||||
for row in session.execute(statement):
|
||||
entry = rollups.setdefault(
|
||||
row.flow,
|
||||
{
|
||||
"executions": 0,
|
||||
"errors": 0,
|
||||
"messages": 0,
|
||||
"duration_sum_ms": 0.0,
|
||||
"lag_sum_ms": 0.0,
|
||||
"items": 0,
|
||||
"spark": [0] * SPARK_SLICES,
|
||||
},
|
||||
)
|
||||
entry["executions"] += row.executions
|
||||
entry["errors"] += row.errors
|
||||
entry["messages"] += row.messages
|
||||
entry["duration_sum_ms"] += row.duration_sum_ms
|
||||
entry["lag_sum_ms"] += row.lag_sum_ms
|
||||
entry["items"] += row.items
|
||||
# A slot sits a whole number of slices from `since`, so this rounds
|
||||
# rather than truncates: a float a hair short would lose a slice. The
|
||||
# clamp holds the bucket landing exactly on the far edge in range.
|
||||
index = min(
|
||||
SPARK_SLICES - 1,
|
||||
max(0, round((row.slot.timestamp() - start) / window * SPARK_SLICES)),
|
||||
)
|
||||
entry["spark"][index] += row.executions
|
||||
|
||||
# `.all()` first: a Result has `keys()`, so dict() would read it as a
|
||||
# mapping and subscript it.
|
||||
last_errors = dict(
|
||||
session.exec(
|
||||
select(col(EngineEvent.flow), func.max(col(EngineEvent.ts)))
|
||||
.where(col(EngineEvent.type) == "node_error", col(EngineEvent.ts) >= since)
|
||||
.group_by(col(EngineEvent.flow))
|
||||
).all()
|
||||
)
|
||||
|
||||
return [
|
||||
FlowRollup(
|
||||
flow=flow,
|
||||
executions=entry["executions"],
|
||||
errors=entry["errors"],
|
||||
messages=entry["messages"],
|
||||
avg_ms=round(entry["duration_sum_ms"] / (entry["executions"] or 1), 2),
|
||||
avg_lag_ms=round(entry["lag_sum_ms"] / (entry["items"] or 1), 2),
|
||||
spark=entry["spark"],
|
||||
last_error_ts=(
|
||||
last_errors[flow].timestamp() if flow in last_errors else None
|
||||
),
|
||||
)
|
||||
for flow, entry in sorted(rollups.items())
|
||||
]
|
||||
|
||||
|
||||
@router.get("/runs", response_model=RunPage)
|
||||
def read_runs(
|
||||
session: SessionDep,
|
||||
flow: str | None = None,
|
||||
status: str | None = None,
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
limit: int = 50,
|
||||
) -> Any:
|
||||
"""Recent cascades, newest first, and how many there were in total.
|
||||
|
||||
``since`` is inclusive and ``until`` exclusive, so a window of one minute
|
||||
holds exactly the runs of the minute bucket the charts are drawn from.
|
||||
"""
|
||||
filters: list[ColumnElement[bool]] = []
|
||||
if flow:
|
||||
filters.append(col(FlowRun.flow) == flow)
|
||||
if status:
|
||||
filters.append(col(FlowRun.status) == status)
|
||||
if since:
|
||||
filters.append(col(FlowRun.started_at) >= _aware(since))
|
||||
if until:
|
||||
filters.append(col(FlowRun.started_at) < _aware(until))
|
||||
|
||||
capped = min(limit, 200)
|
||||
statement = select(FlowRun).where(*filters).order_by(col(FlowRun.started_at).desc())
|
||||
rows = list(session.exec(statement.limit(capped)))
|
||||
# A short page is its own total. The lists poll their whole range every
|
||||
# thirty seconds, and counting on each of those would hand back what
|
||||
# binning the metrics just saved — for a number that only ever says
|
||||
# "there is more here than fits".
|
||||
count = (
|
||||
len(rows)
|
||||
if len(rows) < capped
|
||||
else int(
|
||||
session.exec(
|
||||
select(func.count()).select_from(FlowRun).where(*filters)
|
||||
).one()
|
||||
)
|
||||
)
|
||||
return {"data": rows, "count": count}
|
||||
|
||||
|
||||
@router.get("/events", response_model=list[EventRow])
|
||||
def read_events(
|
||||
session: SessionDep,
|
||||
kind: Literal["failure", "audit"] = "failure",
|
||||
flow: str | None = None,
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
limit: int = 100,
|
||||
) -> Any:
|
||||
"""What went wrong, or who changed what. Newest first.
|
||||
|
||||
``since`` is inclusive and ``until`` exclusive, the same window ``/runs``
|
||||
takes, so a list can cover the span the charts beside it are drawn from.
|
||||
"""
|
||||
statement = select(EngineEvent).order_by(col(EngineEvent.ts).desc())
|
||||
if kind == "audit":
|
||||
statement = statement.where(col(EngineEvent.type) == "audit")
|
||||
else:
|
||||
statement = statement.where(col(EngineEvent.type) != "audit")
|
||||
if flow:
|
||||
statement = statement.where(col(EngineEvent.flow) == flow)
|
||||
if since:
|
||||
statement = statement.where(col(EngineEvent.ts) >= _aware(since))
|
||||
if until:
|
||||
statement = statement.where(col(EngineEvent.ts) < _aware(until))
|
||||
return list(session.exec(statement.limit(min(limit, 500))))
|
||||
|
||||
|
||||
@router.get("/dead-letter", response_model=list[DeadLetter])
|
||||
async def read_dead_letters(controller: FlowControllerDep, limit: int = 50) -> Any:
|
||||
"""Work the engine gave up on, which nothing else surfaces."""
|
||||
if controller.execution is None:
|
||||
return []
|
||||
return await run_in_threadpool(controller.execution.queue.dead_letters, limit)
|
||||
Reference in New Issue
Block a user