Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep the engine's own history, and a screen that reads it

Browse Source 
A second bus subscriber folds executions, errors, timings and queue lag
into per-minute rollups, keeps failures with their traceback and an audit
trail of who published what, and records one row per cascade — manual runs
and previews included, under an id of their own that writes no idempotency
markers. Read back through /observability/*, which always answers 200 so a
degraded engine still renders its own health screen.

Also fixes two things found on the way: node-health alerts read `status`
where the engine publishes `health`, so a device dropping never alerted
anyone, and the Redis queue reported `parked: 0` whatever was held.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt
This commit is contained in:
stroblme
2026-08-16 22:29:32 +02:00
co-authored by Claude Fable 5
parent f300c43f3a
commit af3ba51571
30 changed files with 2610 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/alembic/versions/087c44e16304_add_observability_rollups_events_and_.py
+77
View File
@@ -0,0 +1,77 @@
"""Add observability rollups, events and runs
Revision ID: 087c44e16304
Revises: 59e2606ce144
Create Date: 2026-08-16 19:55:10.772691
"""
from alembic import op
import sqlalchemy as sa
import sqlmodel.sql.sqltypes
# revision identifiers, used by Alembic.
revision = '087c44e16304'
down_revision = '59e2606ce144'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('engine_event',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('ts', sa.DateTime(timezone=True), nullable=False),
sa.Column('type', sqlmodel.sql.sqltypes.AutoString(length=32), nullable=False),
sa.Column('flow', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('node', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('detail', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('actor', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_engine_event_ts'), 'engine_event', ['ts'], unique=False)
op.create_index(op.f('ix_engine_event_type'), 'engine_event', ['type'], unique=False)
op.create_table('flow_run',
sa.Column('id', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False),
sa.Column('flow', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False),
sa.Column('source', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('started_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('nodes', sa.Integer(), nullable=False),
sa.Column('errors', sa.Integer(), nullable=False),
sa.Column('duration_ms', sa.Float(), nullable=False),
sa.Column('deliveries', sa.Integer(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_flow_run_flow'), 'flow_run', ['flow'], unique=False)
op.create_index(op.f('ix_flow_run_started_at'), 'flow_run', ['started_at'], unique=False)
op.create_table('metric_minute',
sa.Column('flow', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False),
sa.Column('node', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False),
sa.Column('bucket', sa.DateTime(timezone=True), nullable=False),
sa.Column('executions', sa.Integer(), nullable=False),
sa.Column('errors', sa.Integer(), nullable=False),
sa.Column('messages', sa.Integer(), nullable=False),
sa.Column('duration_sum_ms', sa.Float(), nullable=False),
sa.Column('duration_max_ms', sa.Float(), nullable=False),
sa.Column('lag_sum_ms', sa.Float(), nullable=False),
sa.Column('lag_max_ms', sa.Float(), nullable=False),
sa.Column('items', sa.Integer(), nullable=False),
sa.PrimaryKeyConstraint('flow', 'node', 'bucket')
)
op.create_index(op.f('ix_metric_minute_bucket'), 'metric_minute', ['bucket'], unique=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_metric_minute_bucket'), table_name='metric_minute')
op.drop_table('metric_minute')
op.drop_index(op.f('ix_flow_run_started_at'), table_name='flow_run')
op.drop_index(op.f('ix_flow_run_flow'), table_name='flow_run')
op.drop_table('flow_run')
op.drop_index(op.f('ix_engine_event_type'), table_name='engine_event')
op.drop_index(op.f('ix_engine_event_ts'), table_name='engine_event')
op.drop_table('engine_event')
# ### end Alembic commands ###
backend/app/api/main.py
+2
View File
@@ -8,6 +8,7 @@ from app.api.routes import (
messages,
modules,
oauth,
observability,
private,
secrets,
users,
@@ -25,6 +26,7 @@ api_router.include_router(alerts.router)
api_router.include_router(dashboards.router)
api_router.include_router(messages.router)
api_router.include_router(modules.router)
api_router.include_router(observability.router)
# Always mounted so the generated SDK stays the same shape; the endpoints
# themselves refuse to work unless MCP is switched on.
api_router.include_router(oauth.router)
backend/app/api/routes/flows.py
+32 -4
View File
@@ -1,6 +1,7 @@
"""The flow API. Everything the editor can do is available here first."""
import asyncio
import time
from typing import Any
from fastapi import (
@@ -14,7 +15,12 @@ from fastapi.concurrency import run_in_threadpool
from pydantic import BaseModel
from sqlmodel import Session
from app.api.deps import FlowControllerDep, get_current_user, user_from_token
from app.api.deps import (
CurrentUser,
FlowControllerDep,
get_current_user,
user_from_token,
)
from app.core.db import engine
from app.flow.controller import FlowController
from app.flow.dashboards import DashboardStore
@@ -184,6 +190,19 @@ def _read_flow(controller: FlowController, name: str) -> FlowDef:
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
def _audit(action: str, flow: str, user: CurrentUser) -> None:
"""Record who changed what. The collector writes it down; the bus carries it."""
event_bus.publish(
{
"type": "audit",
"action": action,
"flow": flow,
"user": user.email,
"ts": time.time(),
}
)
def _source_ref(definition: FlowDef, node_id: str) -> str | None:
"""The library source this node runs, if it is a shared one."""
node = next((n for n in definition.nodes if n.id == node_id), None)
@@ -323,6 +342,7 @@ async def publish_flow(
name: str,
body: PublishRequest,
controller: FlowControllerDep,
user: CurrentUser,
) -> Any:
"""Deploy the unpublished changes: the engine picks them up from here."""
_read_flow(controller, name)
@@ -339,6 +359,7 @@ async def publish_flow(
status_code=409,
detail={"message": str(exc), "current_version": exc.current},
)
_audit("published", name, user)
await controller.reload()
return _detail(controller, published)
@@ -364,12 +385,15 @@ async def discard_draft(name: str, controller: FlowControllerDep) -> Any:
@router.delete("/{name}", response_model=Message)
async def delete_flow(name: str, controller: FlowControllerDep) -> Any:
async def delete_flow(
name: str, controller: FlowControllerDep, user: CurrentUser
) -> Any:
"""Delete a flow and everything in it."""
try:
await run_in_threadpool(controller.store.delete_flow, name)
except FlowNotFound:
raise HTTPException(status_code=404, detail=f"No flow named '{name}'")
_audit("deleted", name, user)
# Its files are gone; its values and queued work would otherwise linger.
await run_in_threadpool(controller.forget_flow, name)
await controller.reload()
@@ -510,18 +534,22 @@ async def unshare_node(
@router.post("/{name}/start", response_model=FlowDetail)
async def start_flow(name: str, controller: FlowControllerDep) -> Any:
async def start_flow(
name: str, controller: FlowControllerDep, user: CurrentUser
) -> Any:
"""Let the engine run this flow again."""
_read_flow(controller, name)
await controller.set_enabled(name, True)
_audit("started", name, user)
return _detail(controller, _read_flow(controller, name))
@router.post("/{name}/stop", response_model=FlowDetail)
async def stop_flow(name: str, controller: FlowControllerDep) -> Any:
async def stop_flow(name: str, controller: FlowControllerDep, user: CurrentUser) -> Any:
"""Take this flow off the engine: no subscriptions, schedules or webhooks."""
_read_flow(controller, name)
await controller.set_enabled(name, False)
_audit("stopped", name, user)
return _detail(controller, _read_flow(controller, name))
backend/app/api/routes/modules.py
+21 -2
View File
@@ -6,13 +6,20 @@ the worker pool retires its processes, and the next node call picks up the new
packages.
"""
import time
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from fastapi.concurrency import run_in_threadpool
from app.api.deps import FlowControllerDep, WorkerPoolDep, get_current_user
from app.api.deps import (
CurrentUser,
FlowControllerDep,
WorkerPoolDep,
get_current_user,
)
from app.flow import modules
from app.flow.events import event_bus
from app.flow.schemas import ApplyRequest, ApplyResult, ModulesInfo
router = APIRouter(
@@ -28,7 +35,10 @@ async def read_modules(controller: FlowControllerDep) -> Any:
@router.post("/apply", response_model=ApplyResult)
async def apply_modules(
body: ApplyRequest, controller: FlowControllerDep, pool: WorkerPoolDep
body: ApplyRequest,
controller: FlowControllerDep,
pool: WorkerPoolDep,
user: CurrentUser,
) -> Any:
"""Install exactly these requirements, then hand them to the workers.
@@ -42,6 +52,15 @@ async def apply_modules(
detail=output or "These requirements could not be installed",
)
await run_in_threadpool(controller.store.write_requirements, body.requirements)
event_bus.publish(
{
"type": "audit",
"action": "installed modules",
"flow": "",
"user": user.email,
"ts": time.time(),
}
)
# Retire the workers first, so the rebuild compiles every node against the
# packages that were just installed — a node that could not import one is
# the reason this was called, and it stays red until it is built again.
backend/app/api/routes/observability.py
+310
View File
@@ -0,0 +1,310 @@
"""What the engine has been doing: health now, and history since.
The live state comes from the controller; everything older than the websocket's
memory comes from the rollups the metrics collector writes. Deliberately not
built on ``/utils/health/``: that endpoint answers 503 when something is wrong,
which the generated SDK turns into a thrown error — and a health page that
cannot render while the engine is degraded is the wrong way round.
"""
from datetime import datetime, timedelta, timezone
from typing import Any, Literal
from fastapi import APIRouter, Depends, Request
from fastapi.concurrency import run_in_threadpool
from pydantic import BaseModel
from sqlalchemy import func
from sqlmodel import col, select
from app.api.deps import FlowControllerDep, SessionDep, get_current_user
from app.flow.controller import NodeStatus
from app.models import EngineEvent, FlowRun, MetricBucket
router = APIRouter(
prefix="/observability",
tags=["observability"],
dependencies=[Depends(get_current_user)],
)
#: How many slices a per-flow sparkline is folded into.
SPARK_SLICES = 60
class HealthSummary(BaseModel):
status: str
problems: list[str]
flows: dict[str, int]
nodes: dict[str, int]
queue: dict[str, Any]
loop_lag: dict[str, float]
failures_24h: int
class SeriesPoint(BaseModel):
ts: float
executions: int
errors: int
messages: int
avg_ms: float
max_ms: float
avg_lag_ms: float
class FlowRollup(BaseModel):
flow: str
executions: int
errors: int
messages: int
avg_ms: float
avg_lag_ms: float
spark: list[int]
last_error_ts: float | None = None
class RunRow(BaseModel):
id: str
flow: str
source: str
status: str
started_at: datetime
finished_at: datetime | None = None
nodes: int
errors: int
duration_ms: float
deliveries: int
class EventRow(BaseModel):
id: int
ts: datetime
type: str
flow: str
node: str
detail: str
actor: str
class DeadLetter(BaseModel):
id: str
ts: float
flow: str
node: str
cause: str
reason: str
def _since(hours: int) -> datetime:
return datetime.now(timezone.utc) - timedelta(hours=hours)
@router.get("/summary", response_model=HealthSummary)
async def read_summary(
request: Request, controller: FlowControllerDep, session: SessionDep
) -> Any:
"""How the engine is doing right now. Always 200, degraded or not."""
watchdog = getattr(request.app.state, "watchdog", None)
problems: list[str] = []
if watchdog is not None and watchdog.degraded:
problems.append("event loop lagging")
queue = await run_in_threadpool(controller.queue_stats)
if queue.get("oldest_pending_s", 0) > 120:
problems.append("queue stalled")
if queue.get("error"):
problems.append("work queue unreachable")
names = await run_in_threadpool(controller.store.list_flows)
quarantined = controller.quarantined
paused = set(controller.paused_flows())
entries = list(controller.loaded.values())
errored = [e for e in entries if e.status is NodeStatus.ERROR]
if quarantined:
problems.append(f"{len(quarantined)} flow(s) quarantined")
if errored:
problems.append(f"{len(errored)} node(s) failed to load")
failures = session.exec(
select(func.count())
.select_from(EngineEvent)
.where(col(EngineEvent.ts) >= _since(24), col(EngineEvent.type) != "audit")
).one()
return HealthSummary(
status="degraded" if problems else "ok",
problems=problems,
flows={
"total": len(names),
"running": len(
[n for n in names if controller.is_enabled(n) and n not in quarantined]
),
"paused": len(paused),
"quarantined": len(quarantined),
},
nodes={"total": len(entries), "error": len(errored)},
queue=queue,
loop_lag=(
watchdog.snapshot()
if watchdog is not None
else {"ewma": 0.0, "max_60s": 0.0}
),
failures_24h=int(failures),
)
@router.get("/timeseries", response_model=list[SeriesPoint])
def read_timeseries(
session: SessionDep,
flow: str | None = None,
node: str | None = None,
hours: int = 24,
bucket_s: int = 60,
) -> Any:
"""Executions, errors and timings over time, summed across nodes."""
# ponytail: the fold is in Python — a day is at most 1440 rows per node.
# date_bin() if the window ever grows past that.
statement = select(MetricBucket).where(col(MetricBucket.bucket) >= _since(hours))
if flow:
statement = statement.where(col(MetricBucket.flow) == flow)
if node:
statement = statement.where(col(MetricBucket.node) == node)
slices: dict[float, dict[str, float]] = {}
for row in session.exec(statement.order_by(col(MetricBucket.bucket))):
stamp = row.bucket.timestamp()
key = stamp - stamp % max(60, bucket_s)
point = slices.setdefault(
key,
{
"executions": 0.0,
"errors": 0.0,
"messages": 0.0,
"duration_sum_ms": 0.0,
"max_ms": 0.0,
"lag_sum_ms": 0.0,
"items": 0.0,
},
)
point["executions"] += row.executions
point["errors"] += row.errors
point["messages"] += row.messages
point["duration_sum_ms"] += row.duration_sum_ms
point["max_ms"] = max(point["max_ms"], row.duration_max_ms)
point["lag_sum_ms"] += row.lag_sum_ms
point["items"] += row.items
return [
SeriesPoint(
ts=ts,
executions=int(point["executions"]),
errors=int(point["errors"]),
messages=int(point["messages"]),
avg_ms=round(point["duration_sum_ms"] / (point["executions"] or 1), 2),
max_ms=round(point["max_ms"], 2),
avg_lag_ms=round(point["lag_sum_ms"] / (point["items"] or 1), 2),
)
for ts, point in sorted(slices.items())
]
@router.get("/flows", response_model=list[FlowRollup])
def read_flow_rollups(session: SessionDep, hours: int = 24) -> Any:
"""One row per flow, with a coarse trend of how much it ran."""
since = _since(hours)
window = hours * 3600
start = since.timestamp()
rollups: dict[str, dict[str, Any]] = {}
for row in session.exec(
select(MetricBucket).where(col(MetricBucket.bucket) >= since)
):
entry = rollups.setdefault(
row.flow,
{
"executions": 0,
"errors": 0,
"messages": 0,
"duration_sum_ms": 0.0,
"lag_sum_ms": 0.0,
"items": 0,
"spark": [0] * SPARK_SLICES,
},
)
entry["executions"] += row.executions
entry["errors"] += row.errors
entry["messages"] += row.messages
entry["duration_sum_ms"] += row.duration_sum_ms
entry["lag_sum_ms"] += row.lag_sum_ms
entry["items"] += row.items
slot = min(
SPARK_SLICES - 1,
max(0, int((row.bucket.timestamp() - start) / window * SPARK_SLICES)),
)
entry["spark"][slot] += row.executions
# `.all()` first: a Result has `keys()`, so dict() would read it as a
# mapping and subscript it.
last_errors = dict(
session.exec(
select(col(EngineEvent.flow), func.max(col(EngineEvent.ts)))
.where(col(EngineEvent.type) == "node_error", col(EngineEvent.ts) >= since)
.group_by(col(EngineEvent.flow))
).all()
)
return [
FlowRollup(
flow=flow,
executions=entry["executions"],
errors=entry["errors"],
messages=entry["messages"],
avg_ms=round(entry["duration_sum_ms"] / (entry["executions"] or 1), 2),
avg_lag_ms=round(entry["lag_sum_ms"] / (entry["items"] or 1), 2),
spark=entry["spark"],
last_error_ts=(
last_errors[flow].timestamp() if flow in last_errors else None
),
)
for flow, entry in sorted(rollups.items())
]
@router.get("/runs", response_model=list[RunRow])
def read_runs(
session: SessionDep,
flow: str | None = None,
status: str | None = None,
limit: int = 50,
) -> Any:
"""Recent cascades, newest first."""
statement = select(FlowRun).order_by(col(FlowRun.started_at).desc())
if flow:
statement = statement.where(col(FlowRun.flow) == flow)
if status:
statement = statement.where(col(FlowRun.status) == status)
return list(session.exec(statement.limit(min(limit, 200))))
@router.get("/events", response_model=list[EventRow])
def read_events(
session: SessionDep,
kind: Literal["failure", "audit"] = "failure",
flow: str | None = None,
limit: int = 100,
) -> Any:
"""What went wrong, or who changed what. Newest first."""
statement = select(EngineEvent).order_by(col(EngineEvent.ts).desc())
if kind == "audit":
statement = statement.where(col(EngineEvent.type) == "audit")
else:
statement = statement.where(col(EngineEvent.type) != "audit")
if flow:
statement = statement.where(col(EngineEvent.flow) == flow)
return list(session.exec(statement.limit(min(limit, 500))))
@router.get("/dead-letter", response_model=list[DeadLetter])
async def read_dead_letters(controller: FlowControllerDep, limit: int = 50) -> Any:
"""Work the engine gave up on, which nothing else surfaces."""
if controller.execution is None:
return []
return await run_in_threadpool(controller.execution.queue.dead_letters, limit)
backend/app/core/config.py
+2
View File
@@ -63,6 +63,8 @@ class Settings(BaseSettings):
# node sets its own. Long enough for a slow HTTP call, short enough that a
# runaway loop is not a wedged flow.
FLOW_NODE_TIMEOUT: float = 30.0
# How long the engine's own metrics, events and run records are kept.
OBS_RETENTION_DAYS: int = 30
# Without a Redis host the engine keeps its state in memory.
REDIS_HOST: str | None = None
REDIS_PORT: int = 6379
backend/app/flow/alerts.py
+2 -2
View File
@@ -102,7 +102,7 @@ def describe(event: dict[str, Any]) -> Alert | None:
node=node,
)
if kind == "node_health":
if event.get("status") != "down":
if event.get("health") != "down":
return None
return Alert(
title=f"{node or 'A node'} lost its connection",
@@ -253,7 +253,7 @@ class AlertManager:
if muted_until is not None:
if now < muted_until:
# Still flapping — push the window out and stay quiet.
if event.get("status") == "down":
if event.get("health") == "down":
self._flapping[key] = now + FLAP_WINDOW_S
return True
del self._flapping[key]
backend/app/flow/executor.py
+38
View File
@@ -260,6 +260,8 @@ class ExecutionService:
if item.kind == "flush":
# A rate-limit window ended; nothing to replay, only to let out.
# ponytail: no run record for a flush — it is the tail of the run
# that scheduled it, not a run of its own.
pipeline.flush(node)
return True
@@ -268,10 +270,46 @@ class ExecutionService:
logger.debug("Guard no longer holds for '%s', dropped", item.node)
return True
# Only here is the item certain to run, which is what a run record is.
now = time.time()
self._publish(
{
"type": "cascade_started",
"run": item.entry_id,
"flow": item.flow,
"node": item.node,
"cause": item.cause,
"deliveries": item.deliveries,
"ts": now,
}
)
if item.deliveries == 1:
# A redelivery waited for the reaper, not for the engine.
self._publish(
{
"type": "work_latency",
"flow": item.flow,
"node": item.node,
"lag_ms": max(
0.0,
(now - max(item.enqueued_at, item.not_before)) * 1000,
),
"ts": now,
}
)
pipeline.apply_outputs(node, item.outputs or None)
pipeline.run_downstream(
node, entry_id=item.entry_id, replay=item.deliveries > 1
)
self._publish(
{
"type": "cascade_finished",
"run": item.entry_id,
"flow": item.flow,
"ts": time.time(),
}
)
return True
# -------------------------------------------------------------------------
backend/app/flow/metrics.py
+356
View File
@@ -0,0 +1,356 @@
"""What the engine did, kept long enough to answer for it.
The event bus already carries every execution, error and cascade; until now
nothing wrote any of it down, so "was it slow yesterday?" had no answer. This
subscriber folds those events into per-minute rollups, keeps the failures and
the audit trail whole, and records one row per cascade.
Accumulation is in memory and flushed every few seconds: a node firing at
10 Hz must not be 10 inserts a second, and the arithmetic that turns it into
one row a minute is cheaper than the round trip would be.
"""
from __future__ import annotations
import asyncio
import logging
import time
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import delete, func, update
from sqlalchemy.dialects.postgresql import insert
from sqlmodel import Session, col
from app.core.config import settings
from app.core.db import engine
from app.flow.events import EventBus
from app.models import EngineEvent, FlowRun, MetricBucket
logger = logging.getLogger(__name__)
#: How often the accumulated minute is written out.
FLUSH_INTERVAL_S = 15.0
#: A traceback is worth reading; a whole run of a chatty node is not.
DETAIL_CAP = 8000
#: A cascade still open this long after it started is never finishing.
RUN_STALE_S = 600.0
#: Retention is checked this often, not on every flush.
PRUNE_INTERVAL_S = 3600.0
#: Bucket columns that add up over a minute, and the two that take the larger.
SUMMED = (
"executions",
"errors",
"messages",
"duration_sum_ms",
"lag_sum_ms",
"items",
)
MAXIMA = ("duration_max_ms", "lag_max_ms")
#: Engine events kept as rows. Everything else on the bus is traffic.
RECORDED = {
"flow_quarantined",
"task_crashed",
"engine_degraded",
"engine_fatal",
"cascade_dropped",
"queue_unavailable",
}
def _minute(ts: float) -> datetime:
return datetime.fromtimestamp(ts, timezone.utc).replace(second=0, microsecond=0)
def _detail(event: dict[str, Any]) -> str:
text = str(event.get("error") or event.get("reason") or event.get("detail") or "")
if event.get("type") == "cascade_dropped":
text = f"Given up on after {event.get('deliveries')} deliveries. {text}"
return text[:DETAIL_CAP]
class MetricsCollector:
"""Folds engine events into rollups, failures and run records."""
def __init__(self, events: EventBus, flush_s: float = FLUSH_INTERVAL_S) -> None:
self._events = events
self._flush_s = flush_s
self._buckets: dict[tuple[str, str, datetime], dict[str, float]] = {}
self._runs: dict[str, dict[str, Any]] = {}
self._pending: list[EngineEvent] = []
# The traceback arrives one event before the failure it belongs to.
self._tracebacks: dict[tuple[str, str], str] = {}
self._last_prune = 0.0
# -------------------------------------------------------------------------
# The loop
# -------------------------------------------------------------------------
async def run(self) -> None:
"""Consume the bus until cancelled, flushing on a fixed interval."""
last = time.monotonic()
async with self._events.subscribe() as queue:
while True:
# A busy bus never idles, so the flush is on a deadline rather
# than on the timeout alone.
timeout = max(0.05, self._flush_s - (time.monotonic() - last))
try:
event = await asyncio.wait_for(queue.get(), timeout)
except asyncio.TimeoutError:
pass
else:
try:
self.handle(event)
except Exception:
logger.exception("Could not record %s", event.get("type"))
if time.monotonic() - last >= self._flush_s:
await self.flush()
last = time.monotonic()
# -------------------------------------------------------------------------
# Accumulating
# -------------------------------------------------------------------------
def _bucket(self, event: dict[str, Any]) -> dict[str, float]:
# ponytail: one collector, one row per node per minute; coarsen the
# bucket if the node count ever reaches thousands.
key = (
str(event.get("flow") or ""),
str(event.get("node") or ""),
_minute(float(event.get("ts") or time.time())),
)
return self._buckets.setdefault(
key,
{
"executions": 0,
"errors": 0,
"messages": 0,
"duration_sum_ms": 0.0,
"duration_max_ms": 0.0,
"lag_sum_ms": 0.0,
"lag_max_ms": 0.0,
"items": 0,
},
)
def handle(self, event: dict[str, Any]) -> None:
"""Fold one event in. Synchronous: this is arithmetic on dicts."""
kind = str(event.get("type") or "")
ts = float(event.get("ts") or time.time())
run = self._runs.get(str(event.get("run") or ""))
if kind == "node_executed":
bucket = self._bucket(event)
bucket["executions"] += 1
bucket["messages"] += int(event.get("outputs") or 0)
duration = float(event.get("duration_ms") or 0.0)
bucket["duration_sum_ms"] += duration
bucket["duration_max_ms"] = max(bucket["duration_max_ms"], duration)
if run is not None:
run["nodes"] += 1
return
if kind == "work_latency":
bucket = self._bucket(event)
lag = float(event.get("lag_ms") or 0.0)
bucket["lag_sum_ms"] += lag
bucket["lag_max_ms"] = max(bucket["lag_max_ms"], lag)
bucket["items"] += 1
return
if kind == "node_log":
# Held for the node_error that follows it from the same thread.
if event.get("level") == "error":
key = (str(event.get("flow") or ""), str(event.get("node") or ""))
self._tracebacks[key] = str(event.get("text") or "")
return
if kind == "node_error":
self._bucket(event)["errors"] += 1
if run is not None:
run["errors"] += 1
key = (str(event.get("flow") or ""), str(event.get("node") or ""))
traceback = self._tracebacks.pop(key, "")
error = str(event.get("error") or "")
self._pending.append(
EngineEvent(
ts=datetime.fromtimestamp(ts, timezone.utc),
type="node_error",
flow=str(event.get("flow") or ""),
node=str(event.get("node") or ""),
detail=(f"{error}\n{traceback}" if traceback else error)[
:DETAIL_CAP
],
)
)
return
if kind == "cascade_started":
self._start_run(event, ts)
return
if kind == "cascade_finished":
if run is not None:
run["finished_at"] = datetime.fromtimestamp(ts, timezone.utc)
run["duration_ms"] = round((ts - run["started_ts"]) * 1000, 2)
run["status"] = "error" if run["errors"] else "ok"
return
if kind == "node_health":
if event.get("health") == "down":
self._pending.append(
EngineEvent(
ts=datetime.fromtimestamp(ts, timezone.utc),
type="node_health",
flow=str(event.get("flow") or ""),
node=str(event.get("node") or ""),
detail=_detail(event) or "Reported itself down.",
)
)
return
if kind == "audit":
self._pending.append(
EngineEvent(
ts=datetime.fromtimestamp(ts, timezone.utc),
type="audit",
flow=str(event.get("flow") or ""),
detail=str(event.get("action") or ""),
actor=str(event.get("user") or ""),
)
)
return
if kind in RECORDED:
self._pending.append(
EngineEvent(
ts=datetime.fromtimestamp(ts, timezone.utc),
type=kind,
flow=str(event.get("flow") or ""),
node=str(event.get("node") or event.get("task") or ""),
detail=_detail(event),
)
)
def _start_run(self, event: dict[str, Any], ts: float) -> None:
run_id = str(event.get("run") or "")
if not run_id:
return
existing = self._runs.get(run_id)
if existing is not None:
# A redelivery of the same item: one run, tried again.
existing["deliveries"] = int(event.get("deliveries") or 1)
existing["status"] = "running"
existing["finished_at"] = None
return
self._runs[run_id] = {
"id": run_id,
"flow": str(event.get("flow") or ""),
"source": str(event.get("cause") or ""),
"started_ts": ts,
"started_at": datetime.fromtimestamp(ts, timezone.utc),
"finished_at": None,
"status": "running",
"nodes": 0,
"errors": 0,
"duration_ms": 0.0,
"deliveries": int(event.get("deliveries") or 1),
}
# -------------------------------------------------------------------------
# Writing
# -------------------------------------------------------------------------
async def flush(self) -> None:
buckets, self._buckets = self._buckets, {}
pending, self._pending = self._pending, []
# Open runs stay in memory: their counts are still growing, and the row
# is written from the whole record each time rather than in deltas.
runs = list(self._runs.values())
prune = time.monotonic() - self._last_prune >= PRUNE_INTERVAL_S
if not (buckets or pending or runs or prune):
return
try:
await asyncio.to_thread(self._write, buckets, pending, runs, prune)
except Exception:
logger.exception("Could not write engine metrics")
return
if prune:
self._last_prune = time.monotonic()
cutoff = time.time() - RUN_STALE_S
for run_id, run in list(self._runs.items()):
if run["status"] != "running" or run["started_ts"] < cutoff:
del self._runs[run_id]
# Held tracebacks survive the flush: the log and the failure it belongs
# to are two events, and a flush can fall between them. One per node,
# each replaced by that node's next failure.
def _write(
self,
buckets: dict[tuple[str, str, datetime], dict[str, float]],
pending: list[EngineEvent],
runs: list[dict[str, Any]],
prune: bool,
) -> None:
with Session(engine) as session:
for (flow, node, minute), agg in buckets.items():
statement = insert(MetricBucket).values(
flow=flow, node=node, bucket=minute, **agg
)
# The same minute is written several times, so the counters add
# and the maxima take whichever is larger. Columns are read by
# subscript: `excluded.items` is the collection's own method.
new = statement.excluded
session.execute(
statement.on_conflict_do_update(
index_elements=["flow", "node", "bucket"],
set_={
name: col(getattr(MetricBucket, name)) + new[name]
for name in SUMMED
}
| {
name: func.greatest(
col(getattr(MetricBucket, name)), new[name]
)
for name in MAXIMA
},
)
)
for run in runs:
values = {k: v for k, v in run.items() if k != "started_ts"}
statement = insert(FlowRun).values(**values)
session.execute(
statement.on_conflict_do_update(
index_elements=["id"],
set_={
key: statement.excluded[key]
for key in values
if key != "id"
},
)
)
session.add_all(pending)
if prune:
self._prune(session)
session.commit()
def _prune(self, session: Session) -> None:
now = datetime.now(timezone.utc)
cutoff = now - timedelta(days=settings.OBS_RETENTION_DAYS)
session.execute(delete(MetricBucket).where(col(MetricBucket.bucket) < cutoff))
session.execute(delete(EngineEvent).where(col(EngineEvent.ts) < cutoff))
session.execute(delete(FlowRun).where(col(FlowRun.started_at) < cutoff))
# A run still open long after it started did not finish; saying so is
# more honest than leaving it running forever.
session.execute(
update(FlowRun)
.where(
col(FlowRun.status) == "running",
col(FlowRun.started_at) < now - timedelta(seconds=RUN_STALE_S),
)
.values(status="abandoned", finished_at=now)
)
backend/app/flow/pipeline.py
+49 -5
View File
@@ -11,6 +11,7 @@ from __future__ import annotations
import logging
import threading
import time
import uuid
from collections import deque
from collections.abc import Iterator
from concurrent.futures import Future, ThreadPoolExecutor, wait
@@ -28,6 +29,10 @@ from app.flow.state import MemoryState, StateBackend
logger = logging.getLogger(__name__)
#: A run that never went through the queue. It still gets a run record, so a
#: manual run shows up in the history — but it is no one's idempotency key.
MANUAL_RUN_PREFIX = "manual-"
class ValidationIssue(BaseModel):
"""A problem that keeps a flow from running correctly."""
@@ -589,9 +594,15 @@ class Pipeline:
result = node.execute(inputs)
self.publish_log(node, collected, "")
if entry_id and not node.idempotent and self._queue is not None:
if (
entry_id
and not entry_id.startswith(MANUAL_RUN_PREFIX)
and not node.idempotent
and self._queue is not None
):
# Written after the fact: a crash between the side effect and
# this marker is the one window at-least-once cannot close.
# this marker is the one window at-least-once cannot close. A
# manual run has nothing to be redelivered, so it writes none.
try:
self._queue.mark_done(entry_id, node.id)
except Exception as exc:
@@ -632,6 +643,7 @@ class Pipeline:
# which is a different thing to show than one that emitted.
"outputs": len(result or {}),
"duration_ms": round((time.perf_counter() - started) * 1000, 2),
"run": entry_id,
"ts": time.time(),
}
)
@@ -648,6 +660,7 @@ class Pipeline:
"flow": node.flow,
"node": node.id,
"error": f"{type(exc).__name__}: {exc}",
"run": entry_id,
"ts": time.time(),
}
)
@@ -896,8 +909,40 @@ class Pipeline:
self._enqueue_cascade(node, outputs)
return state
return self._run_here(node, outputs)
def _run_here(
self, node: Node, outputs: dict[str, Any] | None, cause: str = "manual"
) -> StateBackend:
"""Run a cascade in this thread, under a run id of its own.
The queued path gets its run id from the journal entry. A run that never
went through the queue still belongs in the history, so it makes one —
marked as such, because it is no one's idempotency key.
"""
run_id = f"{MANUAL_RUN_PREFIX}{uuid.uuid4().hex[:12]}"
self._publish(
{
"type": "cascade_started",
"run": run_id,
"flow": node.flow,
"node": node.id,
"cause": cause,
"deliveries": 1,
"ts": time.time(),
}
)
self.apply_outputs(node, outputs)
return self.run_downstream(node)
state = self.run_downstream(node, entry_id=run_id)
self._publish(
{
"type": "cascade_finished",
"run": run_id,
"flow": node.flow,
"ts": time.time(),
}
)
return state
def publish(
self, values: dict[str, Any], source: ValueSource | None = None
@@ -1008,8 +1053,7 @@ class Pipeline:
}
)
# Losing the value outright would be worse than running it here.
self.apply_outputs(node, outputs)
self.run_downstream(node)
self._run_here(node, outputs, cause="external")
def values(self, flow: str | None = None) -> dict[str, dict[str, Any]]:
"""Last value and timestamp of every message, optionally one flow's."""
backend/app/flow/queue.py
+36 -1
View File
@@ -53,6 +53,8 @@ class WorkItem:
:param entry_id: Set by the queue on claim; stable across redeliveries,
which is what makes it usable as an idempotency key.
:param deliveries: How many times this item has been handed out.
:param enqueued_at: When the item was made. Carried rather than read off
the entry id, because only the Redis queue has timestamps in its ids.
"""
kind: str
@@ -65,6 +67,7 @@ class WorkItem:
guard_value: str = ""
entry_id: str = ""
deliveries: int = 1
enqueued_at: float = field(default_factory=time.time)
def to_fields(self) -> dict[str, str]:
return {
@@ -76,6 +79,7 @@ class WorkItem:
"not_before": str(self.not_before),
"guard_key": self.guard_key,
"guard_value": self.guard_value,
"enqueued_at": str(self.enqueued_at),
}
@classmethod
@@ -93,6 +97,7 @@ class WorkItem:
guard_value=fields.get("guard_value", ""),
entry_id=entry_id,
deliveries=deliveries,
enqueued_at=float(fields.get("enqueued_at") or time.time()),
)
@@ -147,6 +152,10 @@ class WorkQueue(ABC):
def stats(self) -> dict[str, Any]:
"""Queue depth and age, for the health endpoint."""
@abstractmethod
def dead_letters(self, count: int = 50) -> list[dict[str, Any]]:
"""What was given up on, newest first."""
@abstractmethod
def mark_done(self, entry_id: str, node: str) -> None:
"""Record that a side effect already happened for this delivery."""
@@ -251,6 +260,10 @@ class MemoryWorkQueue(WorkQueue):
"durable": False,
}
def dead_letters(self, count: int = 50) -> list[dict[str, Any]]:
"""Nothing is kept: a dropped item here died with the process."""
return []
def mark_done(self, entry_id: str, node: str) -> None:
with self._lock:
self._done.add((entry_id, node))
@@ -416,15 +429,37 @@ class RedisWorkQueue(WorkQueue):
)
if records:
oldest = records[0]["time_since_delivered"] / 1000.0
parked = sum(
cast(int, self._redis.llen(key))
for key in self._redis.scan_iter(f"{self._ns}:__parked__:*")
)
return {
"depth": cast(int, self._redis.xlen(self._stream)),
"pending": count,
"delayed": cast(int, self._redis.zcard(self._delayed_key)),
"parked": 0,
"parked": parked,
"oldest_pending_s": round(oldest, 1),
"durable": True,
}
def dead_letters(self, count: int = 50) -> list[dict[str, Any]]:
"""What was set aside, newest first. The entry id carries the time."""
entries = cast(
list[tuple[str, dict[str, str]]],
self._redis.xrevrange(self._dead_key, count=count),
)
return [
{
"id": entry_id,
"ts": int(entry_id.split("-")[0]) / 1000.0,
"flow": fields.get("flow", ""),
"node": fields.get("node", ""),
"cause": fields.get("cause", ""),
"reason": fields.get("reason", ""),
}
for entry_id, fields in entries
]
def mark_done(self, entry_id: str, node: str) -> None:
# An hour outlives any redelivery; after that the marker is noise.
self._redis.set(self._done_key(entry_id, node), "1", ex=3600)
backend/app/main.py
+5
View File
@@ -20,6 +20,7 @@ from app.flow.controller import FlowController
from app.flow.dashboards import DashboardStore
from app.flow.events import event_bus
from app.flow.executor import ExecutionService
from app.flow.metrics import MetricsCollector
from app.flow.nodes.http import close_shared_client
from app.flow.plugins import load_plugins
from app.flow.queue import MemoryWorkQueue, RedisWorkQueue, WorkQueue
@@ -107,6 +108,9 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
app.state.watchdog = watchdog
watchdog_task = asyncio.create_task(watchdog.run(), name="loop-watchdog")
alerts_task = asyncio.create_task(alerts.run(), name="alert-manager")
metrics_task = asyncio.create_task(
MetricsCollector(event_bus).run(), name="metrics-collector"
)
await controller.start()
try:
# A mounted sub-app gets no lifespan of its own, so the MCP session
@@ -116,6 +120,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
finally:
watchdog_task.cancel()
alerts_task.cancel()
metrics_task.cancel()
await controller.stop()
pool.stop()
close_shared_client()
backend/app/mcp/server.py
+42
View File
@@ -265,3 +265,45 @@ async def apply_modules(requirements: str) -> Any:
manifest that does not resolve changes nothing.
"""
return await _call("POST", "/modules/apply", json={"requirements": requirements})
# -----------------------------------------------------------------------------
# Observability
# -----------------------------------------------------------------------------
@mcp.tool()
async def get_health() -> Any:
"""How the engine is doing: flows, nodes, queue, loop lag and recent failures."""
return await _call("GET", "/observability/summary")
@mcp.tool()
async def get_metrics(
flow: str | None = None, node: str | None = None, hours: int = 24
) -> Any:
"""Executions, errors and timings per minute over the last few hours."""
params: dict[str, Any] = {"hours": hours}
if flow:
params["flow"] = flow
if node:
params["node"] = node
return await _call("GET", "/observability/timeseries", params=params)
@mcp.tool()
async def list_failures(flow: str | None = None, limit: int = 50) -> Any:
"""Recent failures with their tracebacks, newest first."""
params: dict[str, Any] = {"kind": "failure", "limit": limit}
if flow:
params["flow"] = flow
return await _call("GET", "/observability/events", params=params)
@mcp.tool()
async def list_runs(flow: str | None = None, limit: int = 50) -> Any:
"""Recent cascades: what triggered them, how long they took, how they ended."""
params: dict[str, Any] = {"limit": limit}
if flow:
params["flow"] = flow
return await _call("GET", "/observability/runs", params=params)
backend/app/models.py
+81
View File
@@ -198,3 +198,84 @@ class OAuthAuthorizeRequest(SQLModel):
class OAuthAuthorizeResponse(SQLModel):
redirect_url: str
# -----------------------------------------------------------------------------
# Observability
#
# The engine's own history: what ran, how long it took, and what went wrong.
# Rolled up per minute rather than kept per execution — a node firing every
# second is 86 400 rows a day raw, and nobody reads a row of that.
# -----------------------------------------------------------------------------
class MetricBucket(SQLModel, table=True):
"""One node's minute: how much ran, how long it took, how late it was."""
__tablename__ = "metric_minute"
flow: str = Field(primary_key=True, max_length=255)
node: str = Field(primary_key=True, max_length=255)
bucket: datetime = Field(
primary_key=True,
index=True,
sa_type=DateTime(timezone=True), # type: ignore
)
executions: int = 0
errors: int = 0
#: Messages emitted, which is not the same as executions: a node can run
#: and publish nothing.
messages: int = 0
duration_sum_ms: float = 0.0
duration_max_ms: float = 0.0
#: How long queued work waited before it ran.
lag_sum_ms: float = 0.0
lag_max_ms: float = 0.0
#: Items the lag sums are over, so an average can be taken.
items: int = 0
class EngineEvent(SQLModel, table=True):
"""Something worth keeping after the websocket has forgotten it."""
__tablename__ = "engine_event"
id: int | None = Field(default=None, primary_key=True)
ts: datetime = Field(
index=True,
sa_type=DateTime(timezone=True), # type: ignore
)
#: node_error, flow_quarantined, engine_degraded, …, or audit.
type: str = Field(max_length=32, index=True)
flow: str = ""
node: str = ""
#: The error and its traceback, or what an audit entry says was done.
detail: str = ""
#: Who did it, on audit rows.
actor: str = ""
class FlowRun(SQLModel, table=True):
"""One cascade, from the item that started it to the last node in it."""
__tablename__ = "flow_run"
#: The queue entry id, or a `manual-` one for a run that never queued.
id: str = Field(primary_key=True, max_length=64)
flow: str = Field(index=True, max_length=255)
#: What caused it: the work item's cause, or "manual".
source: str = ""
started_at: datetime = Field(
index=True,
sa_type=DateTime(timezone=True), # type: ignore
)
finished_at: datetime | None = Field(
default=None,
sa_type=DateTime(timezone=True), # type: ignore
)
#: running, ok, error or abandoned.
status: str = "running"
nodes: int = 0
errors: int = 0
duration_ms: float = 0.0
deliveries: int = 1
backend/tests/api/routes/test_observability.py
+104
View File
@@ -0,0 +1,104 @@
from datetime import datetime, timedelta, timezone
from fastapi.testclient import TestClient
from sqlmodel import Session
from app.core.config import settings
from app.models import EngineEvent, FlowRun, MetricBucket
PREFIX = f"{settings.API_V1_STR}/observability"
FLOW = "observability-test"
def _seed(db: Session) -> None:
now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
db.add(
MetricBucket(
flow=FLOW,
node=f"{FLOW}.calc",
bucket=now - timedelta(minutes=1),
executions=4,
errors=1,
messages=6,
duration_sum_ms=40.0,
duration_max_ms=25.0,
lag_sum_ms=100.0,
lag_max_ms=60.0,
items=4,
)
)
db.add(
EngineEvent(
ts=now,
type="node_error",
flow=FLOW,
node=f"{FLOW}.calc",
detail="ValueError: bad input\nTraceback",
)
)
db.add(
EngineEvent(
ts=now, type="audit", flow=FLOW, detail="published", actor="a@example.com"
)
)
db.add(
FlowRun(
id="9-0",
flow=FLOW,
source="external",
started_at=now,
finished_at=now,
status="ok",
nodes=3,
duration_ms=12.5,
)
)
db.commit()
def test_observability_requires_authentication(client: TestClient) -> None:
assert client.get(f"{PREFIX}/summary").status_code == 401
def test_the_summary_answers_even_when_degraded(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
response = client.get(f"{PREFIX}/summary", headers=superuser_token_headers)
assert response.status_code == 200
body = response.json()
assert body["status"] in {"ok", "degraded"}
assert set(body["flows"]) == {"total", "running", "paused", "quarantined"}
assert "error" in body["nodes"]
def test_the_history_reads_back(
client: TestClient, superuser_token_headers: dict[str, str], db: Session
) -> None:
_seed(db)
points = client.get(f"{PREFIX}/timeseries", headers=superuser_token_headers).json()
mine = [point for point in points if point["executions"]]
assert mine and mine[-1]["avg_ms"] > 0
flows = client.get(f"{PREFIX}/flows", headers=superuser_token_headers).json()
row = next(entry for entry in flows if entry["flow"] == FLOW)
assert (row["executions"], row["errors"], row["messages"]) == (4, 1, 6)
assert len(row["spark"]) == 60
assert row["last_error_ts"] is not None
runs = client.get(f"{PREFIX}/runs", headers=superuser_token_headers).json()
assert any(run["id"] == "9-0" and run["status"] == "ok" for run in runs)
failures = client.get(f"{PREFIX}/events", headers=superuser_token_headers).json()
assert any("Traceback" in event["detail"] for event in failures)
assert all(event["type"] != "audit" for event in failures)
audit = client.get(
f"{PREFIX}/events", headers=superuser_token_headers, params={"kind": "audit"}
).json()
assert any(event["actor"] == "a@example.com" for event in audit)
dead = client.get(f"{PREFIX}/dead-letter", headers=superuser_token_headers)
assert dead.status_code == 200
assert isinstance(dead.json(), list)
backend/tests/flow/test_alerts.py
+9 -3
View File
@@ -122,11 +122,11 @@ def test_a_flapping_connection_goes_quiet():
async def scenario():
for _ in range(FLAP_THRESHOLD * 2 + 6):
await alerts.handle(
{"type": "node_health", "node": "heating.pump", "status": "down"}
{"type": "node_health", "node": "heating.pump", "health": "down"}
)
clock.advance(5)
await alerts.handle(
{"type": "node_health", "node": "heating.pump", "status": "ok"}
{"type": "node_health", "node": "heating.pump", "health": "ok"}
)
clock.advance(5)
@@ -185,7 +185,13 @@ def test_alerting_can_be_switched_off():
"The work queue is unreachable",
),
({"type": "engine_degraded", "reason": "lag"}, "The engine is struggling"),
({"type": "node_health", "status": "ok"}, None),
({"type": "node_health", "health": "ok"}, None),
# The engine publishes `health`, not `status`: reading the wrong key
# meant a device dropping never alerted anyone.
(
{"type": "node_health", "node": "heating.pump", "health": "down"},
"heating.pump lost its connection",
),
],
)
def test_every_alerting_event_reads_as_a_sentence(event, expected):
backend/tests/test_metrics.py
+149
View File
@@ -0,0 +1,149 @@
"""The collector writes down what the bus only ever broadcast.
Not under ``tests/flow`` with the rest of the engine: that package opts out of
the database, and writing to it is this module's whole job.
"""
import asyncio
import time
from datetime import datetime, timezone
from sqlmodel import Session, select
from app.flow.events import EventBus
from app.flow.metrics import MetricsCollector
from app.models import EngineEvent, FlowRun, MetricBucket
FLOW = "metrics-test"
NODE = "metrics-test.calc"
def _events(collector: MetricsCollector, ts: float, run: str) -> None:
collector.handle(
{
"type": "cascade_started",
"run": run,
"flow": FLOW,
"node": "metrics-test.in",
"cause": "external",
"deliveries": 1,
"ts": ts,
}
)
collector.handle(
{
"type": "node_executed",
"flow": FLOW,
"node": NODE,
"outputs": 2,
"duration_ms": 5.0,
"run": run,
"ts": ts,
}
)
collector.handle(
{"type": "work_latency", "flow": FLOW, "node": NODE, "lag_ms": 12.0, "ts": ts}
)
# The traceback arrives as its own event, just before the failure.
collector.handle(
{
"type": "node_log",
"flow": FLOW,
"node": NODE,
"level": "error",
"text": "Traceback: line 3, in run",
"ts": ts,
}
)
collector.handle(
{
"type": "node_error",
"flow": FLOW,
"node": NODE,
"error": "ValueError: bad input",
"run": run,
"ts": ts,
}
)
def test_events_become_rollups_failures_runs_and_audit(db: Session) -> None:
collector = MetricsCollector(EventBus())
# The current minute, pinned: the second flush has to land in the same
# bucket, and anything past the retention window is pruned on write.
ts = datetime.now(timezone.utc).replace(second=0, microsecond=0).timestamp()
_events(collector, ts, "1-0")
collector.handle(
{
"type": "audit",
"action": "published",
"flow": FLOW,
"user": "someone@example.com",
"ts": ts,
}
)
collector.handle(
{"type": "cascade_finished", "run": "1-0", "flow": FLOW, "ts": ts + 0.5}
)
asyncio.run(collector.flush())
bucket = db.exec(
select(MetricBucket).where(MetricBucket.flow == FLOW, MetricBucket.node == NODE)
).one()
assert (bucket.executions, bucket.errors, bucket.messages) == (1, 1, 2)
assert bucket.duration_max_ms == 5.0
assert (bucket.lag_max_ms, bucket.items) == (12.0, 1)
failure = db.exec(
select(EngineEvent).where(
EngineEvent.flow == FLOW, EngineEvent.type == "node_error"
)
).one()
assert "ValueError: bad input" in failure.detail
assert "line 3, in run" in failure.detail
audit = db.exec(
select(EngineEvent).where(EngineEvent.flow == FLOW, EngineEvent.type == "audit")
).one()
assert (audit.actor, audit.detail) == ("someone@example.com", "published")
run = db.exec(select(FlowRun).where(FlowRun.id == "1-0")).one()
assert (run.status, run.flow, run.source) == ("error", FLOW, "external")
assert (run.nodes, run.errors) == (1, 1)
assert run.duration_ms > 0
# The same minute, written again: the counters add rather than duplicate.
_events(collector, ts + 10, "2-0")
asyncio.run(collector.flush())
db.expire_all()
bucket = db.exec(
select(MetricBucket).where(MetricBucket.flow == FLOW, MetricBucket.node == NODE)
).one()
assert (bucket.executions, bucket.errors) == (2, 2)
# Never finished, so it is still open — the prune is what closes it.
open_run = db.exec(select(FlowRun).where(FlowRun.id == "2-0")).one()
assert open_run.status == "running"
def test_a_run_that_did_not_fail_reads_ok(db: Session) -> None:
collector = MetricsCollector(EventBus())
ts = time.time()
collector.handle(
{
"type": "cascade_started",
"run": "manual-abc",
"flow": FLOW,
"node": "metrics-test.in",
"cause": "manual",
"deliveries": 1,
"ts": ts,
}
)
collector.handle(
{"type": "cascade_finished", "run": "manual-abc", "flow": FLOW, "ts": ts + 0.1}
)
asyncio.run(collector.flush())
run = db.exec(select(FlowRun).where(FlowRun.id == "manual-abc")).one()
assert (run.status, run.source) == ("ok", "manual")