Let Postgres fold the rollups, and say when a run list was cut short

/observability/timeseries and /flows read every metric_minute row in the window
and folded them in Python, so the 7d preset pulled a week of rows on each 30 s
poll. date_bin() does the binning now — the row count drops to the slices asked
for, and to flows × 60 for the sparklines. A window of zero hours used to divide
by nothing and answer 500; windows are clamped to an hour at the low end and to
the retention period at the high end, past which there is nothing to find.

/observability/runs returns {data, count} rather than a bare list, so a minute
busier than the 200-row cap says so instead of quietly showing its newest 200.
The count is only queried when the page comes back full, which keeps the poll
from handing back what the fold just saved.

failures_24h leaves the summary — the Home tile counts errors over the selected
window from the rollups, and nothing had read the field since.

Deleting a flow now takes its Run rows and their nodes, metrics and artifacts
with it. This lives in the route rather than in forget_flow because renaming a
flow calls that too, and a rename must keep its history. The observability
rollups stay: they are the record of what ran, and retention already prunes them.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uq8mtNb97A7praJLyeEYgs
This commit is contained in:
2026-08-21 10:11:19 +02:00
co-authored by Claude Opus 5
parent 08ccbbac5b
commit d375ea97cc
9 changed files with 291 additions and 86 deletions
+25 -3
View File
@@ -15,11 +15,13 @@ from fastapi import (
from fastapi.concurrency import run_in_threadpool from fastapi.concurrency import run_in_threadpool
from jwt.exceptions import InvalidTokenError from jwt.exceptions import InvalidTokenError
from pydantic import BaseModel from pydantic import BaseModel
from sqlmodel import Session from sqlalchemy import delete
from sqlmodel import Session, col, select
from app.api.deps import ( from app.api.deps import (
CurrentUser, CurrentUser,
FlowControllerDep, FlowControllerDep,
SessionDep,
decode_token, decode_token,
get_current_user, get_current_user,
user_from_token, user_from_token,
@@ -55,7 +57,7 @@ from app.flow.store import (
LibNotFound, LibNotFound,
StaleVersion, StaleVersion,
) )
from app.models import Message from app.models import Message, Run, RunArtifact, RunMetric, RunNode
router = APIRouter( router = APIRouter(
prefix="/flows", tags=["flows"], dependencies=[Depends(get_current_user)] prefix="/flows", tags=["flows"], dependencies=[Depends(get_current_user)]
@@ -209,6 +211,25 @@ def _audit(action: str, flow: str, user: CurrentUser) -> None:
) )
def _forget_runs(session: Session, flow: str) -> None:
"""A deleted flow's runs, and everything hanging off them.
Here rather than in ``FlowController.forget_flow`` because renaming a flow
calls that too, and a rename must keep its experiment history.
Only the run tables: ``flow_run``, ``metric_minute`` and ``engine_event``
are the observability rollups, deliberately kept as a record of what ran
and already pruned at OBS_RETENTION_DAYS.
"""
# A subquery, not a materialised list of ids: a demo can hold thousands.
runs = select(col(Run.id)).where(col(Run.flow) == flow)
session.execute(delete(RunNode).where(col(RunNode.run_id).in_(runs)))
session.execute(delete(RunMetric).where(col(RunMetric.run_id).in_(runs)))
session.execute(delete(RunArtifact).where(col(RunArtifact.run_id).in_(runs)))
session.execute(delete(Run).where(col(Run.flow) == flow))
session.commit()
def _source_ref(definition: FlowDef, node_id: str) -> str | None: def _source_ref(definition: FlowDef, node_id: str) -> str | None:
"""The library source this node runs, if it is a shared one.""" """The library source this node runs, if it is a shared one."""
node = next((n for n in definition.nodes if n.id == node_id), None) node = next((n for n in definition.nodes if n.id == node_id), None)
@@ -400,7 +421,7 @@ async def discard_draft(name: str, controller: FlowControllerDep) -> Any:
@router.delete("/{name}", response_model=Message) @router.delete("/{name}", response_model=Message)
async def delete_flow( async def delete_flow(
name: str, controller: FlowControllerDep, user: CurrentUser name: str, controller: FlowControllerDep, user: CurrentUser, session: SessionDep
) -> Any: ) -> Any:
"""Delete a flow and everything in it.""" """Delete a flow and everything in it."""
try: try:
@@ -410,6 +431,7 @@ async def delete_flow(
_audit("deleted", name, user) _audit("deleted", name, user)
# Its files are gone; its values and queued work would otherwise linger. # Its files are gone; its values and queued work would otherwise linger.
await run_in_threadpool(controller.forget_flow, name) await run_in_threadpool(controller.forget_flow, name)
await run_in_threadpool(_forget_runs, session, name)
await controller.reload() await controller.reload()
return Message(message=f"Deleted flow '{name}'") return Message(message=f"Deleted flow '{name}'")
+107 -63
View File
@@ -13,10 +13,12 @@ from typing import Any, Literal
from fastapi import APIRouter, Depends, Request from fastapi import APIRouter, Depends, Request
from fastapi.concurrency import run_in_threadpool from fastapi.concurrency import run_in_threadpool
from pydantic import BaseModel from pydantic import BaseModel
from sqlalchemy import func from sqlalchemy import ColumnElement, DateTime, Interval, cast, func, literal
from sqlalchemy import select as sa_select
from sqlmodel import col, select from sqlmodel import col, select
from app.api.deps import FlowControllerDep, SessionDep, get_current_user from app.api.deps import FlowControllerDep, SessionDep, get_current_user
from app.core.config import settings
from app.flow.controller import ADVISORY_ISSUES, NodeStatus from app.flow.controller import ADVISORY_ISSUES, NodeStatus
from app.models import EngineEvent, FlowRun, MetricBucket from app.models import EngineEvent, FlowRun, MetricBucket
@@ -29,6 +31,10 @@ router = APIRouter(
#: How many slices a per-flow sparkline is folded into. #: How many slices a per-flow sparkline is folded into.
SPARK_SLICES = 60 SPARK_SLICES = 60
#: The origin fixed-stride slots are aligned to, which is the alignment the
#: fold used to get from ``stamp - stamp % bucket_s``.
EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
class HealthSummary(BaseModel): class HealthSummary(BaseModel):
status: str status: str
@@ -37,7 +43,6 @@ class HealthSummary(BaseModel):
nodes: dict[str, int] nodes: dict[str, int]
queue: dict[str, Any] queue: dict[str, Any]
loop_lag: dict[str, float] loop_lag: dict[str, float]
failures_24h: int
class SeriesPoint(BaseModel): class SeriesPoint(BaseModel):
@@ -74,6 +79,11 @@ class RunRow(BaseModel):
deliveries: int deliveries: int
class RunPage(BaseModel):
data: list[RunRow]
count: int
class EventRow(BaseModel): class EventRow(BaseModel):
id: int id: int
ts: datetime ts: datetime
@@ -97,15 +107,22 @@ def _since(hours: int) -> datetime:
return datetime.now(timezone.utc) - timedelta(hours=hours) return datetime.now(timezone.utc) - timedelta(hours=hours)
def _window_hours(hours: int) -> int:
"""A window the rollups can answer for: an hour at least, retention at most.
Zero used to divide by nothing and answer 500, and nothing older than
retention exists, so a larger window is a scan that can only find less.
"""
return max(1, min(hours, settings.OBS_RETENTION_DAYS * 24))
def _aware(when: datetime) -> datetime: def _aware(when: datetime) -> datetime:
"""A bound as the columns store it. A naive one is read as UTC.""" """A bound as the columns store it. A naive one is read as UTC."""
return when if when.tzinfo else when.replace(tzinfo=timezone.utc) return when if when.tzinfo else when.replace(tzinfo=timezone.utc)
@router.get("/summary", response_model=HealthSummary) @router.get("/summary", response_model=HealthSummary)
async def read_summary( async def read_summary(request: Request, controller: FlowControllerDep) -> Any:
request: Request, controller: FlowControllerDep, session: SessionDep
) -> Any:
"""How the engine is doing right now. Always 200, degraded or not.""" """How the engine is doing right now. Always 200, degraded or not."""
watchdog = getattr(request.app.state, "watchdog", None) watchdog = getattr(request.app.state, "watchdog", None)
problems: list[str] = [] problems: list[str] = []
@@ -144,14 +161,6 @@ async def read_summary(
if invalid: if invalid:
problems.append(f"{len(invalid)} flow(s) cannot run: {', '.join(invalid)}") problems.append(f"{len(invalid)} flow(s) cannot run: {', '.join(invalid)}")
statement = (
select(func.count())
.select_from(EngineEvent)
.where(col(EngineEvent.ts) >= _since(24), col(EngineEvent.type) != "audit")
)
# The health page polls this every ten seconds; the driver is synchronous.
failures = await run_in_threadpool(lambda: session.exec(statement).one())
return HealthSummary( return HealthSummary(
status="degraded" if problems else "ok", status="degraded" if problems else "ok",
problems=problems, problems=problems,
@@ -181,7 +190,6 @@ async def read_summary(
if watchdog is not None if watchdog is not None
else {"ewma": 0.0, "max_60s": 0.0} else {"ewma": 0.0, "max_60s": 0.0}
), ),
failures_24h=int(failures),
) )
@@ -194,63 +202,79 @@ def read_timeseries(
bucket_s: int = 60, bucket_s: int = 60,
) -> Any: ) -> Any:
"""Executions, errors and timings over time, summed across nodes.""" """Executions, errors and timings over time, summed across nodes."""
# ponytail: the fold is in Python — a day is at most 1440 rows per node. hours = _window_hours(hours)
# date_bin() if the window ever grows past that. # Postgres does the fold: a week of minute rows per node used to cross the
statement = select(MetricBucket).where(col(MetricBucket.bucket) >= _since(hours)) # wire on every poll, and only the slices need to. The casts are load
# bearing — date_bin() is overloaded on timestamp and timestamptz, and an
# untyped bind parameter leaves the call ambiguous.
stride = timedelta(seconds=max(60, bucket_s))
slot = func.date_bin(
cast(literal(stride), Interval),
col(MetricBucket.bucket),
cast(literal(EPOCH), DateTime(timezone=True)),
).label("slot")
statement = sa_select(
slot,
func.sum(col(MetricBucket.executions)).label("executions"),
func.sum(col(MetricBucket.errors)).label("errors"),
func.sum(col(MetricBucket.messages)).label("messages"),
func.sum(col(MetricBucket.duration_sum_ms)).label("duration_sum_ms"),
func.max(col(MetricBucket.duration_max_ms)).label("max_ms"),
func.sum(col(MetricBucket.lag_sum_ms)).label("lag_sum_ms"),
func.sum(col(MetricBucket.items)).label("items"),
).where(col(MetricBucket.bucket) >= _since(hours))
if flow: if flow:
statement = statement.where(col(MetricBucket.flow) == flow) statement = statement.where(col(MetricBucket.flow) == flow)
if node: if node:
statement = statement.where(col(MetricBucket.node) == node) statement = statement.where(col(MetricBucket.node) == node)
slices: dict[float, dict[str, float]] = {}
for row in session.exec(statement.order_by(col(MetricBucket.bucket))):
stamp = row.bucket.timestamp()
key = stamp - stamp % max(60, bucket_s)
point = slices.setdefault(
key,
{
"executions": 0.0,
"errors": 0.0,
"messages": 0.0,
"duration_sum_ms": 0.0,
"max_ms": 0.0,
"lag_sum_ms": 0.0,
"items": 0.0,
},
)
point["executions"] += row.executions
point["errors"] += row.errors
point["messages"] += row.messages
point["duration_sum_ms"] += row.duration_sum_ms
point["max_ms"] = max(point["max_ms"], row.duration_max_ms)
point["lag_sum_ms"] += row.lag_sum_ms
point["items"] += row.items
return [ return [
SeriesPoint( SeriesPoint(
ts=ts, ts=row.slot.timestamp(),
executions=int(point["executions"]), executions=int(row.executions),
errors=int(point["errors"]), errors=int(row.errors),
messages=int(point["messages"]), messages=int(row.messages),
avg_ms=round(point["duration_sum_ms"] / (point["executions"] or 1), 2), avg_ms=round(row.duration_sum_ms / (row.executions or 1), 2),
max_ms=round(point["max_ms"], 2), max_ms=round(row.max_ms, 2),
avg_lag_ms=round(point["lag_sum_ms"] / (point["items"] or 1), 2), avg_lag_ms=round(row.lag_sum_ms / (row.items or 1), 2),
) )
for ts, point in sorted(slices.items()) for row in session.execute(statement.group_by(slot).order_by(slot))
] ]
@router.get("/flows", response_model=list[FlowRollup]) @router.get("/flows", response_model=list[FlowRollup])
def read_flow_rollups(session: SessionDep, hours: int = 24) -> Any: def read_flow_rollups(session: SessionDep, hours: int = 24) -> Any:
"""One row per flow, with a coarse trend of how much it ran.""" """One row per flow, with a coarse trend of how much it ran."""
hours = _window_hours(hours)
since = _since(hours) since = _since(hours)
window = hours * 3600 window = hours * 3600
start = since.timestamp() start = since.timestamp()
# Binned to the sparkline slice rather than the minute, so a flow costs at
# most SPARK_SLICES rows however long the window is. The slice is the
# window over SPARK_SLICES, which for whole hours is whole minutes.
slot = func.date_bin(
cast(literal(timedelta(minutes=hours)), Interval),
col(MetricBucket.bucket),
cast(literal(since), DateTime(timezone=True)),
).label("slot")
statement = (
sa_select(
col(MetricBucket.flow),
slot,
func.sum(col(MetricBucket.executions)).label("executions"),
func.sum(col(MetricBucket.errors)).label("errors"),
func.sum(col(MetricBucket.messages)).label("messages"),
func.sum(col(MetricBucket.duration_sum_ms)).label("duration_sum_ms"),
func.sum(col(MetricBucket.lag_sum_ms)).label("lag_sum_ms"),
func.sum(col(MetricBucket.items)).label("items"),
)
.where(col(MetricBucket.bucket) >= since)
.group_by(col(MetricBucket.flow), slot)
)
rollups: dict[str, dict[str, Any]] = {} rollups: dict[str, dict[str, Any]] = {}
for row in session.exec( for row in session.execute(statement):
select(MetricBucket).where(col(MetricBucket.bucket) >= since)
):
entry = rollups.setdefault( entry = rollups.setdefault(
row.flow, row.flow,
{ {
@@ -269,11 +293,14 @@ def read_flow_rollups(session: SessionDep, hours: int = 24) -> Any:
entry["duration_sum_ms"] += row.duration_sum_ms entry["duration_sum_ms"] += row.duration_sum_ms
entry["lag_sum_ms"] += row.lag_sum_ms entry["lag_sum_ms"] += row.lag_sum_ms
entry["items"] += row.items entry["items"] += row.items
slot = min( # A slot sits a whole number of slices from `since`, so this rounds
# rather than truncates: a float a hair short would lose a slice. The
# clamp holds the bucket landing exactly on the far edge in range.
index = min(
SPARK_SLICES - 1, SPARK_SLICES - 1,
max(0, int((row.bucket.timestamp() - start) / window * SPARK_SLICES)), max(0, round((row.slot.timestamp() - start) / window * SPARK_SLICES)),
) )
entry["spark"][slot] += row.executions entry["spark"][index] += row.executions
# `.all()` first: a Result has `keys()`, so dict() would read it as a # `.all()` first: a Result has `keys()`, so dict() would read it as a
# mapping and subscript it. # mapping and subscript it.
@@ -302,7 +329,7 @@ def read_flow_rollups(session: SessionDep, hours: int = 24) -> Any:
] ]
@router.get("/runs", response_model=list[RunRow]) @router.get("/runs", response_model=RunPage)
def read_runs( def read_runs(
session: SessionDep, session: SessionDep,
flow: str | None = None, flow: str | None = None,
@@ -311,21 +338,38 @@ def read_runs(
until: datetime | None = None, until: datetime | None = None,
limit: int = 50, limit: int = 50,
) -> Any: ) -> Any:
"""Recent cascades, newest first. """Recent cascades, newest first, and how many there were in total.
``since`` is inclusive and ``until`` exclusive, so a window of one minute ``since`` is inclusive and ``until`` exclusive, so a window of one minute
holds exactly the runs of the minute bucket the charts are drawn from. holds exactly the runs of the minute bucket the charts are drawn from.
""" """
statement = select(FlowRun).order_by(col(FlowRun.started_at).desc()) filters: list[ColumnElement[bool]] = []
if flow: if flow:
statement = statement.where(col(FlowRun.flow) == flow) filters.append(col(FlowRun.flow) == flow)
if status: if status:
statement = statement.where(col(FlowRun.status) == status) filters.append(col(FlowRun.status) == status)
if since: if since:
statement = statement.where(col(FlowRun.started_at) >= _aware(since)) filters.append(col(FlowRun.started_at) >= _aware(since))
if until: if until:
statement = statement.where(col(FlowRun.started_at) < _aware(until)) filters.append(col(FlowRun.started_at) < _aware(until))
return list(session.exec(statement.limit(min(limit, 200))))
capped = min(limit, 200)
statement = select(FlowRun).where(*filters).order_by(col(FlowRun.started_at).desc())
rows = list(session.exec(statement.limit(capped)))
# A short page is its own total. The lists poll their whole range every
# thirty seconds, and counting on each of those would hand back what
# binning the metrics just saved — for a number that only ever says
# "there is more here than fits".
count = (
len(rows)
if len(rows) < capped
else int(
session.exec(
select(func.count()).select_from(FlowRun).where(*filters)
).one()
)
)
return {"data": rows, "count": count}
@router.get("/events", response_model=list[EventRow]) @router.get("/events", response_model=list[EventRow])
+6 -2
View File
@@ -284,7 +284,7 @@ async def apply_modules(requirements: str) -> Any:
@mcp.tool() @mcp.tool()
async def get_health() -> Any: async def get_health() -> Any:
"""How the engine is doing: flows, nodes, queue, loop lag and recent failures.""" """How the engine is doing right now: flows, nodes, queue and loop lag."""
return await _call("GET", "/observability/summary") return await _call("GET", "/observability/summary")
@@ -312,7 +312,11 @@ async def list_failures(flow: str | None = None, limit: int = 50) -> Any:
@mcp.tool() @mcp.tool()
async def list_runs(flow: str | None = None, limit: int = 50) -> Any: async def list_runs(flow: str | None = None, limit: int = 50) -> Any:
"""Recent cascades: what triggered them, how long they took, how they ended.""" """Recent cascades: what triggered them, how long they took, how they ended.
A page of rows plus the total number matching, which says whether the limit
cut anything off.
"""
params: dict[str, Any] = {"limit": limit} params: dict[str, Any] = {"limit": limit}
if flow: if flow:
params["flow"] = flow params["flow"] = flow
+16 -1
View File
@@ -1,6 +1,11 @@
from datetime import UTC, datetime
from fastapi.testclient import TestClient from fastapi.testclient import TestClient
from sqlalchemy import func
from sqlmodel import Session, select
from app.core.config import settings from app.core.config import settings
from app.models import Run, RunArtifact, RunMetric, RunNode
PREFIX = f"{settings.API_V1_STR}/flows" PREFIX = f"{settings.API_V1_STR}/flows"
@@ -273,9 +278,14 @@ def test_unconnected_input_is_surfaced(
def test_delete_flow( def test_delete_flow(
client: TestClient, superuser_token_headers: dict[str, str] client: TestClient, superuser_token_headers: dict[str, str], db: Session
) -> None: ) -> None:
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow()) client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
db.add(Run(id="run-1", flow="demo", created_at=datetime.now(UTC)))
db.add(RunNode(run_id="run-1", node="sensor"))
db.add(RunMetric(run_id="run-1", name="loss", step=-1))
db.add(RunArtifact(run_id="run-1", name="model.pt"))
db.commit()
assert ( assert (
client.delete(f"{PREFIX}/demo", headers=superuser_token_headers).status_code client.delete(f"{PREFIX}/demo", headers=superuser_token_headers).status_code
@@ -285,6 +295,11 @@ def test_delete_flow(
client.get(f"{PREFIX}/demo", headers=superuser_token_headers).status_code == 404 client.get(f"{PREFIX}/demo", headers=superuser_token_headers).status_code == 404
) )
# Deleting the flow takes its runs with it, so a reseeded demo starts clean.
db.expire_all()
for model in (Run, RunNode, RunMetric, RunArtifact):
assert db.exec(select(func.count()).select_from(model)).one() == 0
def test_node_types_are_listed( def test_node_types_are_listed(
client: TestClient, superuser_token_headers: dict[str, str] client: TestClient, superuser_token_headers: dict[str, str]
+84 -2
View File
@@ -76,6 +76,9 @@ def test_the_summary_answers_even_when_degraded(
"invalid", "invalid",
} }
assert "error" in body["nodes"] assert "error" in body["nodes"]
# A windowed count comes from /flows?hours= now, which is what the tile
# that used to read this actually sums.
assert "failures_24h" not in body
def test_a_flow_that_cannot_run_makes_the_summary_degraded( def test_a_flow_that_cannot_run_makes_the_summary_degraded(
@@ -140,7 +143,7 @@ def test_the_history_reads_back(
assert row["last_error_ts"] is not None assert row["last_error_ts"] is not None
runs = client.get(f"{PREFIX}/runs", headers=superuser_token_headers).json() runs = client.get(f"{PREFIX}/runs", headers=superuser_token_headers).json()
assert any(run["id"] == "9-0" and run["status"] == "ok" for run in runs) assert any(run["id"] == "9-0" and run["status"] == "ok" for run in runs["data"])
failures = client.get(f"{PREFIX}/events", headers=superuser_token_headers).json() failures = client.get(f"{PREFIX}/events", headers=superuser_token_headers).json()
assert any("Traceback" in event["detail"] for event in failures) assert any("Traceback" in event["detail"] for event in failures)
@@ -156,6 +159,85 @@ def test_the_history_reads_back(
assert isinstance(dead.json(), list) assert isinstance(dead.json(), list)
def test_the_timeseries_folds_into_the_requested_bucket(
client: TestClient, superuser_token_headers: dict[str, str], db: Session
) -> None:
"""The fold moved into SQL, so the slices still have to line up as before.
Slots are aligned to the epoch, which is what the old modulo did, and a
slice only exists where a row does.
"""
flow = "bucket-fold-test"
# The start of the current quarter hour, half an hour back so both slices
# sit inside a one hour window.
now = datetime.now(timezone.utc).timestamp()
first = datetime.fromtimestamp(now // 900 * 900 - 1800, timezone.utc)
for offset, executions in ((0, 1), (2, 2), (5, 4), (15, 8)):
db.add(
MetricBucket(
flow=flow,
node=f"{flow}.calc",
bucket=first + timedelta(minutes=offset),
executions=executions,
)
)
db.commit()
points = client.get(
f"{PREFIX}/timeseries",
headers=superuser_token_headers,
params={"flow": flow, "hours": 1, "bucket_s": 900},
).json()
assert len(points) == 2
assert points[0]["executions"] == 1 + 2 + 4
assert points[1]["executions"] == 8
assert points[0]["ts"] % 900 == 0
def test_a_zero_hour_window_is_still_an_hour(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
"""`hours=0` divided by nothing and answered 500."""
for path in ("timeseries", "flows"):
response = client.get(
f"{PREFIX}/{path}", headers=superuser_token_headers, params={"hours": 0}
)
assert response.status_code == 200
def test_the_runs_page_carries_its_total(
client: TestClient, superuser_token_headers: dict[str, str], db: Session
) -> None:
"""A full page says how much it left behind."""
minute = datetime.now(timezone.utc).replace(second=0, microsecond=0) - timedelta(
hours=5
)
for index in range(3):
db.add(
FlowRun(
id=f"paged-{index}",
flow=FLOW,
started_at=minute + timedelta(seconds=index),
status="ok",
)
)
db.commit()
body = client.get(
f"{PREFIX}/runs",
headers=superuser_token_headers,
params={
"since": minute.isoformat(),
"until": (minute + timedelta(minutes=1)).isoformat(),
"limit": 2,
},
).json()
assert len(body["data"]) == 2
assert body["count"] == 3
def test_runs_narrow_to_one_minute( def test_runs_narrow_to_one_minute(
client: TestClient, superuser_token_headers: dict[str, str], db: Session client: TestClient, superuser_token_headers: dict[str, str], db: Session
) -> None: ) -> None:
@@ -185,7 +267,7 @@ def test_runs_narrow_to_one_minute(
# The upper bound is exclusive, so the run starting the next minute is not # The upper bound is exclusive, so the run starting the next minute is not
# in this one. # in this one.
assert [run["id"] for run in runs] == ["minute-in"] assert [run["id"] for run in runs["data"]] == ["minute-in"]
def test_events_narrow_to_one_minute( def test_events_narrow_to_one_minute(
+20 -5
View File
@@ -1081,14 +1081,10 @@ export const HealthSummarySchema = {
}, },
type: 'object', type: 'object',
title: 'Loop Lag' title: 'Loop Lag'
},
failures_24h: {
type: 'integer',
title: 'Failures 24H'
} }
}, },
type: 'object', type: 'object',
required: ['status', 'problems', 'flows', 'nodes', 'queue', 'loop_lag', 'failures_24h'], required: ['status', 'problems', 'flows', 'nodes', 'queue', 'loop_lag'],
title: 'HealthSummary' title: 'HealthSummary'
} as const; } as const;
@@ -2421,6 +2417,25 @@ export const RunNodeRowSchema = {
title: 'RunNodeRow' title: 'RunNodeRow'
} as const; } as const;
export const RunPageSchema = {
properties: {
data: {
items: {
'$ref': '#/components/schemas/app__api__routes__observability__RunRow'
},
type: 'array',
title: 'Data'
},
count: {
type: 'integer',
title: 'Count'
}
},
type: 'object',
required: ['data', 'count'],
title: 'RunPage'
} as const;
export const RunRequestSchema = { export const RunRequestSchema = {
properties: { properties: {
inputs: { inputs: {
+2 -2
View File
@@ -1308,7 +1308,7 @@ export class ObservabilityService {
/** /**
* Read Runs * Read Runs
* Recent cascades, newest first. * Recent cascades, newest first, and how many there were in total.
* *
* ``since`` is inclusive and ``until`` exclusive, so a window of one minute * ``since`` is inclusive and ``until`` exclusive, so a window of one minute
* holds exactly the runs of the minute bucket the charts are drawn from. * holds exactly the runs of the minute bucket the charts are drawn from.
@@ -1318,7 +1318,7 @@ export class ObservabilityService {
* @param data.since * @param data.since
* @param data.until * @param data.until
* @param data.limit * @param data.limit
* @returns app__api__routes__observability__RunRow Successful Response * @returns RunPage Successful Response
* @throws ApiError * @throws ApiError
*/ */
public static readRuns(data: ObservabilityReadRunsData = {}): CancelablePromise<ObservabilityReadRunsResponse> { public static readRuns(data: ObservabilityReadRunsData = {}): CancelablePromise<ObservabilityReadRunsResponse> {
+6 -2
View File
@@ -408,7 +408,6 @@ export type HealthSummary = {
loop_lag: { loop_lag: {
[key: string]: (number); [key: string]: (number);
}; };
failures_24h: number;
}; };
/** /**
@@ -856,6 +855,11 @@ export type RunNodeRow = {
logs: string; logs: string;
}; };
export type RunPage = {
data: Array<app__api__routes__observability__RunRow>;
count: number;
};
export type RunRequest = { export type RunRequest = {
inputs?: { inputs?: {
[key: string]: unknown; [key: string]: unknown;
@@ -1399,7 +1403,7 @@ export type ObservabilityReadRunsData = {
until?: (string | null); until?: (string | null);
}; };
export type ObservabilityReadRunsResponse = (Array<app__api__routes__observability__RunRow>); export type ObservabilityReadRunsResponse = (RunPage);
export type ObservabilityReadEventsData = { export type ObservabilityReadEventsData = {
flow?: (string | null); flow?: (string | null);
@@ -8,7 +8,7 @@ import { UplotChart } from "@/components/Common/UplotChart"
import { useEngineEvents } from "@/components/Flow/liveStore" import { useEngineEvents } from "@/components/Flow/liveStore"
import { PANEL_SECTION } from "@/components/Flow/SidePanel" import { PANEL_SECTION } from "@/components/Flow/SidePanel"
import { Button } from "@/components/ui/button" import { Button } from "@/components/ui/button"
import { cn, dur } from "@/lib/utils" import { cn, dur, si } from "@/lib/utils"
import { import {
ago, ago,
auditQueryOptions, auditQueryOptions,
@@ -138,7 +138,15 @@ function Chart({
} }
/** What a list is showing, and the way back to all of it. */ /** What a list is showing, and the way back to all of it. */
function ListHeader({ title, moment }: { title: string; moment: Moment }) { function ListHeader({
title,
moment,
note,
}: {
title: string
moment: Moment
note?: string
}) {
return ( return (
<div className="flex min-h-6 flex-wrap items-center gap-2"> <div className="flex min-h-6 flex-wrap items-center gap-2">
<h2 className={PANEL_SECTION}>{title}</h2> <h2 className={PANEL_SECTION}>{title}</h2>
@@ -147,6 +155,9 @@ function ListHeader({ title, moment }: { title: string; moment: Moment }) {
{moment.pinned !== null ? "pinned to" : "showing"} {clock(moment.at)} {moment.pinned !== null ? "pinned to" : "showing"} {clock(moment.at)}
</span> </span>
) : null} ) : null}
{note ? (
<span className="text-xs text-muted-foreground">{note}</span>
) : null}
{moment.pinned !== null ? <ClearPin moment={moment} /> : null} {moment.pinned !== null ? <ClearPin moment={moment} /> : null}
</div> </div>
) )
@@ -249,10 +260,18 @@ export function HealthActivity({ range }: { range: Range }) {
// minute the pointer rests on. // minute the pointer rests on.
const shownRuns = const shownRuns =
runsAt.pinned !== null runsAt.pinned !== null
? (pinnedRuns ?? []) ? (pinnedRuns?.data ?? [])
: runsAt.at === null : runsAt.at === null
? (runs ?? []).slice(0, RUNS_SHOWN) ? (runs?.data ?? []).slice(0, RUNS_SHOWN)
: (runs ?? []).filter((run) => minuteOf(run.started_at) === runsAt.at) : (runs?.data ?? []).filter(
(run) => minuteOf(run.started_at) === runsAt.at,
)
// A busy minute writes more runs than one page carries. Without the total
// behind it, a truncated list is indistinguishable from a quiet minute.
const runsNote =
runsAt.pinned !== null && (pinnedRuns?.count ?? 0) > shownRuns.length
? `showing ${shownRuns.length} of ${si(pinnedRuns?.count ?? 0)}`
: undefined
const shownFailures = const shownFailures =
failuresAt.pinned !== null failuresAt.pinned !== null
? (pinnedFailures ?? []) ? (pinnedFailures ?? [])
@@ -290,7 +309,7 @@ export function HealthActivity({ range }: { range: Range }) {
<section className="grid content-start gap-4 lg:grid-cols-2"> <section className="grid content-start gap-4 lg:grid-cols-2">
<div className="grid content-start gap-3"> <div className="grid content-start gap-3">
<ListHeader title="Recent runs" moment={runsAt} /> <ListHeader title="Recent runs" moment={runsAt} note={runsNote} />
<div <div
className={cn( className={cn(
CARD, CARD,