stroblmeandClaude Fable 5
0b91379914
/observability/runs gains since/until, so the throughput chart's pin asks the server for its minute instead of filtering a fixed recent list. This engine writes ~60 runs a minute, so any minute but the newest read empty. since is inclusive and until exclusive, matching the minute buckets the charts are drawn from. Hover stays the client-side preview it was: scrubbing a day would otherwise be a request per minute rested on. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt
137 lines
4.2 KiB
Python
137 lines
4.2 KiB
Python
from datetime import datetime, timedelta, timezone
|
|
|
|
from fastapi.testclient import TestClient
|
|
from sqlmodel import Session
|
|
|
|
from app.core.config import settings
|
|
from app.models import EngineEvent, FlowRun, MetricBucket
|
|
|
|
PREFIX = f"{settings.API_V1_STR}/observability"
|
|
FLOW = "observability-test"
|
|
|
|
|
|
def _seed(db: Session) -> None:
|
|
now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
|
|
db.add(
|
|
MetricBucket(
|
|
flow=FLOW,
|
|
node=f"{FLOW}.calc",
|
|
bucket=now - timedelta(minutes=1),
|
|
executions=4,
|
|
errors=1,
|
|
messages=6,
|
|
duration_sum_ms=40.0,
|
|
duration_max_ms=25.0,
|
|
lag_sum_ms=100.0,
|
|
lag_max_ms=60.0,
|
|
items=4,
|
|
)
|
|
)
|
|
db.add(
|
|
EngineEvent(
|
|
ts=now,
|
|
type="node_error",
|
|
flow=FLOW,
|
|
node=f"{FLOW}.calc",
|
|
detail="ValueError: bad input\nTraceback",
|
|
)
|
|
)
|
|
db.add(
|
|
EngineEvent(
|
|
ts=now, type="audit", flow=FLOW, detail="published", actor="a@example.com"
|
|
)
|
|
)
|
|
db.add(
|
|
FlowRun(
|
|
id="9-0",
|
|
flow=FLOW,
|
|
source="external",
|
|
started_at=now,
|
|
finished_at=now,
|
|
status="ok",
|
|
nodes=3,
|
|
duration_ms=12.5,
|
|
)
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def test_observability_requires_authentication(client: TestClient) -> None:
|
|
assert client.get(f"{PREFIX}/summary").status_code == 401
|
|
|
|
|
|
def test_the_summary_answers_even_when_degraded(
|
|
client: TestClient, superuser_token_headers: dict[str, str]
|
|
) -> None:
|
|
response = client.get(f"{PREFIX}/summary", headers=superuser_token_headers)
|
|
|
|
assert response.status_code == 200
|
|
body = response.json()
|
|
assert body["status"] in {"ok", "degraded"}
|
|
assert set(body["flows"]) == {"total", "running", "paused", "quarantined"}
|
|
assert "error" in body["nodes"]
|
|
|
|
|
|
def test_the_history_reads_back(
|
|
client: TestClient, superuser_token_headers: dict[str, str], db: Session
|
|
) -> None:
|
|
_seed(db)
|
|
|
|
points = client.get(f"{PREFIX}/timeseries", headers=superuser_token_headers).json()
|
|
mine = [point for point in points if point["executions"]]
|
|
assert mine and mine[-1]["avg_ms"] > 0
|
|
|
|
flows = client.get(f"{PREFIX}/flows", headers=superuser_token_headers).json()
|
|
row = next(entry for entry in flows if entry["flow"] == FLOW)
|
|
assert (row["executions"], row["errors"], row["messages"]) == (4, 1, 6)
|
|
assert len(row["spark"]) == 60
|
|
assert row["last_error_ts"] is not None
|
|
|
|
runs = client.get(f"{PREFIX}/runs", headers=superuser_token_headers).json()
|
|
assert any(run["id"] == "9-0" and run["status"] == "ok" for run in runs)
|
|
|
|
failures = client.get(f"{PREFIX}/events", headers=superuser_token_headers).json()
|
|
assert any("Traceback" in event["detail"] for event in failures)
|
|
assert all(event["type"] != "audit" for event in failures)
|
|
|
|
audit = client.get(
|
|
f"{PREFIX}/events", headers=superuser_token_headers, params={"kind": "audit"}
|
|
).json()
|
|
assert any(event["actor"] == "a@example.com" for event in audit)
|
|
|
|
dead = client.get(f"{PREFIX}/dead-letter", headers=superuser_token_headers)
|
|
assert dead.status_code == 200
|
|
assert isinstance(dead.json(), list)
|
|
|
|
|
|
def test_runs_narrow_to_one_minute(
|
|
client: TestClient, superuser_token_headers: dict[str, str], db: Session
|
|
) -> None:
|
|
"""A minute picked off a chart reaches past what the recent list holds."""
|
|
minute = datetime.now(timezone.utc).replace(second=0, microsecond=0) - timedelta(
|
|
hours=3
|
|
)
|
|
db.add(FlowRun(id="minute-in", flow=FLOW, started_at=minute, status="ok"))
|
|
db.add(
|
|
FlowRun(
|
|
id="minute-after",
|
|
flow=FLOW,
|
|
started_at=minute + timedelta(minutes=1),
|
|
status="ok",
|
|
)
|
|
)
|
|
db.commit()
|
|
|
|
runs = client.get(
|
|
f"{PREFIX}/runs",
|
|
headers=superuser_token_headers,
|
|
params={
|
|
"since": minute.isoformat(),
|
|
"until": (minute + timedelta(minutes=1)).isoformat(),
|
|
},
|
|
).json()
|
|
|
|
# The upper bound is exclusive, so the run starting the next minute is not
|
|
# in this one.
|
|
assert [run["id"] for run in runs] == ["minute-in"]
|