from datetime import datetime, timedelta, timezone from fastapi.testclient import TestClient from sqlmodel import Session from app.core.config import settings from app.models import EngineEvent, FlowRun, MetricBucket PREFIX = f"{settings.API_V1_STR}/observability" FLOW = "observability-test" def _seed(db: Session) -> None: now = datetime.now(timezone.utc).replace(second=0, microsecond=0) db.add( MetricBucket( flow=FLOW, node=f"{FLOW}.calc", bucket=now - timedelta(minutes=1), executions=4, errors=1, messages=6, duration_sum_ms=40.0, duration_max_ms=25.0, lag_sum_ms=100.0, lag_max_ms=60.0, items=4, ) ) db.add( EngineEvent( ts=now, type="node_error", flow=FLOW, node=f"{FLOW}.calc", detail="ValueError: bad input\nTraceback", ) ) db.add( EngineEvent( ts=now, type="audit", flow=FLOW, detail="published", actor="a@example.com" ) ) db.add( FlowRun( id="9-0", flow=FLOW, source="external", started_at=now, finished_at=now, status="ok", nodes=3, duration_ms=12.5, ) ) db.commit() def test_observability_requires_authentication(client: TestClient) -> None: assert client.get(f"{PREFIX}/summary").status_code == 401 def test_the_summary_answers_even_when_degraded( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: response = client.get(f"{PREFIX}/summary", headers=superuser_token_headers) assert response.status_code == 200 body = response.json() assert body["status"] in {"ok", "degraded"} assert set(body["flows"]) == {"total", "running", "paused", "quarantined"} assert "error" in body["nodes"] def test_the_history_reads_back( client: TestClient, superuser_token_headers: dict[str, str], db: Session ) -> None: _seed(db) points = client.get(f"{PREFIX}/timeseries", headers=superuser_token_headers).json() mine = [point for point in points if point["executions"]] assert mine and mine[-1]["avg_ms"] > 0 flows = client.get(f"{PREFIX}/flows", headers=superuser_token_headers).json() row = next(entry for entry in flows if entry["flow"] == FLOW) assert (row["executions"], row["errors"], row["messages"]) == (4, 1, 6) assert len(row["spark"]) == 60 assert row["last_error_ts"] is not None runs = client.get(f"{PREFIX}/runs", headers=superuser_token_headers).json() assert any(run["id"] == "9-0" and run["status"] == "ok" for run in runs) failures = client.get(f"{PREFIX}/events", headers=superuser_token_headers).json() assert any("Traceback" in event["detail"] for event in failures) assert all(event["type"] != "audit" for event in failures) audit = client.get( f"{PREFIX}/events", headers=superuser_token_headers, params={"kind": "audit"} ).json() assert any(event["actor"] == "a@example.com" for event in audit) dead = client.get(f"{PREFIX}/dead-letter", headers=superuser_token_headers) assert dead.status_code == 200 assert isinstance(dead.json(), list) def test_runs_narrow_to_one_minute( client: TestClient, superuser_token_headers: dict[str, str], db: Session ) -> None: """A minute picked off a chart reaches past what the recent list holds.""" minute = datetime.now(timezone.utc).replace(second=0, microsecond=0) - timedelta( hours=3 ) db.add(FlowRun(id="minute-in", flow=FLOW, started_at=minute, status="ok")) db.add( FlowRun( id="minute-after", flow=FLOW, started_at=minute + timedelta(minutes=1), status="ok", ) ) db.commit() runs = client.get( f"{PREFIX}/runs", headers=superuser_token_headers, params={ "since": minute.isoformat(), "until": (minute + timedelta(minutes=1)).isoformat(), }, ).json() # The upper bound is exclusive, so the run starting the next minute is not # in this one. assert [run["id"] for run in runs] == ["minute-in"]