Keep the engine's own history, and a screen that reads it
A second bus subscriber folds executions, errors, timings and queue lag into per-minute rollups, keeps failures with their traceback and an audit trail of who published what, and records one row per cascade — manual runs and previews included, under an id of their own that writes no idempotency markers. Read back through /observability/*, which always answers 200 so a degraded engine still renders its own health screen. Also fixes two things found on the way: node-health alerts read `status` where the engine publishes `health`, so a device dropping never alerted anyone, and the Redis queue reported `parked: 0` whatever was held. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017MeiWk3Yq12n2pTvnQWYvt
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,104 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import Session
|
||||
|
||||
from app.core.config import settings
|
||||
from app.models import EngineEvent, FlowRun, MetricBucket
|
||||
|
||||
PREFIX = f"{settings.API_V1_STR}/observability"
|
||||
FLOW = "observability-test"
|
||||
|
||||
|
||||
def _seed(db: Session) -> None:
|
||||
now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
|
||||
db.add(
|
||||
MetricBucket(
|
||||
flow=FLOW,
|
||||
node=f"{FLOW}.calc",
|
||||
bucket=now - timedelta(minutes=1),
|
||||
executions=4,
|
||||
errors=1,
|
||||
messages=6,
|
||||
duration_sum_ms=40.0,
|
||||
duration_max_ms=25.0,
|
||||
lag_sum_ms=100.0,
|
||||
lag_max_ms=60.0,
|
||||
items=4,
|
||||
)
|
||||
)
|
||||
db.add(
|
||||
EngineEvent(
|
||||
ts=now,
|
||||
type="node_error",
|
||||
flow=FLOW,
|
||||
node=f"{FLOW}.calc",
|
||||
detail="ValueError: bad input\nTraceback",
|
||||
)
|
||||
)
|
||||
db.add(
|
||||
EngineEvent(
|
||||
ts=now, type="audit", flow=FLOW, detail="published", actor="a@example.com"
|
||||
)
|
||||
)
|
||||
db.add(
|
||||
FlowRun(
|
||||
id="9-0",
|
||||
flow=FLOW,
|
||||
source="external",
|
||||
started_at=now,
|
||||
finished_at=now,
|
||||
status="ok",
|
||||
nodes=3,
|
||||
duration_ms=12.5,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_observability_requires_authentication(client: TestClient) -> None:
|
||||
assert client.get(f"{PREFIX}/summary").status_code == 401
|
||||
|
||||
|
||||
def test_the_summary_answers_even_when_degraded(
|
||||
client: TestClient, superuser_token_headers: dict[str, str]
|
||||
) -> None:
|
||||
response = client.get(f"{PREFIX}/summary", headers=superuser_token_headers)
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["status"] in {"ok", "degraded"}
|
||||
assert set(body["flows"]) == {"total", "running", "paused", "quarantined"}
|
||||
assert "error" in body["nodes"]
|
||||
|
||||
|
||||
def test_the_history_reads_back(
|
||||
client: TestClient, superuser_token_headers: dict[str, str], db: Session
|
||||
) -> None:
|
||||
_seed(db)
|
||||
|
||||
points = client.get(f"{PREFIX}/timeseries", headers=superuser_token_headers).json()
|
||||
mine = [point for point in points if point["executions"]]
|
||||
assert mine and mine[-1]["avg_ms"] > 0
|
||||
|
||||
flows = client.get(f"{PREFIX}/flows", headers=superuser_token_headers).json()
|
||||
row = next(entry for entry in flows if entry["flow"] == FLOW)
|
||||
assert (row["executions"], row["errors"], row["messages"]) == (4, 1, 6)
|
||||
assert len(row["spark"]) == 60
|
||||
assert row["last_error_ts"] is not None
|
||||
|
||||
runs = client.get(f"{PREFIX}/runs", headers=superuser_token_headers).json()
|
||||
assert any(run["id"] == "9-0" and run["status"] == "ok" for run in runs)
|
||||
|
||||
failures = client.get(f"{PREFIX}/events", headers=superuser_token_headers).json()
|
||||
assert any("Traceback" in event["detail"] for event in failures)
|
||||
assert all(event["type"] != "audit" for event in failures)
|
||||
|
||||
audit = client.get(
|
||||
f"{PREFIX}/events", headers=superuser_token_headers, params={"kind": "audit"}
|
||||
).json()
|
||||
assert any(event["actor"] == "a@example.com" for event in audit)
|
||||
|
||||
dead = client.get(f"{PREFIX}/dead-letter", headers=superuser_token_headers)
|
||||
assert dead.status_code == 200
|
||||
assert isinstance(dead.json(), list)
|
||||
@@ -122,11 +122,11 @@ def test_a_flapping_connection_goes_quiet():
|
||||
async def scenario():
|
||||
for _ in range(FLAP_THRESHOLD * 2 + 6):
|
||||
await alerts.handle(
|
||||
{"type": "node_health", "node": "heating.pump", "status": "down"}
|
||||
{"type": "node_health", "node": "heating.pump", "health": "down"}
|
||||
)
|
||||
clock.advance(5)
|
||||
await alerts.handle(
|
||||
{"type": "node_health", "node": "heating.pump", "status": "ok"}
|
||||
{"type": "node_health", "node": "heating.pump", "health": "ok"}
|
||||
)
|
||||
clock.advance(5)
|
||||
|
||||
@@ -185,7 +185,13 @@ def test_alerting_can_be_switched_off():
|
||||
"The work queue is unreachable",
|
||||
),
|
||||
({"type": "engine_degraded", "reason": "lag"}, "The engine is struggling"),
|
||||
({"type": "node_health", "status": "ok"}, None),
|
||||
({"type": "node_health", "health": "ok"}, None),
|
||||
# The engine publishes `health`, not `status`: reading the wrong key
|
||||
# meant a device dropping never alerted anyone.
|
||||
(
|
||||
{"type": "node_health", "node": "heating.pump", "health": "down"},
|
||||
"heating.pump lost its connection",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_every_alerting_event_reads_as_a_sentence(event, expected):
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
"""The collector writes down what the bus only ever broadcast.
|
||||
|
||||
Not under ``tests/flow`` with the rest of the engine: that package opts out of
|
||||
the database, and writing to it is this module's whole job.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from app.flow.events import EventBus
|
||||
from app.flow.metrics import MetricsCollector
|
||||
from app.models import EngineEvent, FlowRun, MetricBucket
|
||||
|
||||
FLOW = "metrics-test"
|
||||
NODE = "metrics-test.calc"
|
||||
|
||||
|
||||
def _events(collector: MetricsCollector, ts: float, run: str) -> None:
|
||||
collector.handle(
|
||||
{
|
||||
"type": "cascade_started",
|
||||
"run": run,
|
||||
"flow": FLOW,
|
||||
"node": "metrics-test.in",
|
||||
"cause": "external",
|
||||
"deliveries": 1,
|
||||
"ts": ts,
|
||||
}
|
||||
)
|
||||
collector.handle(
|
||||
{
|
||||
"type": "node_executed",
|
||||
"flow": FLOW,
|
||||
"node": NODE,
|
||||
"outputs": 2,
|
||||
"duration_ms": 5.0,
|
||||
"run": run,
|
||||
"ts": ts,
|
||||
}
|
||||
)
|
||||
collector.handle(
|
||||
{"type": "work_latency", "flow": FLOW, "node": NODE, "lag_ms": 12.0, "ts": ts}
|
||||
)
|
||||
# The traceback arrives as its own event, just before the failure.
|
||||
collector.handle(
|
||||
{
|
||||
"type": "node_log",
|
||||
"flow": FLOW,
|
||||
"node": NODE,
|
||||
"level": "error",
|
||||
"text": "Traceback: line 3, in run",
|
||||
"ts": ts,
|
||||
}
|
||||
)
|
||||
collector.handle(
|
||||
{
|
||||
"type": "node_error",
|
||||
"flow": FLOW,
|
||||
"node": NODE,
|
||||
"error": "ValueError: bad input",
|
||||
"run": run,
|
||||
"ts": ts,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_events_become_rollups_failures_runs_and_audit(db: Session) -> None:
|
||||
collector = MetricsCollector(EventBus())
|
||||
# The current minute, pinned: the second flush has to land in the same
|
||||
# bucket, and anything past the retention window is pruned on write.
|
||||
ts = datetime.now(timezone.utc).replace(second=0, microsecond=0).timestamp()
|
||||
_events(collector, ts, "1-0")
|
||||
collector.handle(
|
||||
{
|
||||
"type": "audit",
|
||||
"action": "published",
|
||||
"flow": FLOW,
|
||||
"user": "someone@example.com",
|
||||
"ts": ts,
|
||||
}
|
||||
)
|
||||
collector.handle(
|
||||
{"type": "cascade_finished", "run": "1-0", "flow": FLOW, "ts": ts + 0.5}
|
||||
)
|
||||
asyncio.run(collector.flush())
|
||||
|
||||
bucket = db.exec(
|
||||
select(MetricBucket).where(MetricBucket.flow == FLOW, MetricBucket.node == NODE)
|
||||
).one()
|
||||
assert (bucket.executions, bucket.errors, bucket.messages) == (1, 1, 2)
|
||||
assert bucket.duration_max_ms == 5.0
|
||||
assert (bucket.lag_max_ms, bucket.items) == (12.0, 1)
|
||||
|
||||
failure = db.exec(
|
||||
select(EngineEvent).where(
|
||||
EngineEvent.flow == FLOW, EngineEvent.type == "node_error"
|
||||
)
|
||||
).one()
|
||||
assert "ValueError: bad input" in failure.detail
|
||||
assert "line 3, in run" in failure.detail
|
||||
|
||||
audit = db.exec(
|
||||
select(EngineEvent).where(EngineEvent.flow == FLOW, EngineEvent.type == "audit")
|
||||
).one()
|
||||
assert (audit.actor, audit.detail) == ("someone@example.com", "published")
|
||||
|
||||
run = db.exec(select(FlowRun).where(FlowRun.id == "1-0")).one()
|
||||
assert (run.status, run.flow, run.source) == ("error", FLOW, "external")
|
||||
assert (run.nodes, run.errors) == (1, 1)
|
||||
assert run.duration_ms > 0
|
||||
|
||||
# The same minute, written again: the counters add rather than duplicate.
|
||||
_events(collector, ts + 10, "2-0")
|
||||
asyncio.run(collector.flush())
|
||||
db.expire_all()
|
||||
bucket = db.exec(
|
||||
select(MetricBucket).where(MetricBucket.flow == FLOW, MetricBucket.node == NODE)
|
||||
).one()
|
||||
assert (bucket.executions, bucket.errors) == (2, 2)
|
||||
|
||||
# Never finished, so it is still open — the prune is what closes it.
|
||||
open_run = db.exec(select(FlowRun).where(FlowRun.id == "2-0")).one()
|
||||
assert open_run.status == "running"
|
||||
|
||||
|
||||
def test_a_run_that_did_not_fail_reads_ok(db: Session) -> None:
|
||||
collector = MetricsCollector(EventBus())
|
||||
ts = time.time()
|
||||
collector.handle(
|
||||
{
|
||||
"type": "cascade_started",
|
||||
"run": "manual-abc",
|
||||
"flow": FLOW,
|
||||
"node": "metrics-test.in",
|
||||
"cause": "manual",
|
||||
"deliveries": 1,
|
||||
"ts": ts,
|
||||
}
|
||||
)
|
||||
collector.handle(
|
||||
{"type": "cascade_finished", "run": "manual-abc", "flow": FLOW, "ts": ts + 0.1}
|
||||
)
|
||||
asyncio.run(collector.flush())
|
||||
|
||||
run = db.exec(select(FlowRun).where(FlowRun.id == "manual-abc")).one()
|
||||
assert (run.status, run.source) == ("ok", "manual")
|
||||
Reference in New Issue
Block a user