Superuser route to flush the metrics collector now

POST /observability/flush handles what the collector has queued and writes
it, so the soak harness no longer sleeps FLUSH_INTERVAL_S before clearing
its own rows.
This commit is contained in:
2026-10-15 09:53:19 +02:00
parent ae81d95fd7
commit 290b9fa705
8 changed files with 106 additions and 11 deletions
+22 -2
View File
@@ -19,12 +19,17 @@ from sqlalchemy import select as sa_select
from sqlmodel import col, select
from fluksio import __version__
from fluksio.api.deps import FlowControllerDep, SessionDep, get_current_user
from fluksio.api.deps import (
FlowControllerDep,
SessionDep,
get_current_active_superuser,
get_current_user,
)
from fluksio.api.routes.runs import elapsed_ms
from fluksio.core.config import settings
from fluksio.flow.controller import NodeStatus
from fluksio.flow.pipeline import ADVISORY_ISSUES
from fluksio.models import EngineEvent, FlowRun, MetricBucket
from fluksio.models import EngineEvent, FlowRun, Message, MetricBucket
router = APIRouter(
prefix="/observability",
@@ -508,3 +513,18 @@ async def read_dead_letters(
if controller.execution is None:
return []
return await run_in_threadpool(controller.execution.queue.dead_letters, limit)
@router.post(
"/flush",
response_model=Message,
dependencies=[Depends(get_current_active_superuser)],
)
async def flush_metrics(request: Request) -> Any:
"""Write what the metrics collector holds now, not at its next interval.
For tooling that reads or clears these tables right after causing events,
such as the soak harness clearing its own rows.
"""
await request.app.state.metrics_collector.flush_now()
return Message(message="Metrics flushed")
+17
View File
@@ -103,6 +103,7 @@ class MetricsCollector:
# held with the time it arrived so an unpaired one does not stay put.
self._tracebacks: dict[tuple[str, str], tuple[float, str]] = {}
self._last_prune = 0.0
self._queue: asyncio.Queue[dict[str, Any]] | None = None
# -------------------------------------------------------------------------
# The loop
@@ -112,6 +113,7 @@ class MetricsCollector:
"""Consume the bus until cancelled, flushing on a fixed interval."""
last = time.monotonic()
async with self._events.subscribe() as queue:
self._queue = queue
while True:
# A busy bus never idles, so the flush is on a deadline rather
# than on the timeout alone.
@@ -129,6 +131,21 @@ class MetricsCollector:
await self.flush()
last = time.monotonic()
async def flush_now(self) -> None:
"""Write what has been heard so far, without waiting for the interval.
Events the bus has handed over and the loop has not reached yet are
taken first, so whoever caused an event and then asks for a flush finds
it written.
"""
while self._queue is not None and not self._queue.empty():
event = self._queue.get_nowait()
try:
self.handle(event)
except Exception:
logger.exception("Could not record %s", event.get("type"))
await self.flush()
# -------------------------------------------------------------------------
# Accumulating
# -------------------------------------------------------------------------
+3 -1
View File
@@ -384,7 +384,9 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
_background(watchdog.run(), "loop-watchdog")
_background(alerts.run(), "alert-manager")
_background(MetricsCollector(event_bus).run(), "metrics-collector")
# On the app so a superuser can ask it to flush now.
app.state.metrics_collector = MetricsCollector(event_bus)
_background(app.state.metrics_collector.run(), "metrics-collector")
_background(_sweep_artifacts(artifacts, controller), "artifact-gc")
_background(_trim_volatile(volatile), "artifact-ring")
await controller.start()