Superuser route to flush the metrics collector now
POST /observability/flush handles what the collector has queued and writes it, so the soak harness no longer sleeps FLUSH_INTERVAL_S before clearing its own rows.
This commit is contained in:
@@ -19,12 +19,17 @@ from sqlalchemy import select as sa_select
|
||||
from sqlmodel import col, select
|
||||
|
||||
from fluksio import __version__
|
||||
from fluksio.api.deps import FlowControllerDep, SessionDep, get_current_user
|
||||
from fluksio.api.deps import (
|
||||
FlowControllerDep,
|
||||
SessionDep,
|
||||
get_current_active_superuser,
|
||||
get_current_user,
|
||||
)
|
||||
from fluksio.api.routes.runs import elapsed_ms
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.flow.controller import NodeStatus
|
||||
from fluksio.flow.pipeline import ADVISORY_ISSUES
|
||||
from fluksio.models import EngineEvent, FlowRun, MetricBucket
|
||||
from fluksio.models import EngineEvent, FlowRun, Message, MetricBucket
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/observability",
|
||||
@@ -508,3 +513,18 @@ async def read_dead_letters(
|
||||
if controller.execution is None:
|
||||
return []
|
||||
return await run_in_threadpool(controller.execution.queue.dead_letters, limit)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/flush",
|
||||
response_model=Message,
|
||||
dependencies=[Depends(get_current_active_superuser)],
|
||||
)
|
||||
async def flush_metrics(request: Request) -> Any:
|
||||
"""Write what the metrics collector holds now, not at its next interval.
|
||||
|
||||
For tooling that reads or clears these tables right after causing events,
|
||||
such as the soak harness clearing its own rows.
|
||||
"""
|
||||
await request.app.state.metrics_collector.flush_now()
|
||||
return Message(message="Metrics flushed")
|
||||
|
||||
@@ -103,6 +103,7 @@ class MetricsCollector:
|
||||
# held with the time it arrived so an unpaired one does not stay put.
|
||||
self._tracebacks: dict[tuple[str, str], tuple[float, str]] = {}
|
||||
self._last_prune = 0.0
|
||||
self._queue: asyncio.Queue[dict[str, Any]] | None = None
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# The loop
|
||||
@@ -112,6 +113,7 @@ class MetricsCollector:
|
||||
"""Consume the bus until cancelled, flushing on a fixed interval."""
|
||||
last = time.monotonic()
|
||||
async with self._events.subscribe() as queue:
|
||||
self._queue = queue
|
||||
while True:
|
||||
# A busy bus never idles, so the flush is on a deadline rather
|
||||
# than on the timeout alone.
|
||||
@@ -129,6 +131,21 @@ class MetricsCollector:
|
||||
await self.flush()
|
||||
last = time.monotonic()
|
||||
|
||||
async def flush_now(self) -> None:
|
||||
"""Write what has been heard so far, without waiting for the interval.
|
||||
|
||||
Events the bus has handed over and the loop has not reached yet are
|
||||
taken first, so whoever caused an event and then asks for a flush finds
|
||||
it written.
|
||||
"""
|
||||
while self._queue is not None and not self._queue.empty():
|
||||
event = self._queue.get_nowait()
|
||||
try:
|
||||
self.handle(event)
|
||||
except Exception:
|
||||
logger.exception("Could not record %s", event.get("type"))
|
||||
await self.flush()
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Accumulating
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@@ -384,7 +384,9 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
|
||||
_background(watchdog.run(), "loop-watchdog")
|
||||
_background(alerts.run(), "alert-manager")
|
||||
_background(MetricsCollector(event_bus).run(), "metrics-collector")
|
||||
# On the app so a superuser can ask it to flush now.
|
||||
app.state.metrics_collector = MetricsCollector(event_bus)
|
||||
_background(app.state.metrics_collector.run(), "metrics-collector")
|
||||
_background(_sweep_artifacts(artifacts, controller), "artifact-gc")
|
||||
_background(_trim_volatile(volatile), "artifact-ring")
|
||||
await controller.start()
|
||||
|
||||
Reference in New Issue
Block a user