History is kept by flow id: runs, cascades, rollups and events file the id in their flow column and nodes as <flow id>.<node>, the API answers flow plus flow_name and shows and takes nodes as <key>.<node>; events carry flow_id from the moment they are published; run queue items, the stage cache and cached curves keep ids and keys apart; endpoints, the brain graph, device bindings and search name the id to open

This commit is contained in:
2026-11-02 19:01:19 +01:00
parent e6027af8fe
commit d4a1b1db47
34 changed files with 1374 additions and 329 deletions
+30 -16
View File
@@ -26,6 +26,7 @@ from fluksio.core.config import settings
from fluksio.core.db import engine
from fluksio.flow import tombstones
from fluksio.flow.events import EventBus
from fluksio.flow.messages import requalify
from fluksio.models import EngineEvent, FlowRun, MetricBucket
logger = logging.getLogger(__name__)
@@ -84,6 +85,19 @@ def _minute(ts: float) -> int:
return int(ts // 60) * 60
def _filed(event: dict[str, Any], node: str = "") -> tuple[str, str]:
"""The flow id and the ``<flow id>.<node>`` an event is kept under.
The event names both by the flow's key and brings the id along from the
bus. One from a bus that knows no ids is kept under the key, which is
what the id of a flow from before ids is.
"""
flow = str(event.get("flow") or "")
flow_id = str(event.get("flow_id") or flow)
node = node or str(event.get("node") or "")
return flow_id[:NAME_MAX], requalify(node, flow, flow_id)[:NAME_MAX]
def _detail(event: dict[str, Any]) -> str:
text = str(event.get("error") or event.get("reason") or event.get("detail") or "")
if event.get("type") == "cascade_dropped":
@@ -154,11 +168,7 @@ class MetricsCollector:
def _bucket(self, event: dict[str, Any]) -> dict[str, float]:
# ponytail: one collector, one row per node per minute; coarsen the
# bucket if the node count ever reaches thousands.
key = (
str(event.get("flow") or "")[:NAME_MAX],
str(event.get("node") or "")[:NAME_MAX],
_minute(float(event.get("ts") or time.time())),
)
key = (*_filed(event), _minute(float(event.get("ts") or time.time())))
return self._buckets.setdefault(
key,
{
@@ -216,12 +226,13 @@ class MetricsCollector:
key = (str(event.get("flow") or ""), str(event.get("node") or ""))
traceback = self._tracebacks.pop(key, (0.0, ""))[1]
error = str(event.get("error") or "")
flow_id, node = _filed(event)
self._pending.append(
EngineEvent(
ts=datetime.fromtimestamp(ts, UTC),
type="node_error",
flow=str(event.get("flow") or ""),
node=str(event.get("node") or ""),
flow_id=flow_id,
node=node,
detail=(f"{error}\n{traceback}" if traceback else error)[
:DETAIL_CAP
],
@@ -252,12 +263,13 @@ class MetricsCollector:
if kind == "node_health":
if event.get("health") == "down":
flow_id, node = _filed(event)
self._pending.append(
EngineEvent(
ts=datetime.fromtimestamp(ts, UTC),
type="node_health",
flow=str(event.get("flow") or ""),
node=str(event.get("node") or ""),
flow_id=flow_id,
node=node,
detail=_detail(event) or "Reported itself down.",
run=run_id,
)
@@ -269,7 +281,7 @@ class MetricsCollector:
EngineEvent(
ts=datetime.fromtimestamp(ts, UTC),
type="audit",
flow=str(event.get("flow") or ""),
flow_id=_filed(event)[0],
detail=str(event.get("action") or ""),
actor=str(event.get("user") or ""),
)
@@ -277,12 +289,13 @@ class MetricsCollector:
return
if kind in RECORDED:
flow_id, node = _filed(event, str(event.get("task") or ""))
self._pending.append(
EngineEvent(
ts=datetime.fromtimestamp(ts, UTC),
type=kind,
flow=str(event.get("flow") or ""),
node=str(event.get("node") or event.get("task") or ""),
flow_id=flow_id,
node=node,
detail=_detail(event),
run=run_id,
)
@@ -307,7 +320,7 @@ class MetricsCollector:
return
self._runs[run_id] = {
"id": run_id,
"flow": str(event.get("flow") or "")[:NAME_MAX],
"flow_id": _filed(event)[0],
"source": str(event.get("cause") or ""),
"parent": str(event.get("parent") or "")[:64],
"started_ts": ts,
@@ -417,12 +430,12 @@ class MetricsCollector:
),
[
{
"flow": flow,
"flow_id": flow_id,
"node": node,
"bucket": datetime.fromtimestamp(minute, UTC),
**agg,
}
for (flow, node, minute), agg in buckets.items()
for (flow_id, node, minute), agg in buckets.items()
],
)
@@ -434,10 +447,11 @@ class MetricsCollector:
session.execute(
statement.on_conflict_do_update(
index_elements=["id"],
# Not the flow: a run is of the one it started in.
set_={
key: statement.excluded[key]
for key in values[0]
if key != "id"
if key not in ("id", "flow_id")
},
),
values,