History is kept by flow id: runs, cascades, rollups and events file the id in their flow column and nodes as <flow id>.<node>, the API answers flow plus flow_name and shows and takes nodes as <key>.<node>; events carry flow_id from the moment they are published; run queue items, the stage cache and cached curves keep ids and keys apart; endpoints, the brain graph, device bindings and search name the id to open
This commit is contained in:
@@ -26,6 +26,7 @@ from fluksio.core.config import settings
|
||||
from fluksio.core.db import engine
|
||||
from fluksio.flow import tombstones
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.messages import requalify
|
||||
from fluksio.models import EngineEvent, FlowRun, MetricBucket
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -84,6 +85,19 @@ def _minute(ts: float) -> int:
|
||||
return int(ts // 60) * 60
|
||||
|
||||
|
||||
def _filed(event: dict[str, Any], node: str = "") -> tuple[str, str]:
|
||||
"""The flow id and the ``<flow id>.<node>`` an event is kept under.
|
||||
|
||||
The event names both by the flow's key and brings the id along from the
|
||||
bus. One from a bus that knows no ids is kept under the key, which is
|
||||
what the id of a flow from before ids is.
|
||||
"""
|
||||
flow = str(event.get("flow") or "")
|
||||
flow_id = str(event.get("flow_id") or flow)
|
||||
node = node or str(event.get("node") or "")
|
||||
return flow_id[:NAME_MAX], requalify(node, flow, flow_id)[:NAME_MAX]
|
||||
|
||||
|
||||
def _detail(event: dict[str, Any]) -> str:
|
||||
text = str(event.get("error") or event.get("reason") or event.get("detail") or "")
|
||||
if event.get("type") == "cascade_dropped":
|
||||
@@ -154,11 +168,7 @@ class MetricsCollector:
|
||||
def _bucket(self, event: dict[str, Any]) -> dict[str, float]:
|
||||
# ponytail: one collector, one row per node per minute; coarsen the
|
||||
# bucket if the node count ever reaches thousands.
|
||||
key = (
|
||||
str(event.get("flow") or "")[:NAME_MAX],
|
||||
str(event.get("node") or "")[:NAME_MAX],
|
||||
_minute(float(event.get("ts") or time.time())),
|
||||
)
|
||||
key = (*_filed(event), _minute(float(event.get("ts") or time.time())))
|
||||
return self._buckets.setdefault(
|
||||
key,
|
||||
{
|
||||
@@ -216,12 +226,13 @@ class MetricsCollector:
|
||||
key = (str(event.get("flow") or ""), str(event.get("node") or ""))
|
||||
traceback = self._tracebacks.pop(key, (0.0, ""))[1]
|
||||
error = str(event.get("error") or "")
|
||||
flow_id, node = _filed(event)
|
||||
self._pending.append(
|
||||
EngineEvent(
|
||||
ts=datetime.fromtimestamp(ts, UTC),
|
||||
type="node_error",
|
||||
flow=str(event.get("flow") or ""),
|
||||
node=str(event.get("node") or ""),
|
||||
flow_id=flow_id,
|
||||
node=node,
|
||||
detail=(f"{error}\n{traceback}" if traceback else error)[
|
||||
:DETAIL_CAP
|
||||
],
|
||||
@@ -252,12 +263,13 @@ class MetricsCollector:
|
||||
|
||||
if kind == "node_health":
|
||||
if event.get("health") == "down":
|
||||
flow_id, node = _filed(event)
|
||||
self._pending.append(
|
||||
EngineEvent(
|
||||
ts=datetime.fromtimestamp(ts, UTC),
|
||||
type="node_health",
|
||||
flow=str(event.get("flow") or ""),
|
||||
node=str(event.get("node") or ""),
|
||||
flow_id=flow_id,
|
||||
node=node,
|
||||
detail=_detail(event) or "Reported itself down.",
|
||||
run=run_id,
|
||||
)
|
||||
@@ -269,7 +281,7 @@ class MetricsCollector:
|
||||
EngineEvent(
|
||||
ts=datetime.fromtimestamp(ts, UTC),
|
||||
type="audit",
|
||||
flow=str(event.get("flow") or ""),
|
||||
flow_id=_filed(event)[0],
|
||||
detail=str(event.get("action") or ""),
|
||||
actor=str(event.get("user") or ""),
|
||||
)
|
||||
@@ -277,12 +289,13 @@ class MetricsCollector:
|
||||
return
|
||||
|
||||
if kind in RECORDED:
|
||||
flow_id, node = _filed(event, str(event.get("task") or ""))
|
||||
self._pending.append(
|
||||
EngineEvent(
|
||||
ts=datetime.fromtimestamp(ts, UTC),
|
||||
type=kind,
|
||||
flow=str(event.get("flow") or ""),
|
||||
node=str(event.get("node") or event.get("task") or ""),
|
||||
flow_id=flow_id,
|
||||
node=node,
|
||||
detail=_detail(event),
|
||||
run=run_id,
|
||||
)
|
||||
@@ -307,7 +320,7 @@ class MetricsCollector:
|
||||
return
|
||||
self._runs[run_id] = {
|
||||
"id": run_id,
|
||||
"flow": str(event.get("flow") or "")[:NAME_MAX],
|
||||
"flow_id": _filed(event)[0],
|
||||
"source": str(event.get("cause") or ""),
|
||||
"parent": str(event.get("parent") or "")[:64],
|
||||
"started_ts": ts,
|
||||
@@ -417,12 +430,12 @@ class MetricsCollector:
|
||||
),
|
||||
[
|
||||
{
|
||||
"flow": flow,
|
||||
"flow_id": flow_id,
|
||||
"node": node,
|
||||
"bucket": datetime.fromtimestamp(minute, UTC),
|
||||
**agg,
|
||||
}
|
||||
for (flow, node, minute), agg in buckets.items()
|
||||
for (flow_id, node, minute), agg in buckets.items()
|
||||
],
|
||||
)
|
||||
|
||||
@@ -434,10 +447,11 @@ class MetricsCollector:
|
||||
session.execute(
|
||||
statement.on_conflict_do_update(
|
||||
index_elements=["id"],
|
||||
# Not the flow: a run is of the one it started in.
|
||||
set_={
|
||||
key: statement.excluded[key]
|
||||
for key in values[0]
|
||||
if key != "id"
|
||||
if key not in ("id", "flow_id")
|
||||
},
|
||||
),
|
||||
values,
|
||||
|
||||
Reference in New Issue
Block a user