History is kept by flow id: runs, cascades, rollups and events file the id in their flow column and nodes as <flow id>.<node>, the API answers flow plus flow_name and shows and takes nodes as <key>.<node>; events carry flow_id from the moment they are published; run queue items, the stage cache and cached curves keep ids and keys apart; endpoints, the brain graph, device bindings and search name the id to open
This commit is contained in:
@@ -25,7 +25,7 @@ import traceback
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum
|
||||
from typing import Any, TypeGuard, cast
|
||||
|
||||
@@ -228,11 +228,13 @@ class RunContext:
|
||||
class Wiring:
|
||||
"""One published node and the qualified messages it is bound to."""
|
||||
|
||||
#: The flow's key, which the messages are qualified with, and its id.
|
||||
flow: str
|
||||
node: str
|
||||
type: str
|
||||
provides: list[str]
|
||||
requires: list[str]
|
||||
flow_id: str = ""
|
||||
|
||||
|
||||
#: Nothing held, as distinct from a node that yielded ``None``.
|
||||
@@ -977,10 +979,9 @@ class FlowController:
|
||||
async def reload_lib_users(self, ref: str) -> list[str]:
|
||||
"""Rebuild the flows running a shared node whose source has changed.
|
||||
|
||||
Returns their ids, which is what `usages` qualifies with.
|
||||
Returns their ids.
|
||||
"""
|
||||
usages = self.store.usages(ref, draft=False)
|
||||
flows = sorted({usage.split(".", 1)[0] for usage in usages})
|
||||
flows = self.store.lib_users(ref)
|
||||
for flow in flows:
|
||||
await self.reload_flow(flow)
|
||||
return flows
|
||||
@@ -1718,19 +1719,20 @@ class FlowController:
|
||||
)
|
||||
if provides or requires:
|
||||
index.append(
|
||||
Wiring(other.name, node.id, node.type, provides, requires)
|
||||
Wiring(
|
||||
other.name, node.id, node.type, provides, requires, other.id
|
||||
)
|
||||
)
|
||||
self._wiring_cache = (revision, index)
|
||||
return index
|
||||
|
||||
def cross_flow_nodes(
|
||||
self, flow: str
|
||||
) -> list[tuple[str, str, list[str], list[str]]]:
|
||||
def cross_flow_nodes(self, flow: str) -> list[Wiring]:
|
||||
"""Nodes in other flows on the far side of a dotted message name.
|
||||
|
||||
Returns ``(flow, node id, messages of `flow` it provides, ones it
|
||||
requires)``. This is what link-in and link-out mean: the wiring is
|
||||
already real — the canvas simply had no way to draw the other end.
|
||||
Returns the far node's `Wiring`, narrowed to the messages of `flow` it
|
||||
provides and requires. This is what link-in and link-out mean: the
|
||||
wiring is already real — the canvas simply had no way to draw the
|
||||
other end. *flow* is a key.
|
||||
"""
|
||||
prefix = f"{flow}."
|
||||
found = []
|
||||
@@ -1740,7 +1742,7 @@ class FlowController:
|
||||
provides = [name for name in entry.provides if name.startswith(prefix)]
|
||||
requires = [name for name in entry.requires if name.startswith(prefix)]
|
||||
if provides or requires:
|
||||
found.append((entry.flow, entry.node, provides, requires))
|
||||
found.append(replace(entry, provides=provides, requires=requires))
|
||||
return found
|
||||
|
||||
def message_node(
|
||||
@@ -1818,6 +1820,7 @@ class FlowController:
|
||||
group.members.append(member)
|
||||
if flow.name not in group.flows:
|
||||
group.flows.append(flow.name)
|
||||
group.flow_ids.append(flow.id)
|
||||
# A merged neuron stands for several nodes, so one of them being
|
||||
# unable to run is enough to mark it; the first reason wins.
|
||||
if group.issue is None:
|
||||
|
||||
@@ -979,7 +979,7 @@ class DashboardStore:
|
||||
return copy
|
||||
|
||||
def bindings_for(self, flow: str) -> Bindings:
|
||||
"""Every widget bound to a message of ``flow``.
|
||||
"""Every widget bound to a message of the flow with key ``flow``.
|
||||
|
||||
What the canvas draws as an endpoint: a control that sets one of this
|
||||
flow's messages, or a tile that shows one. Without this a dashboard is
|
||||
@@ -993,6 +993,8 @@ class DashboardStore:
|
||||
defn = DashboardDef.model_validate_json(path.read_text())
|
||||
except Exception:
|
||||
continue
|
||||
# By id, which is what opening it takes; the title is the label.
|
||||
dash_id = defn.id or path.parent.name
|
||||
# A bound setting is a consumer too — the dashboard itself reading
|
||||
# a message rather than any tile on it — so the canvas accounts for
|
||||
# it the same way. ``widget`` is what the endpoint id is built
|
||||
@@ -1003,7 +1005,7 @@ class DashboardStore:
|
||||
continue
|
||||
found.append(
|
||||
{
|
||||
"dashboard": defn.name,
|
||||
"dashboard": dash_id,
|
||||
"dashboard_title": defn.title or defn.name,
|
||||
"widget": f"settings.{name}",
|
||||
"title": f"{defn.title or defn.name} {name}",
|
||||
@@ -1020,7 +1022,7 @@ class DashboardStore:
|
||||
continue
|
||||
found.append(
|
||||
{
|
||||
"dashboard": defn.name,
|
||||
"dashboard": dash_id,
|
||||
"dashboard_title": defn.title or defn.name,
|
||||
"widget": widget.id,
|
||||
"title": widget.title or widget.id,
|
||||
|
||||
@@ -77,6 +77,7 @@ class Binding:
|
||||
"""One port of a device, wired to a message by a device node."""
|
||||
|
||||
device: str
|
||||
#: The flow's key, which ``message`` is qualified with.
|
||||
flow: str
|
||||
node: str
|
||||
port: str
|
||||
@@ -88,6 +89,8 @@ class Binding:
|
||||
dtype: str
|
||||
#: Passed through to the device unread.
|
||||
config: dict[str, Any] = field(default_factory=dict)
|
||||
#: The flow's id, which is where wiring the port is edited.
|
||||
flow_id: str = ""
|
||||
|
||||
|
||||
class DeviceIo(BaseModel):
|
||||
@@ -102,7 +105,9 @@ class DeviceIo(BaseModel):
|
||||
class DeviceBinding(BaseModel):
|
||||
"""Where a port is wired: the device node holding it."""
|
||||
|
||||
#: The flow's key, and its id to open it by.
|
||||
flow: str
|
||||
flow_id: str = ""
|
||||
node: str
|
||||
port: str
|
||||
message: str
|
||||
@@ -129,6 +134,7 @@ def bindings_of(flow: FlowDef) -> list[Binding]:
|
||||
Binding(
|
||||
device=device,
|
||||
flow=flow.name,
|
||||
flow_id=flow.id,
|
||||
node=node.id,
|
||||
port=spec.port,
|
||||
message=qualify(flow.name, spec.name),
|
||||
@@ -145,9 +151,10 @@ class DeviceDef(BaseModel):
|
||||
|
||||
id: str
|
||||
title: str = ""
|
||||
#: Ordered. The first one is what the device opens after pairing, and the
|
||||
#: rail follows this order. A name that no longer resolves is a dashboard
|
||||
#: someone deleted, and the device skips it.
|
||||
#: Dashboard ids, ordered. The first one is what the device opens after
|
||||
#: pairing, and the rail follows this order. An id that no longer resolves
|
||||
#: is a dashboard someone deleted, and the device skips it. A dashboard from
|
||||
#: before ids has its key as its id, so a list from then still holds.
|
||||
dashboards: list[str] = Field(default_factory=list)
|
||||
#: Bigger controls, a bigger rail and no hover states, for a screen that is
|
||||
#: touched rather than pointed at. A fact about the screen, not about any
|
||||
@@ -513,6 +520,7 @@ class DeviceStore:
|
||||
(
|
||||
DeviceBinding(
|
||||
flow=b.flow,
|
||||
flow_id=b.flow_id,
|
||||
node=b.node,
|
||||
port=b.port,
|
||||
message=b.message,
|
||||
@@ -1013,7 +1021,7 @@ def dashboards_for(
|
||||
) -> list[DashboardDef]:
|
||||
"""The published documents this device shows, in rail order.
|
||||
|
||||
Published, since that is what a device draws. A name that no longer
|
||||
Published, since that is what a device draws. An id that no longer
|
||||
resolves is skipped, and a device that is gone shows nothing.
|
||||
|
||||
Handed back whole rather than walked here, because what a device may do
|
||||
@@ -1022,7 +1030,7 @@ def dashboards_for(
|
||||
"""
|
||||
if device is None:
|
||||
return []
|
||||
return [defn for name in device.dashboards if (defn := store.published(name))]
|
||||
return [defn for dash_id in device.dashboards if (defn := store.published(dash_id))]
|
||||
|
||||
|
||||
def messages_of(defn: DashboardDef) -> set[str]:
|
||||
|
||||
@@ -13,7 +13,7 @@ import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from collections import deque
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Callable
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
|
||||
@@ -118,6 +118,11 @@ class EventBus:
|
||||
#: Subscribers that lost an event since they last asked. Touched on
|
||||
#: the event loop only, by the flush and the subscribers.
|
||||
self._dropped: set[asyncio.Queue[dict[str, Any]]] = set()
|
||||
#: The id of the flow holding a key, once the store exists. An event
|
||||
#: naming a flow by its key gets its ``flow_id`` here, on the way in,
|
||||
#: so what holds the event later is not left to look up a key that
|
||||
#: may have moved on by then.
|
||||
self.flow_id_of: Callable[[str], str] | None = None
|
||||
|
||||
def bind(self, loop: asyncio.AbstractEventLoop) -> None:
|
||||
"""Attach the bus to the running event loop (called once at startup)."""
|
||||
@@ -125,6 +130,11 @@ class EventBus:
|
||||
|
||||
def publish(self, event: dict[str, Any]) -> None:
|
||||
"""Publish an event from any thread."""
|
||||
resolve = self.flow_id_of
|
||||
if resolve is not None and "flow" in event and "flow_id" not in event:
|
||||
# An event about the engine as a whole names no flow, and no id.
|
||||
flow = str(event["flow"] or "")
|
||||
event["flow_id"] = resolve(flow) if flow else ""
|
||||
kind = event.get("type")
|
||||
if kind == "node_log":
|
||||
# deque.append is atomic, so worker threads need no lock here.
|
||||
|
||||
@@ -26,6 +26,7 @@ from fluksio.core.config import settings
|
||||
from fluksio.core.db import engine
|
||||
from fluksio.flow import tombstones
|
||||
from fluksio.flow.events import EventBus
|
||||
from fluksio.flow.messages import requalify
|
||||
from fluksio.models import EngineEvent, FlowRun, MetricBucket
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -84,6 +85,19 @@ def _minute(ts: float) -> int:
|
||||
return int(ts // 60) * 60
|
||||
|
||||
|
||||
def _filed(event: dict[str, Any], node: str = "") -> tuple[str, str]:
|
||||
"""The flow id and the ``<flow id>.<node>`` an event is kept under.
|
||||
|
||||
The event names both by the flow's key and brings the id along from the
|
||||
bus. One from a bus that knows no ids is kept under the key, which is
|
||||
what the id of a flow from before ids is.
|
||||
"""
|
||||
flow = str(event.get("flow") or "")
|
||||
flow_id = str(event.get("flow_id") or flow)
|
||||
node = node or str(event.get("node") or "")
|
||||
return flow_id[:NAME_MAX], requalify(node, flow, flow_id)[:NAME_MAX]
|
||||
|
||||
|
||||
def _detail(event: dict[str, Any]) -> str:
|
||||
text = str(event.get("error") or event.get("reason") or event.get("detail") or "")
|
||||
if event.get("type") == "cascade_dropped":
|
||||
@@ -154,11 +168,7 @@ class MetricsCollector:
|
||||
def _bucket(self, event: dict[str, Any]) -> dict[str, float]:
|
||||
# ponytail: one collector, one row per node per minute; coarsen the
|
||||
# bucket if the node count ever reaches thousands.
|
||||
key = (
|
||||
str(event.get("flow") or "")[:NAME_MAX],
|
||||
str(event.get("node") or "")[:NAME_MAX],
|
||||
_minute(float(event.get("ts") or time.time())),
|
||||
)
|
||||
key = (*_filed(event), _minute(float(event.get("ts") or time.time())))
|
||||
return self._buckets.setdefault(
|
||||
key,
|
||||
{
|
||||
@@ -216,12 +226,13 @@ class MetricsCollector:
|
||||
key = (str(event.get("flow") or ""), str(event.get("node") or ""))
|
||||
traceback = self._tracebacks.pop(key, (0.0, ""))[1]
|
||||
error = str(event.get("error") or "")
|
||||
flow_id, node = _filed(event)
|
||||
self._pending.append(
|
||||
EngineEvent(
|
||||
ts=datetime.fromtimestamp(ts, UTC),
|
||||
type="node_error",
|
||||
flow=str(event.get("flow") or ""),
|
||||
node=str(event.get("node") or ""),
|
||||
flow_id=flow_id,
|
||||
node=node,
|
||||
detail=(f"{error}\n{traceback}" if traceback else error)[
|
||||
:DETAIL_CAP
|
||||
],
|
||||
@@ -252,12 +263,13 @@ class MetricsCollector:
|
||||
|
||||
if kind == "node_health":
|
||||
if event.get("health") == "down":
|
||||
flow_id, node = _filed(event)
|
||||
self._pending.append(
|
||||
EngineEvent(
|
||||
ts=datetime.fromtimestamp(ts, UTC),
|
||||
type="node_health",
|
||||
flow=str(event.get("flow") or ""),
|
||||
node=str(event.get("node") or ""),
|
||||
flow_id=flow_id,
|
||||
node=node,
|
||||
detail=_detail(event) or "Reported itself down.",
|
||||
run=run_id,
|
||||
)
|
||||
@@ -269,7 +281,7 @@ class MetricsCollector:
|
||||
EngineEvent(
|
||||
ts=datetime.fromtimestamp(ts, UTC),
|
||||
type="audit",
|
||||
flow=str(event.get("flow") or ""),
|
||||
flow_id=_filed(event)[0],
|
||||
detail=str(event.get("action") or ""),
|
||||
actor=str(event.get("user") or ""),
|
||||
)
|
||||
@@ -277,12 +289,13 @@ class MetricsCollector:
|
||||
return
|
||||
|
||||
if kind in RECORDED:
|
||||
flow_id, node = _filed(event, str(event.get("task") or ""))
|
||||
self._pending.append(
|
||||
EngineEvent(
|
||||
ts=datetime.fromtimestamp(ts, UTC),
|
||||
type=kind,
|
||||
flow=str(event.get("flow") or ""),
|
||||
node=str(event.get("node") or event.get("task") or ""),
|
||||
flow_id=flow_id,
|
||||
node=node,
|
||||
detail=_detail(event),
|
||||
run=run_id,
|
||||
)
|
||||
@@ -307,7 +320,7 @@ class MetricsCollector:
|
||||
return
|
||||
self._runs[run_id] = {
|
||||
"id": run_id,
|
||||
"flow": str(event.get("flow") or "")[:NAME_MAX],
|
||||
"flow_id": _filed(event)[0],
|
||||
"source": str(event.get("cause") or ""),
|
||||
"parent": str(event.get("parent") or "")[:64],
|
||||
"started_ts": ts,
|
||||
@@ -417,12 +430,12 @@ class MetricsCollector:
|
||||
),
|
||||
[
|
||||
{
|
||||
"flow": flow,
|
||||
"flow_id": flow_id,
|
||||
"node": node,
|
||||
"bucket": datetime.fromtimestamp(minute, UTC),
|
||||
**agg,
|
||||
}
|
||||
for (flow, node, minute), agg in buckets.items()
|
||||
for (flow_id, node, minute), agg in buckets.items()
|
||||
],
|
||||
)
|
||||
|
||||
@@ -434,10 +447,11 @@ class MetricsCollector:
|
||||
session.execute(
|
||||
statement.on_conflict_do_update(
|
||||
index_elements=["id"],
|
||||
# Not the flow: a run is of the one it started in.
|
||||
set_={
|
||||
key: statement.excluded[key]
|
||||
for key in values[0]
|
||||
if key != "id"
|
||||
if key not in ("id", "flow_id")
|
||||
},
|
||||
),
|
||||
values,
|
||||
|
||||
@@ -49,9 +49,10 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlmodel import Session, col, select
|
||||
|
||||
from fluksio.core.db import engine as db_engine
|
||||
from fluksio.flow import tombstones
|
||||
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
|
||||
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
|
||||
from fluksio.flow.messages import qualify
|
||||
from fluksio.flow.messages import qualify, requalify
|
||||
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
|
||||
from fluksio.flow.queue import WorkItem, WorkQueue
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||
@@ -531,8 +532,14 @@ class MetricSink:
|
||||
batch: int = METRIC_BATCH,
|
||||
interval: float = METRIC_FLUSH_S,
|
||||
on_flush: Callable[[list[str]], None] | None = None,
|
||||
key: str = "",
|
||||
flow_id: str = "",
|
||||
) -> None:
|
||||
self.run_id = run_id
|
||||
#: The run's flow, by the key its nodes are named with and by the id
|
||||
#: a row names them with: ``<flow id>.<node>``.
|
||||
self._key = key
|
||||
self._flow_id = flow_id
|
||||
self._batch = batch
|
||||
self._interval = interval
|
||||
self._on_flush = on_flush
|
||||
@@ -562,7 +569,7 @@ class MetricSink:
|
||||
run_id=self.run_id,
|
||||
name=name[:128],
|
||||
step=step,
|
||||
node=node_id[:255],
|
||||
node=requalify(node_id, self._key, self._flow_id)[:255],
|
||||
ts=now,
|
||||
value=float(value),
|
||||
)
|
||||
@@ -802,8 +809,17 @@ class RunCache:
|
||||
not a hit: the reference would name bytes nobody can open.
|
||||
"""
|
||||
|
||||
def __init__(self, artifacts: ArtifactStore | None = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
artifacts: ArtifactStore | None = None,
|
||||
key_of: Callable[[str], str] | None = None,
|
||||
) -> None:
|
||||
self.artifacts = artifacts
|
||||
#: The key a source run's outputs are named with, from its flow's id:
|
||||
#: the flow's key now, or the last one it had if it is gone. Without
|
||||
#: one the id is taken for the key, which it is for a flow from before
|
||||
#: ids.
|
||||
self.key_of = key_of or (lambda flow_id: flow_id)
|
||||
|
||||
def lookup(self, key: str) -> CacheHit | None:
|
||||
if not key:
|
||||
@@ -829,7 +845,7 @@ class RunCache:
|
||||
continue
|
||||
source = session.get(Run, row.run_id)
|
||||
return CacheHit(
|
||||
flow=source.flow if source is not None else "",
|
||||
flow=self.key_of(source.flow_id) if source else "",
|
||||
outputs=outputs,
|
||||
# A row that was itself restored holds no series, so
|
||||
# follow its pointer rather than adding a hop to it.
|
||||
@@ -862,7 +878,9 @@ class RunService:
|
||||
self.controller = controller
|
||||
self.queue = queue
|
||||
self._artifacts = artifacts
|
||||
self._cache = RunCache(artifacts)
|
||||
self._cache = RunCache(
|
||||
artifacts, lambda flow_id: tombstones.name_of(flow_id, controller.store)
|
||||
)
|
||||
# Without one, a run gets a private in-memory state — which is exactly
|
||||
# the isolation it wants, minus surviving the process.
|
||||
self._state_factory = state_factory or (lambda _ns: MemoryState())
|
||||
@@ -912,10 +930,11 @@ class RunService:
|
||||
"""
|
||||
with Session(db_engine) as session:
|
||||
waiting = session.exec(
|
||||
select(Run.id, Run.flow).where(col(Run.status) == "queued")
|
||||
select(Run.id, Run.flow_id).where(col(Run.status) == "queued")
|
||||
).all()
|
||||
for run_id, flow in waiting:
|
||||
self.queue.add(WorkItem(kind="run", node="", flow=flow, run_id=run_id))
|
||||
# A run item names its flow by id, as `submit` queues it.
|
||||
for run_id, flow_id in waiting:
|
||||
self.queue.add(WorkItem(kind="run", node="", flow=flow_id, run_id=run_id))
|
||||
if waiting:
|
||||
logger.info(
|
||||
"Requeued %d run(s) left queued by an earlier engine", len(waiting)
|
||||
@@ -940,7 +959,7 @@ class RunService:
|
||||
|
||||
def submit(
|
||||
self,
|
||||
flow_name: str,
|
||||
flow_id: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
seed: int | None = None,
|
||||
group_id: str | None = None,
|
||||
@@ -953,10 +972,12 @@ class RunService:
|
||||
passed: list[str] | None = None,
|
||||
for_node: str | None = None,
|
||||
) -> Run:
|
||||
"""Journal a run and wake an engine up for it. Never blocks on it.
|
||||
"""Journal a run of the flow *flow_id* and wake an engine up for it.
|
||||
Never blocks on it.
|
||||
|
||||
``passed`` overrides which of ``params`` count as the caller's own, for
|
||||
a retry handing on its parent's whole params.
|
||||
a retry handing on its parent's whole params. ``for_node`` is a
|
||||
``<flow id>.<node>``.
|
||||
"""
|
||||
# Before anything else, including reading the flow: a caller retrying a
|
||||
# submit it never got an answer for is owed the run that answer was
|
||||
@@ -965,7 +986,7 @@ class RunService:
|
||||
existing = self._by_key(idempotency_key)
|
||||
if existing is not None:
|
||||
return existing
|
||||
flow = self.controller.store.read_flow(flow_name, draft=draft)
|
||||
flow = self.controller.store.read_flow(flow_id, draft=draft)
|
||||
issues = batch_issues(flow, self.controller.store, draft=draft)
|
||||
if issues:
|
||||
raise RunRejected(" ".join(issues))
|
||||
@@ -973,7 +994,7 @@ class RunService:
|
||||
|
||||
run = Run(
|
||||
id=new_run_id(),
|
||||
flow=flow.name,
|
||||
flow_id=flow.id,
|
||||
flow_version=flow.version,
|
||||
commit=self.controller.store.head(),
|
||||
origin_commit=origin_commit(flow),
|
||||
@@ -1011,7 +1032,8 @@ class RunService:
|
||||
return existing
|
||||
session.refresh(run)
|
||||
|
||||
self.queue.add(WorkItem(kind="run", node="", flow=flow.name, run_id=run.id))
|
||||
# By id, like the run: what the item names outlives a rename of the key.
|
||||
self.queue.add(WorkItem(kind="run", node="", flow=flow.id, run_id=run.id))
|
||||
return run
|
||||
|
||||
def _complete(
|
||||
@@ -1087,10 +1109,10 @@ class RunService:
|
||||
{"external_id": external_id, "id": existing, "created": False}
|
||||
)
|
||||
continue
|
||||
name = str(entry["flow"])
|
||||
if name not in flows:
|
||||
flows[name] = self.controller.store.read_flow(name)
|
||||
flow = flows[name]
|
||||
flow_id = str(entry["flow"])
|
||||
if flow_id not in flows:
|
||||
flows[flow_id] = self.controller.store.read_flow(flow_id)
|
||||
flow = flows[flow_id]
|
||||
try:
|
||||
params, seed, passed = self._complete(
|
||||
flow, entry["params"], entry["seed"]
|
||||
@@ -1100,7 +1122,7 @@ class RunService:
|
||||
started, finished = entry["started_at"], entry["finished_at"]
|
||||
run = Run(
|
||||
id=new_run_id(),
|
||||
flow=flow.name,
|
||||
flow_id=flow.id,
|
||||
flow_version=flow.version,
|
||||
params=params,
|
||||
passed=passed,
|
||||
@@ -1207,7 +1229,7 @@ class RunService:
|
||||
# The gate first, so nothing new is submitted while the running nodes
|
||||
# are being killed; a gated node is never handed to the executor, so
|
||||
# the graph drains instead of going further.
|
||||
pipeline.pause(self._flow_of(run_id) or "")
|
||||
pipeline.pause(self._key_of(run_id))
|
||||
workers = self.controller.workers
|
||||
if workers is not None:
|
||||
# Keyed by run, so a sweep cancelling one config leaves the others
|
||||
@@ -1239,7 +1261,7 @@ class RunService:
|
||||
raise RunRejected("That run has not finished; cancel it first")
|
||||
source = run.model_copy()
|
||||
return self.submit(
|
||||
source.flow,
|
||||
source.flow_id,
|
||||
params=dict(source.params),
|
||||
seed=source.seed,
|
||||
group_id=source.group_id,
|
||||
@@ -1254,10 +1276,11 @@ class RunService:
|
||||
for_node=source.for_node,
|
||||
)
|
||||
|
||||
def _flow_of(self, run_id: str) -> str | None:
|
||||
def _key_of(self, run_id: str) -> str:
|
||||
"""The key a run's pipeline runs its flow under, which it pauses by."""
|
||||
with Session(db_engine) as session:
|
||||
run = session.get(Run, run_id)
|
||||
return run.flow if run else None
|
||||
return tombstones.name_of(run.flow_id, self.controller.store) if run else ""
|
||||
|
||||
def _finish_unstarted(self, run_id: str, status: str) -> bool:
|
||||
"""Cancel a run nothing of has executed, if it is still in ``status``."""
|
||||
@@ -1464,23 +1487,31 @@ class RunService:
|
||||
state: StateBackend | None = None
|
||||
errors = 0
|
||||
|
||||
# The key its nodes and messages are named with, the id its rows are
|
||||
# filed under. Read from the store and its tombstones rather than the
|
||||
# flow, which may be gone before the run gets as far as reading it.
|
||||
key = tombstones.name_of(run.flow_id, self.controller.store)
|
||||
|
||||
def observe(outcome: NodeOutcome) -> None:
|
||||
nonlocal errors
|
||||
if not outcome.ok:
|
||||
errors += 1
|
||||
self._record_node(run_id, outcome)
|
||||
self._record_node(run, key, outcome)
|
||||
|
||||
sink = MetricSink(
|
||||
run_id,
|
||||
on_flush=lambda names: self._publish_event(
|
||||
{
|
||||
"type": "run_metric",
|
||||
"flow": run.flow,
|
||||
"flow": key,
|
||||
"flow_id": run.flow_id,
|
||||
"run": run_id,
|
||||
"names": names,
|
||||
"ts": time.time(),
|
||||
}
|
||||
),
|
||||
key=key,
|
||||
flow_id=run.flow_id,
|
||||
)
|
||||
try:
|
||||
with self._lock:
|
||||
@@ -1488,7 +1519,7 @@ class RunService:
|
||||
# building one loads every python node in a worker, and waits
|
||||
# for one while all of them are busy — for longer than a lease.
|
||||
self._active[run_id] = None
|
||||
flow = self.controller.store.read_flow(run.flow, draft=run.draft)
|
||||
flow = self.controller.store.read_flow(run.flow_id, draft=run.draft)
|
||||
# Read again, now that it is this run's turn: a sweep queues every
|
||||
# run at once, and the code on disk is free to move in the hours
|
||||
# before the last of them starts. What the record must name is the
|
||||
@@ -1518,7 +1549,7 @@ class RunService:
|
||||
status, reason = "cancelled", "Cancelled before it started"
|
||||
return
|
||||
self._active[run_id] = pipeline
|
||||
self._publish(run, "run_started")
|
||||
self._publish(run, key, "run_started")
|
||||
pipeline.run(seed_values(flow, run.params, run.seed))
|
||||
result = collect_result(flow, state)
|
||||
with self._lock:
|
||||
@@ -1542,7 +1573,7 @@ class RunService:
|
||||
duration = round((time.perf_counter() - started) * 1000, 2)
|
||||
self._finish(run_id, status, reason, result, duration)
|
||||
run.status = status
|
||||
self._publish(run, "run_finished")
|
||||
self._publish(run, key, "run_finished")
|
||||
self._release_cards(run)
|
||||
# Its values were only ever this run's; nothing reads them once it
|
||||
# has a result. On Redis the namespace would expire anyway.
|
||||
@@ -1582,11 +1613,15 @@ class RunService:
|
||||
except Exception:
|
||||
logger.warning("Could not retire the GPU workers of run %s", run.id)
|
||||
|
||||
def _record_node(self, run_id: str, outcome: NodeOutcome) -> None:
|
||||
def _record_node(self, run: Run, key: str, outcome: NodeOutcome) -> None:
|
||||
"""Write what one node did, under ``<flow id>.<node>``. The outputs
|
||||
and artifacts stay named by message, which is by the flow's *key*."""
|
||||
run_id = run.id
|
||||
node = requalify(outcome.node, key, run.flow_id)[:255]
|
||||
outputs = _cacheable(outcome)
|
||||
row = RunNode(
|
||||
run_id=run_id,
|
||||
node=outcome.node[:255],
|
||||
node=node,
|
||||
status="cached" if outcome.cached else ("ok" if outcome.ok else "error"),
|
||||
started_at=datetime.now(UTC),
|
||||
duration_ms=outcome.duration_ms,
|
||||
@@ -1614,7 +1649,7 @@ class RunService:
|
||||
run_id=run_id,
|
||||
name=message[:255],
|
||||
filename=str(ref.get("name") or "")[:255] or None,
|
||||
node=outcome.node[:255],
|
||||
node=node,
|
||||
digest=str(ref.get("digest") or "")[:71],
|
||||
size=int(ref.get("size") or 0),
|
||||
media_type=str(
|
||||
@@ -1667,11 +1702,12 @@ class RunService:
|
||||
except Exception:
|
||||
logger.exception("Could not close run %s", run_id)
|
||||
|
||||
def _publish(self, run: Run, kind: str) -> None:
|
||||
def _publish(self, run: Run, key: str, kind: str) -> None:
|
||||
self._publish_event(
|
||||
{
|
||||
"type": kind,
|
||||
"flow": run.flow,
|
||||
"flow": key,
|
||||
"flow_id": run.flow_id,
|
||||
"run": run.id,
|
||||
"status": run.status,
|
||||
"group": run.group_id or "",
|
||||
|
||||
@@ -529,7 +529,10 @@ class BrainNode(BaseModel):
|
||||
label: str
|
||||
kind: str
|
||||
members: list[str] = Field(default_factory=list)
|
||||
#: The keys of the flows its members sit in, and their ids in the same
|
||||
#: order, which is what opening one goes by.
|
||||
flows: list[str] = Field(default_factory=list)
|
||||
flow_ids: list[str] = Field(default_factory=list)
|
||||
issue: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
|
||||
@@ -29,12 +29,12 @@ import shutil
|
||||
import string
|
||||
import subprocess
|
||||
import threading
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
from fluksio.flow.schemas import FlowDef, _validate_name
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef, _validate_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fluksio.flow.tombstones import Tombstones
|
||||
@@ -603,23 +603,30 @@ class FlowStore:
|
||||
self._commit(f"Detach and delete shared node '{name}'")
|
||||
|
||||
def usages(self, lib_name: str, draft: bool = True) -> list[str]:
|
||||
"""Every ``flow.node`` pointing at this shared source.
|
||||
"""Every ``<key>.<node>`` pointing at this shared source.
|
||||
|
||||
Read from the drafts the editor shows, or with ``draft`` off from the
|
||||
published flows the engine runs.
|
||||
"""
|
||||
used = []
|
||||
for name in self.list_flows():
|
||||
return sorted(
|
||||
f"{flow.name}.{node.id}" for flow, node in self._lib_nodes(lib_name, draft)
|
||||
)
|
||||
|
||||
def lib_users(self, lib_name: str) -> list[str]:
|
||||
"""The ids of the published flows running this shared source."""
|
||||
return sorted({flow.id for flow, _ in self._lib_nodes(lib_name, False)})
|
||||
|
||||
def _lib_nodes(
|
||||
self, lib_name: str, draft: bool
|
||||
) -> Iterator[tuple[FlowDef, NodeDef]]:
|
||||
for flow_id in self.list_flows():
|
||||
try:
|
||||
definition = self.read_flow(name, draft=draft)
|
||||
definition = self.read_flow(flow_id, draft=draft)
|
||||
except FlowNotFound:
|
||||
continue
|
||||
used += [
|
||||
f"{name}.{node.id}"
|
||||
for node in definition.nodes
|
||||
if node.source_ref == lib_name
|
||||
]
|
||||
return sorted(used)
|
||||
for node in definition.nodes:
|
||||
if node.source_ref == lib_name:
|
||||
yield definition, node
|
||||
|
||||
def _set_source_ref(self, flow: str, node_id: str, ref: str | None) -> None:
|
||||
"""Point a node at a shared source, in the draft the editor is on."""
|
||||
|
||||
@@ -5,6 +5,10 @@ flow's id is opaque, so a tombstone keeps the key it last had, and `names_of`
|
||||
reads that back for anything showing the history. A flow deleted before
|
||||
tombstones existed has its old key as its id, which already reads right.
|
||||
|
||||
History names a node ``<flow id>.<node>``. The API shows and takes it as
|
||||
``<key>.<node>``, the way the engine names it, and `shown_nodes` and
|
||||
`stored_node` translate between the two.
|
||||
|
||||
The flow store is the one writer: it buries a flow before removing its files,
|
||||
asks before issuing an id, and lifts the tombstone of a flow restored under a
|
||||
pinned id once the restored document is on disk.
|
||||
@@ -35,20 +39,20 @@ def referenced(session: Session, flow_id: str) -> bool:
|
||||
queries = [
|
||||
select(Run.id).where(
|
||||
or_(
|
||||
col(Run.flow) == flow_id,
|
||||
col(Run.flow_id) == flow_id,
|
||||
col(Run.for_node).startswith(nodes, autoescape=True),
|
||||
)
|
||||
),
|
||||
select(FlowRun.id).where(col(FlowRun.flow) == flow_id),
|
||||
select(MetricBucket.flow).where(
|
||||
select(FlowRun.id).where(col(FlowRun.flow_id) == flow_id),
|
||||
select(MetricBucket.flow_id).where(
|
||||
or_(
|
||||
col(MetricBucket.flow) == flow_id,
|
||||
col(MetricBucket.flow_id) == flow_id,
|
||||
col(MetricBucket.node).startswith(nodes, autoescape=True),
|
||||
)
|
||||
),
|
||||
select(EngineEvent.id).where(
|
||||
or_(
|
||||
col(EngineEvent.flow) == flow_id,
|
||||
col(EngineEvent.flow_id) == flow_id,
|
||||
col(EngineEvent.node).startswith(nodes, autoescape=True),
|
||||
)
|
||||
),
|
||||
@@ -118,6 +122,44 @@ def name_of(flow_id: str, store: FlowStore) -> str:
|
||||
return names_of([flow_id], store)[flow_id]
|
||||
|
||||
|
||||
def id_of(key: str, store: FlowStore) -> str:
|
||||
"""The flow id behind *key*: the live flow's, else that of the one deleted
|
||||
flow that last held it, else *key* itself, as for history from before ids.
|
||||
"""
|
||||
live = store.key_index().get(key)
|
||||
if live is not None:
|
||||
return live
|
||||
with Session(engine) as session:
|
||||
graves = session.exec(
|
||||
select(FlowTombstone.id).where(col(FlowTombstone.name) == key).limit(2)
|
||||
).all()
|
||||
return graves[0] if len(graves) == 1 else key
|
||||
|
||||
|
||||
def shown_nodes(nodes: Iterable[str], store: FlowStore) -> dict[str, str]:
|
||||
"""Each stored ``<flow id>.<node>`` as ``<key>.<node>``, with one lookup."""
|
||||
split = {node: node.partition(".") for node in nodes}
|
||||
names = names_of({flow for flow, dot, _ in split.values() if dot}, store)
|
||||
return {
|
||||
node: f"{names[flow]}.{rest}" if dot else node
|
||||
for node, (flow, dot, rest) in split.items()
|
||||
}
|
||||
|
||||
|
||||
def stored_node(node: str, store: FlowStore, flow_id: str = "") -> str:
|
||||
"""A ``<key>.<node>`` as history stores it, ``<flow id>.<node>``.
|
||||
|
||||
With *flow_id*, a key that is that flow's own resolves to it, which is how
|
||||
a node of a deleted or renamed flow is still found by the name it shows.
|
||||
"""
|
||||
key, dot, rest = node.partition(".")
|
||||
if not dot:
|
||||
return node
|
||||
if flow_id and name_of(flow_id, store) == key:
|
||||
return f"{flow_id}.{rest}"
|
||||
return f"{id_of(key, store)}.{rest}"
|
||||
|
||||
|
||||
def prune(session: Session) -> int:
|
||||
"""Drop the tombstones no history row names any more. Returns how many."""
|
||||
stale = [
|
||||
|
||||
Reference in New Issue
Block a user