History is kept by flow id: runs, cascades, rollups and events file the id in their flow column and nodes as <flow id>.<node>, the API answers flow plus flow_name and shows and takes nodes as <key>.<node>; events carry flow_id from the moment they are published; run queue items, the stage cache and cached curves keep ids and keys apart; endpoints, the brain graph, device bindings and search name the id to open

This commit is contained in:
2026-11-02 19:01:19 +01:00
parent e6027af8fe
commit d4a1b1db47
34 changed files with 1374 additions and 329 deletions
+15 -12
View File
@@ -25,7 +25,7 @@ import traceback
from collections import Counter
from collections.abc import Callable
from contextlib import suppress
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from enum import Enum
from typing import Any, TypeGuard, cast
@@ -228,11 +228,13 @@ class RunContext:
class Wiring:
"""One published node and the qualified messages it is bound to."""
#: The flow's key, which the messages are qualified with, and its id.
flow: str
node: str
type: str
provides: list[str]
requires: list[str]
flow_id: str = ""
#: Nothing held, as distinct from a node that yielded ``None``.
@@ -977,10 +979,9 @@ class FlowController:
async def reload_lib_users(self, ref: str) -> list[str]:
"""Rebuild the flows running a shared node whose source has changed.
Returns their ids, which is what `usages` qualifies with.
Returns their ids.
"""
usages = self.store.usages(ref, draft=False)
flows = sorted({usage.split(".", 1)[0] for usage in usages})
flows = self.store.lib_users(ref)
for flow in flows:
await self.reload_flow(flow)
return flows
@@ -1718,19 +1719,20 @@ class FlowController:
)
if provides or requires:
index.append(
Wiring(other.name, node.id, node.type, provides, requires)
Wiring(
other.name, node.id, node.type, provides, requires, other.id
)
)
self._wiring_cache = (revision, index)
return index
def cross_flow_nodes(
self, flow: str
) -> list[tuple[str, str, list[str], list[str]]]:
def cross_flow_nodes(self, flow: str) -> list[Wiring]:
"""Nodes in other flows on the far side of a dotted message name.
Returns ``(flow, node id, messages of `flow` it provides, ones it
requires)``. This is what link-in and link-out mean: the wiring is
already real — the canvas simply had no way to draw the other end.
Returns the far node's `Wiring`, narrowed to the messages of `flow` it
provides and requires. This is what link-in and link-out mean: the
wiring is already real — the canvas simply had no way to draw the
other end. *flow* is a key.
"""
prefix = f"{flow}."
found = []
@@ -1740,7 +1742,7 @@ class FlowController:
provides = [name for name in entry.provides if name.startswith(prefix)]
requires = [name for name in entry.requires if name.startswith(prefix)]
if provides or requires:
found.append((entry.flow, entry.node, provides, requires))
found.append(replace(entry, provides=provides, requires=requires))
return found
def message_node(
@@ -1818,6 +1820,7 @@ class FlowController:
group.members.append(member)
if flow.name not in group.flows:
group.flows.append(flow.name)
group.flow_ids.append(flow.id)
# A merged neuron stands for several nodes, so one of them being
# unable to run is enough to mark it; the first reason wins.
if group.issue is None:
+5 -3
View File
@@ -979,7 +979,7 @@ class DashboardStore:
return copy
def bindings_for(self, flow: str) -> Bindings:
"""Every widget bound to a message of ``flow``.
"""Every widget bound to a message of the flow with key ``flow``.
What the canvas draws as an endpoint: a control that sets one of this
flow's messages, or a tile that shows one. Without this a dashboard is
@@ -993,6 +993,8 @@ class DashboardStore:
defn = DashboardDef.model_validate_json(path.read_text())
except Exception:
continue
# By id, which is what opening it takes; the title is the label.
dash_id = defn.id or path.parent.name
# A bound setting is a consumer too — the dashboard itself reading
# a message rather than any tile on it — so the canvas accounts for
# it the same way. ``widget`` is what the endpoint id is built
@@ -1003,7 +1005,7 @@ class DashboardStore:
continue
found.append(
{
"dashboard": defn.name,
"dashboard": dash_id,
"dashboard_title": defn.title or defn.name,
"widget": f"settings.{name}",
"title": f"{defn.title or defn.name} {name}",
@@ -1020,7 +1022,7 @@ class DashboardStore:
continue
found.append(
{
"dashboard": defn.name,
"dashboard": dash_id,
"dashboard_title": defn.title or defn.name,
"widget": widget.id,
"title": widget.title or widget.id,
+13 -5
View File
@@ -77,6 +77,7 @@ class Binding:
"""One port of a device, wired to a message by a device node."""
device: str
#: The flow's key, which ``message`` is qualified with.
flow: str
node: str
port: str
@@ -88,6 +89,8 @@ class Binding:
dtype: str
#: Passed through to the device unread.
config: dict[str, Any] = field(default_factory=dict)
#: The flow's id, which is where wiring the port is edited.
flow_id: str = ""
class DeviceIo(BaseModel):
@@ -102,7 +105,9 @@ class DeviceIo(BaseModel):
class DeviceBinding(BaseModel):
"""Where a port is wired: the device node holding it."""
#: The flow's key, and its id to open it by.
flow: str
flow_id: str = ""
node: str
port: str
message: str
@@ -129,6 +134,7 @@ def bindings_of(flow: FlowDef) -> list[Binding]:
Binding(
device=device,
flow=flow.name,
flow_id=flow.id,
node=node.id,
port=spec.port,
message=qualify(flow.name, spec.name),
@@ -145,9 +151,10 @@ class DeviceDef(BaseModel):
id: str
title: str = ""
#: Ordered. The first one is what the device opens after pairing, and the
#: rail follows this order. A name that no longer resolves is a dashboard
#: someone deleted, and the device skips it.
#: Dashboard ids, ordered. The first one is what the device opens after
#: pairing, and the rail follows this order. An id that no longer resolves
#: is a dashboard someone deleted, and the device skips it. A dashboard from
#: before ids has its key as its id, so a list from then still holds.
dashboards: list[str] = Field(default_factory=list)
#: Bigger controls, a bigger rail and no hover states, for a screen that is
#: touched rather than pointed at. A fact about the screen, not about any
@@ -513,6 +520,7 @@ class DeviceStore:
(
DeviceBinding(
flow=b.flow,
flow_id=b.flow_id,
node=b.node,
port=b.port,
message=b.message,
@@ -1013,7 +1021,7 @@ def dashboards_for(
) -> list[DashboardDef]:
"""The published documents this device shows, in rail order.
Published, since that is what a device draws. A name that no longer
Published, since that is what a device draws. An id that no longer
resolves is skipped, and a device that is gone shows nothing.
Handed back whole rather than walked here, because what a device may do
@@ -1022,7 +1030,7 @@ def dashboards_for(
"""
if device is None:
return []
return [defn for name in device.dashboards if (defn := store.published(name))]
return [defn for dash_id in device.dashboards if (defn := store.published(dash_id))]
def messages_of(defn: DashboardDef) -> set[str]:
+11 -1
View File
@@ -13,7 +13,7 @@ import asyncio
import logging
import threading
from collections import deque
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Callable
from contextlib import asynccontextmanager
from typing import Any
@@ -118,6 +118,11 @@ class EventBus:
#: Subscribers that lost an event since they last asked. Touched on
#: the event loop only, by the flush and the subscribers.
self._dropped: set[asyncio.Queue[dict[str, Any]]] = set()
#: The id of the flow holding a key, once the store exists. An event
#: naming a flow by its key gets its ``flow_id`` here, on the way in,
#: so what holds the event later is not left to look up a key that
#: may have moved on by then.
self.flow_id_of: Callable[[str], str] | None = None
def bind(self, loop: asyncio.AbstractEventLoop) -> None:
"""Attach the bus to the running event loop (called once at startup)."""
@@ -125,6 +130,11 @@ class EventBus:
def publish(self, event: dict[str, Any]) -> None:
"""Publish an event from any thread."""
resolve = self.flow_id_of
if resolve is not None and "flow" in event and "flow_id" not in event:
# An event about the engine as a whole names no flow, and no id.
flow = str(event["flow"] or "")
event["flow_id"] = resolve(flow) if flow else ""
kind = event.get("type")
if kind == "node_log":
# deque.append is atomic, so worker threads need no lock here.
+30 -16
View File
@@ -26,6 +26,7 @@ from fluksio.core.config import settings
from fluksio.core.db import engine
from fluksio.flow import tombstones
from fluksio.flow.events import EventBus
from fluksio.flow.messages import requalify
from fluksio.models import EngineEvent, FlowRun, MetricBucket
logger = logging.getLogger(__name__)
@@ -84,6 +85,19 @@ def _minute(ts: float) -> int:
return int(ts // 60) * 60
def _filed(event: dict[str, Any], node: str = "") -> tuple[str, str]:
"""The flow id and the ``<flow id>.<node>`` an event is kept under.
The event names both by the flow's key and brings the id along from the
bus. One from a bus that knows no ids is kept under the key, which is
what the id of a flow from before ids is.
"""
flow = str(event.get("flow") or "")
flow_id = str(event.get("flow_id") or flow)
node = node or str(event.get("node") or "")
return flow_id[:NAME_MAX], requalify(node, flow, flow_id)[:NAME_MAX]
def _detail(event: dict[str, Any]) -> str:
text = str(event.get("error") or event.get("reason") or event.get("detail") or "")
if event.get("type") == "cascade_dropped":
@@ -154,11 +168,7 @@ class MetricsCollector:
def _bucket(self, event: dict[str, Any]) -> dict[str, float]:
# ponytail: one collector, one row per node per minute; coarsen the
# bucket if the node count ever reaches thousands.
key = (
str(event.get("flow") or "")[:NAME_MAX],
str(event.get("node") or "")[:NAME_MAX],
_minute(float(event.get("ts") or time.time())),
)
key = (*_filed(event), _minute(float(event.get("ts") or time.time())))
return self._buckets.setdefault(
key,
{
@@ -216,12 +226,13 @@ class MetricsCollector:
key = (str(event.get("flow") or ""), str(event.get("node") or ""))
traceback = self._tracebacks.pop(key, (0.0, ""))[1]
error = str(event.get("error") or "")
flow_id, node = _filed(event)
self._pending.append(
EngineEvent(
ts=datetime.fromtimestamp(ts, UTC),
type="node_error",
flow=str(event.get("flow") or ""),
node=str(event.get("node") or ""),
flow_id=flow_id,
node=node,
detail=(f"{error}\n{traceback}" if traceback else error)[
:DETAIL_CAP
],
@@ -252,12 +263,13 @@ class MetricsCollector:
if kind == "node_health":
if event.get("health") == "down":
flow_id, node = _filed(event)
self._pending.append(
EngineEvent(
ts=datetime.fromtimestamp(ts, UTC),
type="node_health",
flow=str(event.get("flow") or ""),
node=str(event.get("node") or ""),
flow_id=flow_id,
node=node,
detail=_detail(event) or "Reported itself down.",
run=run_id,
)
@@ -269,7 +281,7 @@ class MetricsCollector:
EngineEvent(
ts=datetime.fromtimestamp(ts, UTC),
type="audit",
flow=str(event.get("flow") or ""),
flow_id=_filed(event)[0],
detail=str(event.get("action") or ""),
actor=str(event.get("user") or ""),
)
@@ -277,12 +289,13 @@ class MetricsCollector:
return
if kind in RECORDED:
flow_id, node = _filed(event, str(event.get("task") or ""))
self._pending.append(
EngineEvent(
ts=datetime.fromtimestamp(ts, UTC),
type=kind,
flow=str(event.get("flow") or ""),
node=str(event.get("node") or event.get("task") or ""),
flow_id=flow_id,
node=node,
detail=_detail(event),
run=run_id,
)
@@ -307,7 +320,7 @@ class MetricsCollector:
return
self._runs[run_id] = {
"id": run_id,
"flow": str(event.get("flow") or "")[:NAME_MAX],
"flow_id": _filed(event)[0],
"source": str(event.get("cause") or ""),
"parent": str(event.get("parent") or "")[:64],
"started_ts": ts,
@@ -417,12 +430,12 @@ class MetricsCollector:
),
[
{
"flow": flow,
"flow_id": flow_id,
"node": node,
"bucket": datetime.fromtimestamp(minute, UTC),
**agg,
}
for (flow, node, minute), agg in buckets.items()
for (flow_id, node, minute), agg in buckets.items()
],
)
@@ -434,10 +447,11 @@ class MetricsCollector:
session.execute(
statement.on_conflict_do_update(
index_elements=["id"],
# Not the flow: a run is of the one it started in.
set_={
key: statement.excluded[key]
for key in values[0]
if key != "id"
if key not in ("id", "flow_id")
},
),
values,
+69 -33
View File
@@ -49,9 +49,10 @@ from sqlalchemy.exc import IntegrityError
from sqlmodel import Session, col, select
from fluksio.core.db import engine as db_engine
from fluksio.flow import tombstones
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
from fluksio.flow.messages import qualify
from fluksio.flow.messages import qualify, requalify
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
from fluksio.flow.queue import WorkItem, WorkQueue
from fluksio.flow.schemas import FlowDef, NodeDef
@@ -531,8 +532,14 @@ class MetricSink:
batch: int = METRIC_BATCH,
interval: float = METRIC_FLUSH_S,
on_flush: Callable[[list[str]], None] | None = None,
key: str = "",
flow_id: str = "",
) -> None:
self.run_id = run_id
#: The run's flow, by the key its nodes are named with and by the id
#: a row names them with: ``<flow id>.<node>``.
self._key = key
self._flow_id = flow_id
self._batch = batch
self._interval = interval
self._on_flush = on_flush
@@ -562,7 +569,7 @@ class MetricSink:
run_id=self.run_id,
name=name[:128],
step=step,
node=node_id[:255],
node=requalify(node_id, self._key, self._flow_id)[:255],
ts=now,
value=float(value),
)
@@ -802,8 +809,17 @@ class RunCache:
not a hit: the reference would name bytes nobody can open.
"""
def __init__(self, artifacts: ArtifactStore | None = None) -> None:
def __init__(
self,
artifacts: ArtifactStore | None = None,
key_of: Callable[[str], str] | None = None,
) -> None:
self.artifacts = artifacts
#: The key a source run's outputs are named with, from its flow's id:
#: the flow's key now, or the last one it had if it is gone. Without
#: one the id is taken for the key, which it is for a flow from before
#: ids.
self.key_of = key_of or (lambda flow_id: flow_id)
def lookup(self, key: str) -> CacheHit | None:
if not key:
@@ -829,7 +845,7 @@ class RunCache:
continue
source = session.get(Run, row.run_id)
return CacheHit(
flow=source.flow if source is not None else "",
flow=self.key_of(source.flow_id) if source else "",
outputs=outputs,
# A row that was itself restored holds no series, so
# follow its pointer rather than adding a hop to it.
@@ -862,7 +878,9 @@ class RunService:
self.controller = controller
self.queue = queue
self._artifacts = artifacts
self._cache = RunCache(artifacts)
self._cache = RunCache(
artifacts, lambda flow_id: tombstones.name_of(flow_id, controller.store)
)
# Without one, a run gets a private in-memory state — which is exactly
# the isolation it wants, minus surviving the process.
self._state_factory = state_factory or (lambda _ns: MemoryState())
@@ -912,10 +930,11 @@ class RunService:
"""
with Session(db_engine) as session:
waiting = session.exec(
select(Run.id, Run.flow).where(col(Run.status) == "queued")
select(Run.id, Run.flow_id).where(col(Run.status) == "queued")
).all()
for run_id, flow in waiting:
self.queue.add(WorkItem(kind="run", node="", flow=flow, run_id=run_id))
# A run item names its flow by id, as `submit` queues it.
for run_id, flow_id in waiting:
self.queue.add(WorkItem(kind="run", node="", flow=flow_id, run_id=run_id))
if waiting:
logger.info(
"Requeued %d run(s) left queued by an earlier engine", len(waiting)
@@ -940,7 +959,7 @@ class RunService:
def submit(
self,
flow_name: str,
flow_id: str,
params: dict[str, Any] | None = None,
seed: int | None = None,
group_id: str | None = None,
@@ -953,10 +972,12 @@ class RunService:
passed: list[str] | None = None,
for_node: str | None = None,
) -> Run:
"""Journal a run and wake an engine up for it. Never blocks on it.
"""Journal a run of the flow *flow_id* and wake an engine up for it.
Never blocks on it.
``passed`` overrides which of ``params`` count as the caller's own, for
a retry handing on its parent's whole params.
a retry handing on its parent's whole params. ``for_node`` is a
``<flow id>.<node>``.
"""
# Before anything else, including reading the flow: a caller retrying a
# submit it never got an answer for is owed the run that answer was
@@ -965,7 +986,7 @@ class RunService:
existing = self._by_key(idempotency_key)
if existing is not None:
return existing
flow = self.controller.store.read_flow(flow_name, draft=draft)
flow = self.controller.store.read_flow(flow_id, draft=draft)
issues = batch_issues(flow, self.controller.store, draft=draft)
if issues:
raise RunRejected(" ".join(issues))
@@ -973,7 +994,7 @@ class RunService:
run = Run(
id=new_run_id(),
flow=flow.name,
flow_id=flow.id,
flow_version=flow.version,
commit=self.controller.store.head(),
origin_commit=origin_commit(flow),
@@ -1011,7 +1032,8 @@ class RunService:
return existing
session.refresh(run)
self.queue.add(WorkItem(kind="run", node="", flow=flow.name, run_id=run.id))
# By id, like the run: what the item names outlives a rename of the key.
self.queue.add(WorkItem(kind="run", node="", flow=flow.id, run_id=run.id))
return run
def _complete(
@@ -1087,10 +1109,10 @@ class RunService:
{"external_id": external_id, "id": existing, "created": False}
)
continue
name = str(entry["flow"])
if name not in flows:
flows[name] = self.controller.store.read_flow(name)
flow = flows[name]
flow_id = str(entry["flow"])
if flow_id not in flows:
flows[flow_id] = self.controller.store.read_flow(flow_id)
flow = flows[flow_id]
try:
params, seed, passed = self._complete(
flow, entry["params"], entry["seed"]
@@ -1100,7 +1122,7 @@ class RunService:
started, finished = entry["started_at"], entry["finished_at"]
run = Run(
id=new_run_id(),
flow=flow.name,
flow_id=flow.id,
flow_version=flow.version,
params=params,
passed=passed,
@@ -1207,7 +1229,7 @@ class RunService:
# The gate first, so nothing new is submitted while the running nodes
# are being killed; a gated node is never handed to the executor, so
# the graph drains instead of going further.
pipeline.pause(self._flow_of(run_id) or "")
pipeline.pause(self._key_of(run_id))
workers = self.controller.workers
if workers is not None:
# Keyed by run, so a sweep cancelling one config leaves the others
@@ -1239,7 +1261,7 @@ class RunService:
raise RunRejected("That run has not finished; cancel it first")
source = run.model_copy()
return self.submit(
source.flow,
source.flow_id,
params=dict(source.params),
seed=source.seed,
group_id=source.group_id,
@@ -1254,10 +1276,11 @@ class RunService:
for_node=source.for_node,
)
def _flow_of(self, run_id: str) -> str | None:
def _key_of(self, run_id: str) -> str:
"""The key a run's pipeline runs its flow under, which it pauses by."""
with Session(db_engine) as session:
run = session.get(Run, run_id)
return run.flow if run else None
return tombstones.name_of(run.flow_id, self.controller.store) if run else ""
def _finish_unstarted(self, run_id: str, status: str) -> bool:
"""Cancel a run nothing of has executed, if it is still in ``status``."""
@@ -1464,23 +1487,31 @@ class RunService:
state: StateBackend | None = None
errors = 0
# The key its nodes and messages are named with, the id its rows are
# filed under. Read from the store and its tombstones rather than the
# flow, which may be gone before the run gets as far as reading it.
key = tombstones.name_of(run.flow_id, self.controller.store)
def observe(outcome: NodeOutcome) -> None:
nonlocal errors
if not outcome.ok:
errors += 1
self._record_node(run_id, outcome)
self._record_node(run, key, outcome)
sink = MetricSink(
run_id,
on_flush=lambda names: self._publish_event(
{
"type": "run_metric",
"flow": run.flow,
"flow": key,
"flow_id": run.flow_id,
"run": run_id,
"names": names,
"ts": time.time(),
}
),
key=key,
flow_id=run.flow_id,
)
try:
with self._lock:
@@ -1488,7 +1519,7 @@ class RunService:
# building one loads every python node in a worker, and waits
# for one while all of them are busy — for longer than a lease.
self._active[run_id] = None
flow = self.controller.store.read_flow(run.flow, draft=run.draft)
flow = self.controller.store.read_flow(run.flow_id, draft=run.draft)
# Read again, now that it is this run's turn: a sweep queues every
# run at once, and the code on disk is free to move in the hours
# before the last of them starts. What the record must name is the
@@ -1518,7 +1549,7 @@ class RunService:
status, reason = "cancelled", "Cancelled before it started"
return
self._active[run_id] = pipeline
self._publish(run, "run_started")
self._publish(run, key, "run_started")
pipeline.run(seed_values(flow, run.params, run.seed))
result = collect_result(flow, state)
with self._lock:
@@ -1542,7 +1573,7 @@ class RunService:
duration = round((time.perf_counter() - started) * 1000, 2)
self._finish(run_id, status, reason, result, duration)
run.status = status
self._publish(run, "run_finished")
self._publish(run, key, "run_finished")
self._release_cards(run)
# Its values were only ever this run's; nothing reads them once it
# has a result. On Redis the namespace would expire anyway.
@@ -1582,11 +1613,15 @@ class RunService:
except Exception:
logger.warning("Could not retire the GPU workers of run %s", run.id)
def _record_node(self, run_id: str, outcome: NodeOutcome) -> None:
def _record_node(self, run: Run, key: str, outcome: NodeOutcome) -> None:
"""Write what one node did, under ``<flow id>.<node>``. The outputs
and artifacts stay named by message, which is by the flow's *key*."""
run_id = run.id
node = requalify(outcome.node, key, run.flow_id)[:255]
outputs = _cacheable(outcome)
row = RunNode(
run_id=run_id,
node=outcome.node[:255],
node=node,
status="cached" if outcome.cached else ("ok" if outcome.ok else "error"),
started_at=datetime.now(UTC),
duration_ms=outcome.duration_ms,
@@ -1614,7 +1649,7 @@ class RunService:
run_id=run_id,
name=message[:255],
filename=str(ref.get("name") or "")[:255] or None,
node=outcome.node[:255],
node=node,
digest=str(ref.get("digest") or "")[:71],
size=int(ref.get("size") or 0),
media_type=str(
@@ -1667,11 +1702,12 @@ class RunService:
except Exception:
logger.exception("Could not close run %s", run_id)
def _publish(self, run: Run, kind: str) -> None:
def _publish(self, run: Run, key: str, kind: str) -> None:
self._publish_event(
{
"type": kind,
"flow": run.flow,
"flow": key,
"flow_id": run.flow_id,
"run": run.id,
"status": run.status,
"group": run.group_id or "",
+3
View File
@@ -529,7 +529,10 @@ class BrainNode(BaseModel):
label: str
kind: str
members: list[str] = Field(default_factory=list)
#: The keys of the flows its members sit in, and their ids in the same
#: order, which is what opening one goes by.
flows: list[str] = Field(default_factory=list)
flow_ids: list[str] = Field(default_factory=list)
issue: str | None = Field(
default=None,
description=(
+19 -12
View File
@@ -29,12 +29,12 @@ import shutil
import string
import subprocess
import threading
from collections.abc import Callable, Iterable
from collections.abc import Callable, Iterable, Iterator
from datetime import UTC, datetime
from pathlib import Path
from typing import TYPE_CHECKING, Literal
from fluksio.flow.schemas import FlowDef, _validate_name
from fluksio.flow.schemas import FlowDef, NodeDef, _validate_name
if TYPE_CHECKING:
from fluksio.flow.tombstones import Tombstones
@@ -603,23 +603,30 @@ class FlowStore:
self._commit(f"Detach and delete shared node '{name}'")
def usages(self, lib_name: str, draft: bool = True) -> list[str]:
"""Every ``flow.node`` pointing at this shared source.
"""Every ``<key>.<node>`` pointing at this shared source.
Read from the drafts the editor shows, or with ``draft`` off from the
published flows the engine runs.
"""
used = []
for name in self.list_flows():
return sorted(
f"{flow.name}.{node.id}" for flow, node in self._lib_nodes(lib_name, draft)
)
def lib_users(self, lib_name: str) -> list[str]:
"""The ids of the published flows running this shared source."""
return sorted({flow.id for flow, _ in self._lib_nodes(lib_name, False)})
def _lib_nodes(
self, lib_name: str, draft: bool
) -> Iterator[tuple[FlowDef, NodeDef]]:
for flow_id in self.list_flows():
try:
definition = self.read_flow(name, draft=draft)
definition = self.read_flow(flow_id, draft=draft)
except FlowNotFound:
continue
used += [
f"{name}.{node.id}"
for node in definition.nodes
if node.source_ref == lib_name
]
return sorted(used)
for node in definition.nodes:
if node.source_ref == lib_name:
yield definition, node
def _set_source_ref(self, flow: str, node_id: str, ref: str | None) -> None:
"""Point a node at a shared source, in the draft the editor is on."""
+47 -5
View File
@@ -5,6 +5,10 @@ flow's id is opaque, so a tombstone keeps the key it last had, and `names_of`
reads that back for anything showing the history. A flow deleted before
tombstones existed has its old key as its id, which already reads right.
History names a node ``<flow id>.<node>``. The API shows and takes it as
``<key>.<node>``, the way the engine names it, and `shown_nodes` and
`stored_node` translate between the two.
The flow store is the one writer: it buries a flow before removing its files,
asks before issuing an id, and lifts the tombstone of a flow restored under a
pinned id once the restored document is on disk.
@@ -35,20 +39,20 @@ def referenced(session: Session, flow_id: str) -> bool:
queries = [
select(Run.id).where(
or_(
col(Run.flow) == flow_id,
col(Run.flow_id) == flow_id,
col(Run.for_node).startswith(nodes, autoescape=True),
)
),
select(FlowRun.id).where(col(FlowRun.flow) == flow_id),
select(MetricBucket.flow).where(
select(FlowRun.id).where(col(FlowRun.flow_id) == flow_id),
select(MetricBucket.flow_id).where(
or_(
col(MetricBucket.flow) == flow_id,
col(MetricBucket.flow_id) == flow_id,
col(MetricBucket.node).startswith(nodes, autoescape=True),
)
),
select(EngineEvent.id).where(
or_(
col(EngineEvent.flow) == flow_id,
col(EngineEvent.flow_id) == flow_id,
col(EngineEvent.node).startswith(nodes, autoescape=True),
)
),
@@ -118,6 +122,44 @@ def name_of(flow_id: str, store: FlowStore) -> str:
return names_of([flow_id], store)[flow_id]
def id_of(key: str, store: FlowStore) -> str:
"""The flow id behind *key*: the live flow's, else that of the one deleted
flow that last held it, else *key* itself, as for history from before ids.
"""
live = store.key_index().get(key)
if live is not None:
return live
with Session(engine) as session:
graves = session.exec(
select(FlowTombstone.id).where(col(FlowTombstone.name) == key).limit(2)
).all()
return graves[0] if len(graves) == 1 else key
def shown_nodes(nodes: Iterable[str], store: FlowStore) -> dict[str, str]:
"""Each stored ``<flow id>.<node>`` as ``<key>.<node>``, with one lookup."""
split = {node: node.partition(".") for node in nodes}
names = names_of({flow for flow, dot, _ in split.values() if dot}, store)
return {
node: f"{names[flow]}.{rest}" if dot else node
for node, (flow, dot, rest) in split.items()
}
def stored_node(node: str, store: FlowStore, flow_id: str = "") -> str:
"""A ``<key>.<node>`` as history stores it, ``<flow id>.<node>``.
With *flow_id*, a key that is that flow's own resolves to it, which is how
a node of a deleted or renamed flow is still found by the name it shows.
"""
key, dot, rest = node.partition(".")
if not dot:
return node
if flow_id and name_of(flow_id, store) == key:
return f"{flow_id}.{rest}"
return f"{id_of(key, store)}.{rest}"
def prune(session: Session) -> int:
"""Drop the tombstones no history row names any more. Returns how many."""
stale = [