History is kept by flow id: runs, cascades, rollups and events file the id in their flow column and nodes as <flow id>.<node>, the API answers flow plus flow_name and shows and takes nodes as <key>.<node>; events carry flow_id from the moment they are published; run queue items, the stage cache and cached curves keep ids and keys apart; endpoints, the brain graph, device bindings and search name the id to open

This commit is contained in:
2026-11-02 19:01:19 +01:00
parent e6027af8fe
commit d4a1b1db47
34 changed files with 1374 additions and 329 deletions
+69 -33
View File
@@ -49,9 +49,10 @@ from sqlalchemy.exc import IntegrityError
from sqlmodel import Session, col, select
from fluksio.core.db import engine as db_engine
from fluksio.flow import tombstones
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
from fluksio.flow.messages import qualify
from fluksio.flow.messages import qualify, requalify
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
from fluksio.flow.queue import WorkItem, WorkQueue
from fluksio.flow.schemas import FlowDef, NodeDef
@@ -531,8 +532,14 @@ class MetricSink:
batch: int = METRIC_BATCH,
interval: float = METRIC_FLUSH_S,
on_flush: Callable[[list[str]], None] | None = None,
key: str = "",
flow_id: str = "",
) -> None:
self.run_id = run_id
#: The run's flow, by the key its nodes are named with and by the id
#: a row names them with: ``<flow id>.<node>``.
self._key = key
self._flow_id = flow_id
self._batch = batch
self._interval = interval
self._on_flush = on_flush
@@ -562,7 +569,7 @@ class MetricSink:
run_id=self.run_id,
name=name[:128],
step=step,
node=node_id[:255],
node=requalify(node_id, self._key, self._flow_id)[:255],
ts=now,
value=float(value),
)
@@ -802,8 +809,17 @@ class RunCache:
not a hit: the reference would name bytes nobody can open.
"""
def __init__(self, artifacts: ArtifactStore | None = None) -> None:
def __init__(
self,
artifacts: ArtifactStore | None = None,
key_of: Callable[[str], str] | None = None,
) -> None:
self.artifacts = artifacts
#: The key a source run's outputs are named with, from its flow's id:
#: the flow's key now, or the last one it had if it is gone. Without
#: one the id is taken for the key, which it is for a flow from before
#: ids.
self.key_of = key_of or (lambda flow_id: flow_id)
def lookup(self, key: str) -> CacheHit | None:
if not key:
@@ -829,7 +845,7 @@ class RunCache:
continue
source = session.get(Run, row.run_id)
return CacheHit(
flow=source.flow if source is not None else "",
flow=self.key_of(source.flow_id) if source else "",
outputs=outputs,
# A row that was itself restored holds no series, so
# follow its pointer rather than adding a hop to it.
@@ -862,7 +878,9 @@ class RunService:
self.controller = controller
self.queue = queue
self._artifacts = artifacts
self._cache = RunCache(artifacts)
self._cache = RunCache(
artifacts, lambda flow_id: tombstones.name_of(flow_id, controller.store)
)
# Without one, a run gets a private in-memory state — which is exactly
# the isolation it wants, minus surviving the process.
self._state_factory = state_factory or (lambda _ns: MemoryState())
@@ -912,10 +930,11 @@ class RunService:
"""
with Session(db_engine) as session:
waiting = session.exec(
select(Run.id, Run.flow).where(col(Run.status) == "queued")
select(Run.id, Run.flow_id).where(col(Run.status) == "queued")
).all()
for run_id, flow in waiting:
self.queue.add(WorkItem(kind="run", node="", flow=flow, run_id=run_id))
# A run item names its flow by id, as `submit` queues it.
for run_id, flow_id in waiting:
self.queue.add(WorkItem(kind="run", node="", flow=flow_id, run_id=run_id))
if waiting:
logger.info(
"Requeued %d run(s) left queued by an earlier engine", len(waiting)
@@ -940,7 +959,7 @@ class RunService:
def submit(
self,
flow_name: str,
flow_id: str,
params: dict[str, Any] | None = None,
seed: int | None = None,
group_id: str | None = None,
@@ -953,10 +972,12 @@ class RunService:
passed: list[str] | None = None,
for_node: str | None = None,
) -> Run:
"""Journal a run and wake an engine up for it. Never blocks on it.
"""Journal a run of the flow *flow_id* and wake an engine up for it.
Never blocks on it.
``passed`` overrides which of ``params`` count as the caller's own, for
a retry handing on its parent's whole params.
a retry handing on its parent's whole params. ``for_node`` is a
``<flow id>.<node>``.
"""
# Before anything else, including reading the flow: a caller retrying a
# submit it never got an answer for is owed the run that answer was
@@ -965,7 +986,7 @@ class RunService:
existing = self._by_key(idempotency_key)
if existing is not None:
return existing
flow = self.controller.store.read_flow(flow_name, draft=draft)
flow = self.controller.store.read_flow(flow_id, draft=draft)
issues = batch_issues(flow, self.controller.store, draft=draft)
if issues:
raise RunRejected(" ".join(issues))
@@ -973,7 +994,7 @@ class RunService:
run = Run(
id=new_run_id(),
flow=flow.name,
flow_id=flow.id,
flow_version=flow.version,
commit=self.controller.store.head(),
origin_commit=origin_commit(flow),
@@ -1011,7 +1032,8 @@ class RunService:
return existing
session.refresh(run)
self.queue.add(WorkItem(kind="run", node="", flow=flow.name, run_id=run.id))
# By id, like the run: what the item names outlives a rename of the key.
self.queue.add(WorkItem(kind="run", node="", flow=flow.id, run_id=run.id))
return run
def _complete(
@@ -1087,10 +1109,10 @@ class RunService:
{"external_id": external_id, "id": existing, "created": False}
)
continue
name = str(entry["flow"])
if name not in flows:
flows[name] = self.controller.store.read_flow(name)
flow = flows[name]
flow_id = str(entry["flow"])
if flow_id not in flows:
flows[flow_id] = self.controller.store.read_flow(flow_id)
flow = flows[flow_id]
try:
params, seed, passed = self._complete(
flow, entry["params"], entry["seed"]
@@ -1100,7 +1122,7 @@ class RunService:
started, finished = entry["started_at"], entry["finished_at"]
run = Run(
id=new_run_id(),
flow=flow.name,
flow_id=flow.id,
flow_version=flow.version,
params=params,
passed=passed,
@@ -1207,7 +1229,7 @@ class RunService:
# The gate first, so nothing new is submitted while the running nodes
# are being killed; a gated node is never handed to the executor, so
# the graph drains instead of going further.
pipeline.pause(self._flow_of(run_id) or "")
pipeline.pause(self._key_of(run_id))
workers = self.controller.workers
if workers is not None:
# Keyed by run, so a sweep cancelling one config leaves the others
@@ -1239,7 +1261,7 @@ class RunService:
raise RunRejected("That run has not finished; cancel it first")
source = run.model_copy()
return self.submit(
source.flow,
source.flow_id,
params=dict(source.params),
seed=source.seed,
group_id=source.group_id,
@@ -1254,10 +1276,11 @@ class RunService:
for_node=source.for_node,
)
def _flow_of(self, run_id: str) -> str | None:
def _key_of(self, run_id: str) -> str:
"""The key a run's pipeline runs its flow under, which it pauses by."""
with Session(db_engine) as session:
run = session.get(Run, run_id)
return run.flow if run else None
return tombstones.name_of(run.flow_id, self.controller.store) if run else ""
def _finish_unstarted(self, run_id: str, status: str) -> bool:
"""Cancel a run nothing of has executed, if it is still in ``status``."""
@@ -1464,23 +1487,31 @@ class RunService:
state: StateBackend | None = None
errors = 0
# The key its nodes and messages are named with, the id its rows are
# filed under. Read from the store and its tombstones rather than the
# flow, which may be gone before the run gets as far as reading it.
key = tombstones.name_of(run.flow_id, self.controller.store)
def observe(outcome: NodeOutcome) -> None:
nonlocal errors
if not outcome.ok:
errors += 1
self._record_node(run_id, outcome)
self._record_node(run, key, outcome)
sink = MetricSink(
run_id,
on_flush=lambda names: self._publish_event(
{
"type": "run_metric",
"flow": run.flow,
"flow": key,
"flow_id": run.flow_id,
"run": run_id,
"names": names,
"ts": time.time(),
}
),
key=key,
flow_id=run.flow_id,
)
try:
with self._lock:
@@ -1488,7 +1519,7 @@ class RunService:
# building one loads every python node in a worker, and waits
# for one while all of them are busy — for longer than a lease.
self._active[run_id] = None
flow = self.controller.store.read_flow(run.flow, draft=run.draft)
flow = self.controller.store.read_flow(run.flow_id, draft=run.draft)
# Read again, now that it is this run's turn: a sweep queues every
# run at once, and the code on disk is free to move in the hours
# before the last of them starts. What the record must name is the
@@ -1518,7 +1549,7 @@ class RunService:
status, reason = "cancelled", "Cancelled before it started"
return
self._active[run_id] = pipeline
self._publish(run, "run_started")
self._publish(run, key, "run_started")
pipeline.run(seed_values(flow, run.params, run.seed))
result = collect_result(flow, state)
with self._lock:
@@ -1542,7 +1573,7 @@ class RunService:
duration = round((time.perf_counter() - started) * 1000, 2)
self._finish(run_id, status, reason, result, duration)
run.status = status
self._publish(run, "run_finished")
self._publish(run, key, "run_finished")
self._release_cards(run)
# Its values were only ever this run's; nothing reads them once it
# has a result. On Redis the namespace would expire anyway.
@@ -1582,11 +1613,15 @@ class RunService:
except Exception:
logger.warning("Could not retire the GPU workers of run %s", run.id)
def _record_node(self, run_id: str, outcome: NodeOutcome) -> None:
def _record_node(self, run: Run, key: str, outcome: NodeOutcome) -> None:
"""Write what one node did, under ``<flow id>.<node>``. The outputs
and artifacts stay named by message, which is by the flow's *key*."""
run_id = run.id
node = requalify(outcome.node, key, run.flow_id)[:255]
outputs = _cacheable(outcome)
row = RunNode(
run_id=run_id,
node=outcome.node[:255],
node=node,
status="cached" if outcome.cached else ("ok" if outcome.ok else "error"),
started_at=datetime.now(UTC),
duration_ms=outcome.duration_ms,
@@ -1614,7 +1649,7 @@ class RunService:
run_id=run_id,
name=message[:255],
filename=str(ref.get("name") or "")[:255] or None,
node=outcome.node[:255],
node=node,
digest=str(ref.get("digest") or "")[:71],
size=int(ref.get("size") or 0),
media_type=str(
@@ -1667,11 +1702,12 @@ class RunService:
except Exception:
logger.exception("Could not close run %s", run_id)
def _publish(self, run: Run, kind: str) -> None:
def _publish(self, run: Run, key: str, kind: str) -> None:
self._publish_event(
{
"type": kind,
"flow": run.flow,
"flow": key,
"flow_id": run.flow_id,
"run": run.id,
"status": run.status,
"group": run.group_id or "",