History is kept by flow id: runs, cascades, rollups and events file the id in their flow column and nodes as <flow id>.<node>, the API answers flow plus flow_name and shows and takes nodes as <key>.<node>; events carry flow_id from the moment they are published; run queue items, the stage cache and cached curves keep ids and keys apart; endpoints, the brain graph, device bindings and search name the id to open
This commit is contained in:
@@ -49,9 +49,10 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlmodel import Session, col, select
|
||||
|
||||
from fluksio.core.db import engine as db_engine
|
||||
from fluksio.flow import tombstones
|
||||
from fluksio.flow.artifacts import ArtifactStore, is_reference, valid_digest
|
||||
from fluksio.flow.controller import NODE_TYPES, FlowController, RunContext
|
||||
from fluksio.flow.messages import qualify
|
||||
from fluksio.flow.messages import qualify, requalify
|
||||
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, CacheHit, NodeOutcome, Pipeline
|
||||
from fluksio.flow.queue import WorkItem, WorkQueue
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||
@@ -531,8 +532,14 @@ class MetricSink:
|
||||
batch: int = METRIC_BATCH,
|
||||
interval: float = METRIC_FLUSH_S,
|
||||
on_flush: Callable[[list[str]], None] | None = None,
|
||||
key: str = "",
|
||||
flow_id: str = "",
|
||||
) -> None:
|
||||
self.run_id = run_id
|
||||
#: The run's flow, by the key its nodes are named with and by the id
|
||||
#: a row names them with: ``<flow id>.<node>``.
|
||||
self._key = key
|
||||
self._flow_id = flow_id
|
||||
self._batch = batch
|
||||
self._interval = interval
|
||||
self._on_flush = on_flush
|
||||
@@ -562,7 +569,7 @@ class MetricSink:
|
||||
run_id=self.run_id,
|
||||
name=name[:128],
|
||||
step=step,
|
||||
node=node_id[:255],
|
||||
node=requalify(node_id, self._key, self._flow_id)[:255],
|
||||
ts=now,
|
||||
value=float(value),
|
||||
)
|
||||
@@ -802,8 +809,17 @@ class RunCache:
|
||||
not a hit: the reference would name bytes nobody can open.
|
||||
"""
|
||||
|
||||
def __init__(self, artifacts: ArtifactStore | None = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
artifacts: ArtifactStore | None = None,
|
||||
key_of: Callable[[str], str] | None = None,
|
||||
) -> None:
|
||||
self.artifacts = artifacts
|
||||
#: The key a source run's outputs are named with, from its flow's id:
|
||||
#: the flow's key now, or the last one it had if it is gone. Without
|
||||
#: one the id is taken for the key, which it is for a flow from before
|
||||
#: ids.
|
||||
self.key_of = key_of or (lambda flow_id: flow_id)
|
||||
|
||||
def lookup(self, key: str) -> CacheHit | None:
|
||||
if not key:
|
||||
@@ -829,7 +845,7 @@ class RunCache:
|
||||
continue
|
||||
source = session.get(Run, row.run_id)
|
||||
return CacheHit(
|
||||
flow=source.flow if source is not None else "",
|
||||
flow=self.key_of(source.flow_id) if source else "",
|
||||
outputs=outputs,
|
||||
# A row that was itself restored holds no series, so
|
||||
# follow its pointer rather than adding a hop to it.
|
||||
@@ -862,7 +878,9 @@ class RunService:
|
||||
self.controller = controller
|
||||
self.queue = queue
|
||||
self._artifacts = artifacts
|
||||
self._cache = RunCache(artifacts)
|
||||
self._cache = RunCache(
|
||||
artifacts, lambda flow_id: tombstones.name_of(flow_id, controller.store)
|
||||
)
|
||||
# Without one, a run gets a private in-memory state — which is exactly
|
||||
# the isolation it wants, minus surviving the process.
|
||||
self._state_factory = state_factory or (lambda _ns: MemoryState())
|
||||
@@ -912,10 +930,11 @@ class RunService:
|
||||
"""
|
||||
with Session(db_engine) as session:
|
||||
waiting = session.exec(
|
||||
select(Run.id, Run.flow).where(col(Run.status) == "queued")
|
||||
select(Run.id, Run.flow_id).where(col(Run.status) == "queued")
|
||||
).all()
|
||||
for run_id, flow in waiting:
|
||||
self.queue.add(WorkItem(kind="run", node="", flow=flow, run_id=run_id))
|
||||
# A run item names its flow by id, as `submit` queues it.
|
||||
for run_id, flow_id in waiting:
|
||||
self.queue.add(WorkItem(kind="run", node="", flow=flow_id, run_id=run_id))
|
||||
if waiting:
|
||||
logger.info(
|
||||
"Requeued %d run(s) left queued by an earlier engine", len(waiting)
|
||||
@@ -940,7 +959,7 @@ class RunService:
|
||||
|
||||
def submit(
|
||||
self,
|
||||
flow_name: str,
|
||||
flow_id: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
seed: int | None = None,
|
||||
group_id: str | None = None,
|
||||
@@ -953,10 +972,12 @@ class RunService:
|
||||
passed: list[str] | None = None,
|
||||
for_node: str | None = None,
|
||||
) -> Run:
|
||||
"""Journal a run and wake an engine up for it. Never blocks on it.
|
||||
"""Journal a run of the flow *flow_id* and wake an engine up for it.
|
||||
Never blocks on it.
|
||||
|
||||
``passed`` overrides which of ``params`` count as the caller's own, for
|
||||
a retry handing on its parent's whole params.
|
||||
a retry handing on its parent's whole params. ``for_node`` is a
|
||||
``<flow id>.<node>``.
|
||||
"""
|
||||
# Before anything else, including reading the flow: a caller retrying a
|
||||
# submit it never got an answer for is owed the run that answer was
|
||||
@@ -965,7 +986,7 @@ class RunService:
|
||||
existing = self._by_key(idempotency_key)
|
||||
if existing is not None:
|
||||
return existing
|
||||
flow = self.controller.store.read_flow(flow_name, draft=draft)
|
||||
flow = self.controller.store.read_flow(flow_id, draft=draft)
|
||||
issues = batch_issues(flow, self.controller.store, draft=draft)
|
||||
if issues:
|
||||
raise RunRejected(" ".join(issues))
|
||||
@@ -973,7 +994,7 @@ class RunService:
|
||||
|
||||
run = Run(
|
||||
id=new_run_id(),
|
||||
flow=flow.name,
|
||||
flow_id=flow.id,
|
||||
flow_version=flow.version,
|
||||
commit=self.controller.store.head(),
|
||||
origin_commit=origin_commit(flow),
|
||||
@@ -1011,7 +1032,8 @@ class RunService:
|
||||
return existing
|
||||
session.refresh(run)
|
||||
|
||||
self.queue.add(WorkItem(kind="run", node="", flow=flow.name, run_id=run.id))
|
||||
# By id, like the run: what the item names outlives a rename of the key.
|
||||
self.queue.add(WorkItem(kind="run", node="", flow=flow.id, run_id=run.id))
|
||||
return run
|
||||
|
||||
def _complete(
|
||||
@@ -1087,10 +1109,10 @@ class RunService:
|
||||
{"external_id": external_id, "id": existing, "created": False}
|
||||
)
|
||||
continue
|
||||
name = str(entry["flow"])
|
||||
if name not in flows:
|
||||
flows[name] = self.controller.store.read_flow(name)
|
||||
flow = flows[name]
|
||||
flow_id = str(entry["flow"])
|
||||
if flow_id not in flows:
|
||||
flows[flow_id] = self.controller.store.read_flow(flow_id)
|
||||
flow = flows[flow_id]
|
||||
try:
|
||||
params, seed, passed = self._complete(
|
||||
flow, entry["params"], entry["seed"]
|
||||
@@ -1100,7 +1122,7 @@ class RunService:
|
||||
started, finished = entry["started_at"], entry["finished_at"]
|
||||
run = Run(
|
||||
id=new_run_id(),
|
||||
flow=flow.name,
|
||||
flow_id=flow.id,
|
||||
flow_version=flow.version,
|
||||
params=params,
|
||||
passed=passed,
|
||||
@@ -1207,7 +1229,7 @@ class RunService:
|
||||
# The gate first, so nothing new is submitted while the running nodes
|
||||
# are being killed; a gated node is never handed to the executor, so
|
||||
# the graph drains instead of going further.
|
||||
pipeline.pause(self._flow_of(run_id) or "")
|
||||
pipeline.pause(self._key_of(run_id))
|
||||
workers = self.controller.workers
|
||||
if workers is not None:
|
||||
# Keyed by run, so a sweep cancelling one config leaves the others
|
||||
@@ -1239,7 +1261,7 @@ class RunService:
|
||||
raise RunRejected("That run has not finished; cancel it first")
|
||||
source = run.model_copy()
|
||||
return self.submit(
|
||||
source.flow,
|
||||
source.flow_id,
|
||||
params=dict(source.params),
|
||||
seed=source.seed,
|
||||
group_id=source.group_id,
|
||||
@@ -1254,10 +1276,11 @@ class RunService:
|
||||
for_node=source.for_node,
|
||||
)
|
||||
|
||||
def _flow_of(self, run_id: str) -> str | None:
|
||||
def _key_of(self, run_id: str) -> str:
|
||||
"""The key a run's pipeline runs its flow under, which it pauses by."""
|
||||
with Session(db_engine) as session:
|
||||
run = session.get(Run, run_id)
|
||||
return run.flow if run else None
|
||||
return tombstones.name_of(run.flow_id, self.controller.store) if run else ""
|
||||
|
||||
def _finish_unstarted(self, run_id: str, status: str) -> bool:
|
||||
"""Cancel a run nothing of has executed, if it is still in ``status``."""
|
||||
@@ -1464,23 +1487,31 @@ class RunService:
|
||||
state: StateBackend | None = None
|
||||
errors = 0
|
||||
|
||||
# The key its nodes and messages are named with, the id its rows are
|
||||
# filed under. Read from the store and its tombstones rather than the
|
||||
# flow, which may be gone before the run gets as far as reading it.
|
||||
key = tombstones.name_of(run.flow_id, self.controller.store)
|
||||
|
||||
def observe(outcome: NodeOutcome) -> None:
|
||||
nonlocal errors
|
||||
if not outcome.ok:
|
||||
errors += 1
|
||||
self._record_node(run_id, outcome)
|
||||
self._record_node(run, key, outcome)
|
||||
|
||||
sink = MetricSink(
|
||||
run_id,
|
||||
on_flush=lambda names: self._publish_event(
|
||||
{
|
||||
"type": "run_metric",
|
||||
"flow": run.flow,
|
||||
"flow": key,
|
||||
"flow_id": run.flow_id,
|
||||
"run": run_id,
|
||||
"names": names,
|
||||
"ts": time.time(),
|
||||
}
|
||||
),
|
||||
key=key,
|
||||
flow_id=run.flow_id,
|
||||
)
|
||||
try:
|
||||
with self._lock:
|
||||
@@ -1488,7 +1519,7 @@ class RunService:
|
||||
# building one loads every python node in a worker, and waits
|
||||
# for one while all of them are busy — for longer than a lease.
|
||||
self._active[run_id] = None
|
||||
flow = self.controller.store.read_flow(run.flow, draft=run.draft)
|
||||
flow = self.controller.store.read_flow(run.flow_id, draft=run.draft)
|
||||
# Read again, now that it is this run's turn: a sweep queues every
|
||||
# run at once, and the code on disk is free to move in the hours
|
||||
# before the last of them starts. What the record must name is the
|
||||
@@ -1518,7 +1549,7 @@ class RunService:
|
||||
status, reason = "cancelled", "Cancelled before it started"
|
||||
return
|
||||
self._active[run_id] = pipeline
|
||||
self._publish(run, "run_started")
|
||||
self._publish(run, key, "run_started")
|
||||
pipeline.run(seed_values(flow, run.params, run.seed))
|
||||
result = collect_result(flow, state)
|
||||
with self._lock:
|
||||
@@ -1542,7 +1573,7 @@ class RunService:
|
||||
duration = round((time.perf_counter() - started) * 1000, 2)
|
||||
self._finish(run_id, status, reason, result, duration)
|
||||
run.status = status
|
||||
self._publish(run, "run_finished")
|
||||
self._publish(run, key, "run_finished")
|
||||
self._release_cards(run)
|
||||
# Its values were only ever this run's; nothing reads them once it
|
||||
# has a result. On Redis the namespace would expire anyway.
|
||||
@@ -1582,11 +1613,15 @@ class RunService:
|
||||
except Exception:
|
||||
logger.warning("Could not retire the GPU workers of run %s", run.id)
|
||||
|
||||
def _record_node(self, run_id: str, outcome: NodeOutcome) -> None:
|
||||
def _record_node(self, run: Run, key: str, outcome: NodeOutcome) -> None:
|
||||
"""Write what one node did, under ``<flow id>.<node>``. The outputs
|
||||
and artifacts stay named by message, which is by the flow's *key*."""
|
||||
run_id = run.id
|
||||
node = requalify(outcome.node, key, run.flow_id)[:255]
|
||||
outputs = _cacheable(outcome)
|
||||
row = RunNode(
|
||||
run_id=run_id,
|
||||
node=outcome.node[:255],
|
||||
node=node,
|
||||
status="cached" if outcome.cached else ("ok" if outcome.ok else "error"),
|
||||
started_at=datetime.now(UTC),
|
||||
duration_ms=outcome.duration_ms,
|
||||
@@ -1614,7 +1649,7 @@ class RunService:
|
||||
run_id=run_id,
|
||||
name=message[:255],
|
||||
filename=str(ref.get("name") or "")[:255] or None,
|
||||
node=outcome.node[:255],
|
||||
node=node,
|
||||
digest=str(ref.get("digest") or "")[:71],
|
||||
size=int(ref.get("size") or 0),
|
||||
media_type=str(
|
||||
@@ -1667,11 +1702,12 @@ class RunService:
|
||||
except Exception:
|
||||
logger.exception("Could not close run %s", run_id)
|
||||
|
||||
def _publish(self, run: Run, kind: str) -> None:
|
||||
def _publish(self, run: Run, key: str, kind: str) -> None:
|
||||
self._publish_event(
|
||||
{
|
||||
"type": kind,
|
||||
"flow": run.flow,
|
||||
"flow": key,
|
||||
"flow_id": run.flow_id,
|
||||
"run": run.id,
|
||||
"status": run.status,
|
||||
"group": run.group_id or "",
|
||||
|
||||
Reference in New Issue
Block a user