Add POST /runs/import for runs executed elsewhere

A history recorded before the move (MLflow experiments) had nowhere to
go, since runs were only made by running a flow. Each entry is stored as
it ended: parameters completed and checked like a submit, cause
"import", empty commit and code digest, metrics and artifacts written
as given, nothing queued or announced. Run.external_id (new migration)
makes a second import a no-op and is an export column so a row joins
back. The request is all or nothing and capped at 200 entries;
Client.import_runs pages a longer history.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QgG6TJw5qBZzaLghQSRVrq
This commit is contained in:
2026-09-16 17:01:07 +02:00
co-authored by Claude Opus 5
parent eeaa46d4ae
commit d9630f66f5
12 changed files with 848 additions and 32 deletions
+173 -28
View File
@@ -25,6 +25,8 @@ Three things make a run different from a cascade, and each is deliberate:
from __future__ import annotations
import base64
import binascii
import copy
import hashlib
import json
@@ -103,6 +105,17 @@ def new_run_id() -> str:
return f"{int(time.time() * 1000):013d}-{uuid.uuid4().hex[:8]}"
def _span_ms(started: datetime | None, finished: datetime | None) -> float:
"""Milliseconds between two instants, or 0 without both. Naive is UTC."""
if started is None or finished is None:
return 0.0
utc = [
when if when.tzinfo else when.replace(tzinfo=UTC)
for when in (started, finished)
]
return round((utc[1] - utc[0]).total_seconds() * 1000, 2)
def digest_of(params: dict[str, Any], seed: int | None) -> str:
"""What identifies a run's inputs: same parameters, same digest."""
canonical = json.dumps(
@@ -881,34 +894,7 @@ class RunService:
issues = batch_issues(flow, self.controller.store, draft=draft)
if issues:
raise RunRejected(" ".join(issues))
params = params or {}
# Resolved before it is stored, so what the run records is the same
# reference a python caller would have passed and every later reader —
# the digest, the cache, the run detail — sees one spelling.
params = resolve_references(flow, params, self._artifacts)
# What the run actually starts from, not only what was passed: an input
# left out takes its declared value, and a row that records `{}` cannot
# say which. Folded literally — an initial is a value from the
# definition, never a reference to resolve.
declared = {
one.spec.name: one.initial for one in flow.inputs if one.initial is not None
}
# The run's own seed fills an input of that name, outranking what the
# flow declares and outranked by one passed as a parameter — the order
# `seed_values` applies, moved to where the record is written.
if seed is not None and any(one.spec.name == "seed" for one in flow.inputs):
declared["seed"] = seed
params = {**declared, **params}
# And back the other way, so the run-level column holds the seed the
# run actually used however it arrived. Otherwise `--seed 1` fills one
# column and a declared seed the other, and that is the single field an
# export still has to coalesce.
resolved_seed = params.get("seed")
if isinstance(resolved_seed, int) and not isinstance(resolved_seed, bool):
seed = resolved_seed
# Checked here rather than in the driver: a caller who mistyped a
# parameter should be told now, not by a run that fails in a minute.
seed_values(flow, params, seed)
params, seed = self._complete(flow, params, seed)
run = Run(
id=new_run_id(),
@@ -950,6 +936,40 @@ class RunService:
self.queue.add(WorkItem(kind="run", node="", flow=flow.name, run_id=run.id))
return run
def _complete(
self, flow: FlowDef, params: dict[str, Any] | None, seed: int | None
) -> tuple[dict[str, Any], int | None]:
"""The parameters and seed a run of ``flow`` starts from, checked."""
params = params or {}
# Resolved before it is stored, so what the run records is the same
# reference a python caller would have passed and every later reader —
# the digest, the cache, the run detail — sees one spelling.
params = resolve_references(flow, params, self._artifacts)
# What the run actually starts from, not only what was passed: an input
# left out takes its declared value, and a row that records `{}` cannot
# say which. Folded literally — an initial is a value from the
# definition, never a reference to resolve.
declared = {
one.spec.name: one.initial for one in flow.inputs if one.initial is not None
}
# The run's own seed fills an input of that name, outranking what the
# flow declares and outranked by one passed as a parameter — the order
# `seed_values` applies, moved to where the record is written.
if seed is not None and any(one.spec.name == "seed" for one in flow.inputs):
declared["seed"] = seed
params = {**declared, **params}
# And back the other way, so the run-level column holds the seed the
# run actually used however it arrived. Otherwise `--seed 1` fills one
# column and a declared seed the other, and that is the single field an
# export still has to coalesce.
resolved_seed = params.get("seed")
if isinstance(resolved_seed, int) and not isinstance(resolved_seed, bool):
seed = resolved_seed
# Checked here rather than in the driver: a caller who mistyped a
# parameter should be told now, not by a run that fails in a minute.
seed_values(flow, params, seed)
return params, seed
def _by_key(self, idempotency_key: str) -> Run | None:
"""The run a key already made, if it made one."""
with Session(db_engine) as session:
@@ -957,6 +977,131 @@ class RunService:
select(Run).where(col(Run.idempotency_key) == idempotency_key)
).first()
def import_runs(
self, entries: list[dict[str, Any]], actor: str = ""
) -> list[dict[str, Any]]:
"""Record runs that were executed somewhere else, as they ended there.
Parameters are completed and checked the way `submit` does it, so an
imported row reads like any other. The provenance columns stay empty:
an imported run ran code this engine never saw, and borrowing today's
commit would say otherwise. Nothing is queued and nothing announced —
the runs are over, and a `run_finished` per row would page whatever
alerts on a failure.
All or nothing: every entry is checked before a row is written. One
whose `external_id` is already here answers with that run instead.
"""
answers: list[dict[str, Any]] = []
rows: list[Run | RunMetric | RunArtifact] = []
made: dict[str, str] = {}
flows: dict[str, FlowDef] = {}
for entry in entries:
external_id = str(entry["external_id"])
existing = made.get(external_id) or getattr(
self._by_external(external_id), "id", None
)
if existing:
answers.append(
{"external_id": external_id, "id": existing, "created": False}
)
continue
name = str(entry["flow"])
if name not in flows:
flows[name] = self.controller.store.read_flow(name)
flow = flows[name]
try:
params, seed = self._complete(flow, entry["params"], entry["seed"])
except RunRejected as exc:
raise RunRejected(f"{external_id}: {exc}") from exc
started, finished = entry["started_at"], entry["finished_at"]
run = Run(
id=new_run_id(),
flow=flow.name,
flow_version=flow.version,
params=params,
params_digest=digest_of(params, seed),
seed=seed,
group_id=entry["group_id"],
cause="import",
status=entry["status"],
labels=entry["labels"],
created_at=entry["created_at"],
started_at=started,
finished_at=finished,
duration_ms=_span_ms(started, finished),
result=entry["result"],
actor=actor,
external_id=external_id,
)
rows.append(run)
# Keyed as the table is, so a step reported twice is the later one
# rather than a batch refused over it.
points: dict[tuple[str, int], RunMetric] = {}
for metric in entry["metrics"]:
if not math.isfinite(metric["value"]):
# Refused on the way in by the ports, and not a value the
# column can hold.
continue
point = RunMetric(
run_id=run.id,
name=metric["name"],
step=metric["step"],
ts=metric["ts"],
value=metric["value"],
)
points[(point.name, point.step)] = point
rows.extend(points.values())
for artifact in entry["artifacts"]:
rows.append(self._imported_artifact(run.id, external_id, artifact))
made[external_id] = run.id
answers.append({"external_id": external_id, "id": run.id, "created": True})
if rows:
with Session(db_engine) as session:
session.add_all(rows)
try:
session.commit()
except IntegrityError as exc:
raise RunRejected(
"Some of these runs were imported meanwhile; send them again"
) from exc
return answers
def _imported_artifact(
self, run_id: str, external_id: str, artifact: dict[str, Any]
) -> RunArtifact:
"""Put an imported file's bytes in the store, and the row naming them.
Stored before the batch is known to be good. A batch refused after
this leaves bytes nothing names, which `sweep_artifacts` collects.
"""
if self._artifacts is None:
raise RunRejected("This engine has no artifact store to import into")
try:
data = base64.b64decode(artifact["content_base64"], validate=True)
except binascii.Error as exc:
raise RunRejected(
f"{external_id}: artifact '{artifact['name']}' is not base64"
) from exc
ref = self._artifacts.put(
[data], name=artifact["filename"] or "", media_type=artifact["media_type"]
)
return RunArtifact(
run_id=run_id,
name=artifact["name"],
filename=artifact["filename"],
digest=ref["digest"],
size=ref["size"],
media_type=ref["media_type"],
)
def _by_external(self, external_id: str) -> Run | None:
"""The run an earlier import made under this id, if there is one."""
with Session(db_engine) as session:
return session.exec(
select(Run).where(col(Run.external_id) == external_id)
).first()
def cancel(self, run_id: str) -> bool:
"""Stop a run: kill what it is executing, schedule nothing further."""
with self._lock: