Add POST /runs/import for runs executed elsewhere

A history recorded before the move (MLflow experiments) had nowhere to
go, since runs were only made by running a flow. Each entry is stored as
it ended: parameters completed and checked like a submit, cause
"import", empty commit and code digest, metrics and artifacts written
as given, nothing queued or announced. Run.external_id (new migration)
makes a second import a no-op and is an export column so a row joins
back. The request is all or nothing and capped at 200 entries;
Client.import_runs pages a longer history.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QgG6TJw5qBZzaLghQSRVrq
This commit is contained in:
2026-09-16 17:01:07 +02:00
co-authored by Claude Opus 5
parent eeaa46d4ae
commit d9630f66f5
12 changed files with 848 additions and 32 deletions
+95 -2
View File
@@ -23,6 +23,7 @@ from sqlalchemy import select as sa_select
from sqlmodel import Session, col, select
from fluksio.api.deps import CurrentUser, SessionDep, get_current_user
from fluksio.core.config import settings
from fluksio.core.db import writing
from fluksio.flow.events import event_bus
from fluksio.flow.messages import requalify
@@ -37,6 +38,9 @@ router = APIRouter(
#: A sweep bigger than this is almost always a mistake in a loop.
MAX_SWEEP = 1000
#: Runs per import request. A history is thousands; the client pages it.
MAX_IMPORT = 200
def elapsed_ms(since: datetime) -> float:
"""Milliseconds since an instant the columns stored as UTC."""
@@ -49,8 +53,8 @@ def _aware(when: datetime) -> datetime:
return when if when.tzinfo else when.replace(tzinfo=UTC)
#: Where a caller may say a run came from. "sweep" is not here because the
#: sweep route writes it itself, and neither is a value a client made up: the
#: Where a caller may say a run came from. "sweep" and "import" are not here
#: because their routes write them, and neither is a value a client made up: the
#: column is only worth a table row if it means the same thing every time.
RunCause = Literal["api", "cli", "sdk"]
@@ -83,6 +87,51 @@ class SweepCreate(BaseModel):
no_cache: bool = False
class ImportMetric(BaseModel):
#: Flow-qualified, the way the engine names what a node reports.
name: str = Field(min_length=1, max_length=128)
step: int = -1
ts: float = 0.0
value: float
class ImportArtifact(BaseModel):
name: str = Field(min_length=1, max_length=255)
filename: str | None = Field(default=None, max_length=255)
media_type: str = Field(default="application/octet-stream", max_length=128)
content_base64: str
class ImportEntry(BaseModel):
"""One run executed somewhere else, as it ended there."""
flow: str
#: What it was called where it ran. Importing it again is a no-op.
external_id: str = Field(min_length=1, max_length=255)
params: dict[str, Any] = Field(default_factory=dict)
seed: int | None = None
status: Literal["ok", "error", "cancelled"] = "ok"
created_at: datetime
started_at: datetime | None = None
finished_at: datetime | None = None
result: dict[str, Any] = Field(default_factory=dict)
metrics: list[ImportMetric] = Field(default_factory=list)
artifacts: list[ImportArtifact] = Field(default_factory=list)
labels: list[str] = Field(default_factory=list)
group_id: str | None = Field(default=None, max_length=64)
class ImportCreate(BaseModel):
runs: list[ImportEntry] = Field(default_factory=list)
class Imported(BaseModel):
external_id: str
id: str
#: False when an earlier import already made it.
created: bool
class RunNodeRow(BaseModel):
node: str
status: str
@@ -132,6 +181,9 @@ class RunRow(BaseModel):
#: The flow store's own commit. Short, unlike `result`, so the list
#: carries it: "what code produced this" is a question asked of a table.
commit: str = ""
#: What it was called where it ran, for a run imported rather than
#: executed here.
external_id: str | None = None
seed: int | None
group_id: str | None
#: Whether it ran the working copy rather than what is published.
@@ -265,6 +317,45 @@ async def create_sweep(
raise HTTPException(status_code=422, detail=str(exc)) from exc
@router.post("/import", response_model=list[Imported], status_code=202)
async def import_runs(body: ImportCreate, request: Request, user: CurrentUser) -> Any:
"""Record runs that were executed somewhere else: a history from before.
Stored as they ended, with `cause` "import" and no commit or code digest,
since this engine never saw the code. The whole request is checked before
anything is written, and an `external_id` already here answers with the
run it made rather than a second one.
"""
if not body.runs:
raise HTTPException(status_code=422, detail="An import needs at least one run")
if len(body.runs) > MAX_IMPORT:
raise HTTPException(
status_code=422, detail=f"An import is capped at {MAX_IMPORT} runs"
)
cap = settings.MAX_ARTIFACT_BYTES
# The same limit `PUT /artifacts` holds, estimated from the encoded length:
# otherwise any account could fill the data volume this way instead.
if cap and any(
len(artifact.content_base64) * 3 // 4 > cap
for entry in body.runs
for artifact in entry.artifacts
):
raise HTTPException(
status_code=413, detail=f"An artifact may be at most {cap} bytes"
)
service = _service(request)
try:
return await run_in_threadpool(
service.import_runs,
[entry.model_dump() for entry in body.runs],
user.email,
)
except FlowNotFound as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except RunRejected as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
@router.get("", response_model=list[RunRow])
def read_runs(
session: SessionDep,
@@ -373,6 +464,8 @@ RUN_COLUMNS = (
#: Set on a run made by retrying another, so a sweep that was completed
#: rather than repeated says which row replaced which.
"parent_id",
#: What an imported run was called where it ran, so the row joins back.
"external_id",
"code_digest",
"origin_commit",
)