Add POST /runs/import for runs executed elsewhere
A history recorded before the move (MLflow experiments) had nowhere to go, since runs were only made by running a flow. Each entry is stored as it ended: parameters completed and checked like a submit, cause "import", empty commit and code digest, metrics and artifacts written as given, nothing queued or announced. Run.external_id (new migration) makes a second import a no-op and is an export column so a row joins back. The request is all or nothing and capped at 200 entries; Client.import_runs pages a longer history. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QgG6TJw5qBZzaLghQSRVrq
This commit is contained in:
@@ -23,6 +23,7 @@ from sqlalchemy import select as sa_select
|
||||
from sqlmodel import Session, col, select
|
||||
|
||||
from fluksio.api.deps import CurrentUser, SessionDep, get_current_user
|
||||
from fluksio.core.config import settings
|
||||
from fluksio.core.db import writing
|
||||
from fluksio.flow.events import event_bus
|
||||
from fluksio.flow.messages import requalify
|
||||
@@ -37,6 +38,9 @@ router = APIRouter(
|
||||
#: A sweep bigger than this is almost always a mistake in a loop.
|
||||
MAX_SWEEP = 1000
|
||||
|
||||
#: Runs per import request. A history is thousands; the client pages it.
|
||||
MAX_IMPORT = 200
|
||||
|
||||
|
||||
def elapsed_ms(since: datetime) -> float:
|
||||
"""Milliseconds since an instant the columns stored as UTC."""
|
||||
@@ -49,8 +53,8 @@ def _aware(when: datetime) -> datetime:
|
||||
return when if when.tzinfo else when.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
#: Where a caller may say a run came from. "sweep" is not here because the
|
||||
#: sweep route writes it itself, and neither is a value a client made up: the
|
||||
#: Where a caller may say a run came from. "sweep" and "import" are not here
|
||||
#: because their routes write them, and neither is a value a client made up: the
|
||||
#: column is only worth a table row if it means the same thing every time.
|
||||
RunCause = Literal["api", "cli", "sdk"]
|
||||
|
||||
@@ -83,6 +87,51 @@ class SweepCreate(BaseModel):
|
||||
no_cache: bool = False
|
||||
|
||||
|
||||
class ImportMetric(BaseModel):
|
||||
#: Flow-qualified, the way the engine names what a node reports.
|
||||
name: str = Field(min_length=1, max_length=128)
|
||||
step: int = -1
|
||||
ts: float = 0.0
|
||||
value: float
|
||||
|
||||
|
||||
class ImportArtifact(BaseModel):
|
||||
name: str = Field(min_length=1, max_length=255)
|
||||
filename: str | None = Field(default=None, max_length=255)
|
||||
media_type: str = Field(default="application/octet-stream", max_length=128)
|
||||
content_base64: str
|
||||
|
||||
|
||||
class ImportEntry(BaseModel):
|
||||
"""One run executed somewhere else, as it ended there."""
|
||||
|
||||
flow: str
|
||||
#: What it was called where it ran. Importing it again is a no-op.
|
||||
external_id: str = Field(min_length=1, max_length=255)
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
seed: int | None = None
|
||||
status: Literal["ok", "error", "cancelled"] = "ok"
|
||||
created_at: datetime
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
result: dict[str, Any] = Field(default_factory=dict)
|
||||
metrics: list[ImportMetric] = Field(default_factory=list)
|
||||
artifacts: list[ImportArtifact] = Field(default_factory=list)
|
||||
labels: list[str] = Field(default_factory=list)
|
||||
group_id: str | None = Field(default=None, max_length=64)
|
||||
|
||||
|
||||
class ImportCreate(BaseModel):
|
||||
runs: list[ImportEntry] = Field(default_factory=list)
|
||||
|
||||
|
||||
class Imported(BaseModel):
|
||||
external_id: str
|
||||
id: str
|
||||
#: False when an earlier import already made it.
|
||||
created: bool
|
||||
|
||||
|
||||
class RunNodeRow(BaseModel):
|
||||
node: str
|
||||
status: str
|
||||
@@ -132,6 +181,9 @@ class RunRow(BaseModel):
|
||||
#: The flow store's own commit. Short, unlike `result`, so the list
|
||||
#: carries it: "what code produced this" is a question asked of a table.
|
||||
commit: str = ""
|
||||
#: What it was called where it ran, for a run imported rather than
|
||||
#: executed here.
|
||||
external_id: str | None = None
|
||||
seed: int | None
|
||||
group_id: str | None
|
||||
#: Whether it ran the working copy rather than what is published.
|
||||
@@ -265,6 +317,45 @@ async def create_sweep(
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/import", response_model=list[Imported], status_code=202)
|
||||
async def import_runs(body: ImportCreate, request: Request, user: CurrentUser) -> Any:
|
||||
"""Record runs that were executed somewhere else: a history from before.
|
||||
|
||||
Stored as they ended, with `cause` "import" and no commit or code digest,
|
||||
since this engine never saw the code. The whole request is checked before
|
||||
anything is written, and an `external_id` already here answers with the
|
||||
run it made rather than a second one.
|
||||
"""
|
||||
if not body.runs:
|
||||
raise HTTPException(status_code=422, detail="An import needs at least one run")
|
||||
if len(body.runs) > MAX_IMPORT:
|
||||
raise HTTPException(
|
||||
status_code=422, detail=f"An import is capped at {MAX_IMPORT} runs"
|
||||
)
|
||||
cap = settings.MAX_ARTIFACT_BYTES
|
||||
# The same limit `PUT /artifacts` holds, estimated from the encoded length:
|
||||
# otherwise any account could fill the data volume this way instead.
|
||||
if cap and any(
|
||||
len(artifact.content_base64) * 3 // 4 > cap
|
||||
for entry in body.runs
|
||||
for artifact in entry.artifacts
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=413, detail=f"An artifact may be at most {cap} bytes"
|
||||
)
|
||||
service = _service(request)
|
||||
try:
|
||||
return await run_in_threadpool(
|
||||
service.import_runs,
|
||||
[entry.model_dump() for entry in body.runs],
|
||||
user.email,
|
||||
)
|
||||
except FlowNotFound as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except RunRejected as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.get("", response_model=list[RunRow])
|
||||
def read_runs(
|
||||
session: SessionDep,
|
||||
@@ -373,6 +464,8 @@ RUN_COLUMNS = (
|
||||
#: Set on a run made by retrying another, so a sweep that was completed
|
||||
#: rather than repeated says which row replaced which.
|
||||
"parent_id",
|
||||
#: What an imported run was called where it ran, so the row joins back.
|
||||
"external_id",
|
||||
"code_digest",
|
||||
"origin_commit",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user