Fluksio/app
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
c6ac7f80a9f52eeab9ee3158cc03ec72ce1a7066
app/backend/tests/api/routes/test_runs.py
T

1768 lines
64 KiB
Python
Raw Blame History

"""The stage cache, from the side that needs a database.
The pipeline half — what a hit restores and what a key is made of — is in
`tests/flow/test_runs.py`, which runs without one.
"""
import base64
import csv
import json
import logging
import threading
from datetime import UTC, datetime, timedelta
import pytest
from sqlmodel import Session, col, select
from fluksio.core.config import settings
from fluksio.core.db import engine as db_engine
from fluksio.flow.artifacts import ArtifactStore
from fluksio.flow.controller import FlowController
from fluksio.flow.messages import DType, MessageSpec
from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, NodeOutcome, Pipeline
from fluksio.flow.runs import (
LEASE_STALE_S,
OUTPUT_CAP,
RunCache,
RunRejected,
RunService,
_cacheable,
new_run_id,
prune_stale_cache,
resolve_references,
seed_values,
)
from fluksio.flow.schemas import FlowDef, FlowInput, NodeDef
from fluksio.flow.store import FlowStore
from fluksio.models import FlowRun, Run, RunArtifact, RunMetric, RunNode
def test_a_run_cache_finds_what_an_earlier_run_recorded(tmp_path):
"""The run history is the cache; there is no second store to keep."""
store = ArtifactStore(tmp_path / "artifacts")
reference = store.put([b"payload"], name="data.bin")
plain, with_artifact, collected = "k-plain", "k-artifact", "k-collected"
with Session(db_engine) as session:
for run_id in ("cache-1", "cache-2", "cache-3"):
session.add(
Run(
id=run_id,
flow_id="study",
status="ok",
created_at=datetime.now(UTC),
)
)
session.add(
RunNode(
run_id="cache-1",
node="study.a",
status="ok",
cache_key=plain,
outputs=json.dumps({"study.loss": 1.5}),
)
)
session.add(
RunNode(
run_id="cache-2",
node="study.b",
status="ok",
cache_key=with_artifact,
outputs=json.dumps({"study.data": reference}),
)
)
session.add(
RunNode(
run_id="cache-3",
node="study.c",
status="ok",
cache_key=collected,
outputs=json.dumps(
{"study.data": {**reference, "digest": "sha256:" + "1" * 64}}
),
)
)
session.commit()
cache = RunCache(store)
hit = cache.lookup(plain)
assert hit is not None and hit.outputs == {"study.loss": 1.5}
# Where its series is, so a run reusing it can read the curve back.
assert hit.metrics_run == "cache-1"
found = cache.lookup(with_artifact)
assert found is not None and found.outputs == {"study.data": reference}
# Its bytes have gone from the store, so the reference names nothing a
# restored run could open. That is a miss, not a broken run.
assert cache.lookup(collected) is None
assert cache.lookup("never-seen") is None
assert cache.lookup("") is None
# A rename may have moved some of a row's names and not yet the others.
assert RunCache(store, renaming=lambda: True).lookup(plain) is None
def test_what_may_be_stored_as_a_cache_entry():
"""A row carries a key and its outputs together, or neither."""
ok = NodeOutcome(
node="study.a", ok=True, cache_key="k", output_values={"study.loss": 1.0}
)
assert _cacheable(ok) == '{"study.loss":1.0}'
# A node that published nothing is still an answer worth reusing.
assert _cacheable(ok.model_copy(update={"output_values": None})) == "null"
# Not cacheable: it failed, it has no key, or it returned too much.
assert _cacheable(ok.model_copy(update={"ok": False})) is None
assert _cacheable(ok.model_copy(update={"cache_key": ""})) is None
big = {"study.data": "x" * (OUTPUT_CAP + 1)}
assert _cacheable(ok.model_copy(update={"output_values": big})) is None
def test_a_row_keyed_before_the_current_shape_is_unkeyed():
"""It matches nothing any more, but it is still the run's record."""
outputs = json.dumps({"study.loss": 1.0})
with Session(db_engine) as session:
for run_id, made in (
("shape-old", CACHE_SHAPE_SINCE - timedelta(days=1)),
("shape-new", CACHE_SHAPE_SINCE + timedelta(seconds=1)),
):
session.add(Run(id=run_id, flow_id="study", status="ok", created_at=made))
session.add(
RunNode(
run_id=run_id,
node="study.a",
cache_key=run_id,
outputs=outputs,
)
)
session.commit()
try:
assert prune_stale_cache() == 1
with Session(db_engine) as session:
old = session.get(RunNode, ("shape-old", "study.a"))
new = session.get(RunNode, ("shape-new", "study.a"))
assert old is not None and old.cache_key == "" and old.outputs == outputs
assert new is not None and new.cache_key == "shape-new"
finally:
with Session(db_engine) as session:
for run_id in ("shape-old", "shape-new"):
session.delete(session.get(Run, run_id))
session.commit()
# -----------------------------------------------------------------------------
# Naming an artifact from outside the process that made it
#
# A python caller passes the reference it holds. A shell holds nothing, so the
# same input also takes `@run:<id>.<output>` or a bare digest, resolved here
# rather than in each client.
# -----------------------------------------------------------------------------
def artifact_flow() -> FlowDef:
"""A flow taking a dataset somebody else's run produced."""
dataset = MessageSpec(name="dataset", dtype=DType.ARTIFACT)
return FlowDef(
name="study",
mode="batch",
inputs=[FlowInput(spec=dataset)],
nodes=[NodeDef(id="train", requires=[dataset])],
)
@pytest.fixture
def made_artifact():
"""A finished run with one artifact, as a later run would find it."""
digest = "sha256:" + "a1" * 32
reference = {
"digest": digest,
"size": 12,
"media_type": "text/csv",
"name": "cities.csv",
}
run_id = new_run_id()
with Session(db_engine) as session:
session.add(
Run(
id=run_id,
flow_id="prepare",
status="ok",
result={"dataset": reference},
created_at=datetime.now(UTC),
)
)
session.add(
RunArtifact(
run_id=run_id,
name="prepare.dataset",
filename="cities.csv",
node="load",
digest=digest,
size=12,
)
)
session.commit()
yield run_id, reference
with Session(db_engine) as session:
session.delete(session.get(RunArtifact, (run_id, "prepare.dataset")))
session.delete(session.get(Run, run_id))
session.commit()
def test_a_run_reference_resolves_to_what_that_run_produced(made_artifact):
run_id, reference = made_artifact
resolved = resolve_references(
artifact_flow(), {"dataset": f"@run:{run_id}.dataset"}
)
# The producer's own reference, whatever type it made it — the row beside
# it is the fallback, not the first answer.
assert resolved["dataset"] == reference
def test_the_artifact_row_answers_with_the_name_the_node_gave_the_file(made_artifact):
"""Bytes the flow never declared as an output are reachable through the row."""
run_id, reference = made_artifact
with Session(db_engine) as session:
run = session.get(Run, run_id)
run.result = {}
session.add(run)
session.commit()
resolved = resolve_references(
artifact_flow(), {"dataset": f"@run:{run_id}.dataset"}
)
assert resolved["dataset"]["digest"] == reference["digest"]
# The file name, not the message it happened to leave on.
assert resolved["dataset"]["name"] == "cities.csv"
def test_a_bare_digest_resolves_to_the_bytes_under_it(made_artifact):
_run_id, reference = made_artifact
resolved = resolve_references(artifact_flow(), {"dataset": reference["digest"]})
assert resolved["dataset"]["digest"] == reference["digest"]
assert resolved["dataset"]["size"] == 12
def test_a_resolved_reference_passes_the_input_check(made_artifact):
run_id, _reference = made_artifact
flow = artifact_flow()
resolved = resolve_references(flow, {"dataset": f"@run:{run_id}.dataset"})
assert "study.dataset" in seed_values(flow, resolved)
def test_an_output_a_run_never_made_says_what_it_did(made_artifact):
run_id, _reference = made_artifact
with pytest.raises(RunRejected, match="prepare.dataset"):
resolve_references(artifact_flow(), {"dataset": f"@run:{run_id}.weights"})
def test_a_reference_to_no_run_at_all_is_refused():
with pytest.raises(RunRejected, match="no run"):
resolve_references(artifact_flow(), {"dataset": "@run:nothing.dataset"})
def test_an_unknown_digest_is_refused():
with pytest.raises(RunRejected, match="nothing here"):
resolve_references(artifact_flow(), {"dataset": "sha256:" + "b2" * 32})
def test_a_reference_passed_whole_is_left_alone(made_artifact):
"""A python caller already has the object, and hands it over as one."""
_run_id, reference = made_artifact
assert resolve_references(artifact_flow(), {"dataset": reference}) == {
"dataset": reference
}
def chaining_flow() -> FlowDef:
"""A flow taking a json config and a label another run worked out."""
meta = MessageSpec(name="meta", dtype=DType.JSON)
label = MessageSpec(name="label", dtype=DType.STR)
return FlowDef(
name="study",
mode="batch",
inputs=[FlowInput(spec=meta), FlowInput(spec=label)],
nodes=[NodeDef(id="train", requires=[meta, label])],
)
@pytest.fixture
def made_config():
"""A finished run whose result is an object, not bytes."""
meta = {"rows": 256, "source": "builtin"}
run_id = new_run_id()
with Session(db_engine) as session:
session.add(
Run(
id=run_id,
flow_id="generate",
status="ok",
result={"meta": meta, "label": "run-7"},
created_at=datetime.now(UTC),
)
)
session.commit()
yield run_id, meta
with Session(db_engine) as session:
session.delete(session.get(Run, run_id))
session.commit()
def test_a_json_input_may_name_a_run_s_output(made_config):
"""The gap this closes: chaining without pasting the object into a shell."""
run_id, meta = made_config
flow = chaining_flow()
resolved = resolve_references(flow, {"meta": f"@run:{run_id}.meta"})
assert resolved["meta"] == meta
# And it is the value's own type from here on, so the input check passes.
assert seed_values(flow, resolved)["study.meta"] == meta
def test_the_spelling_is_reserved_on_a_text_input_too(made_config):
run_id, _meta = made_config
resolved = resolve_references(chaining_flow(), {"label": f"@run:{run_id}.label"})
assert resolved["label"] == "run-7"
def test_text_that_names_nothing_is_still_left_alone(made_config):
"""Only the two spellings are read as names; everything else is a value."""
_run_id, _meta = made_config
params = {"label": "@run-of-the-mill", "meta": {"rows": 1}}
assert resolve_references(chaining_flow(), params) == params
def test_the_overview_counts_a_flow_the_list_page_would_not_reach(
client, superuser_token_headers
):
"""The list caps at 500 newest; the flow rail needs whole counts."""
with Session(db_engine) as session:
for index in range(3):
session.add(
Run(
id=f"ov-{index}",
flow_id="overviewed",
status="ok" if index else "running",
created_at=datetime(2026, 1, 1 + index, tzinfo=UTC),
)
)
session.commit()
rows = client.get(
f"{settings.API_V1_STR}/runs/overview", headers=superuser_token_headers
).json()
row = next(r for r in rows if r["flow"] == "overviewed")
assert (row["runs"], row["running"], row["queued"]) == (3, 1, 0)
def test_a_running_run_reports_how_long_it_has_been_going(
client, superuser_token_headers
):
"""A duration is only written at the end; until then, time since it began."""
with Session(db_engine) as session:
session.add(
Run(
id="in-flight",
flow_id="timed",
status="running",
created_at=datetime.now(UTC),
started_at=datetime.now(UTC) - timedelta(seconds=30),
)
)
session.commit()
rows = client.get(
f"{settings.API_V1_STR}/runs",
headers=superuser_token_headers,
params={"flow": "timed"},
).json()
assert rows[0]["duration_ms"] >= 30_000
def test_a_run_records_which_caller_asked_for_it(
client, superuser_token_headers, monkeypatch
):
"""The dashboard, the CLI and the SDK are told apart by what they send.
Client-supplied, so the vocabulary is closed: a column nobody can write
free text into is one a table can group by.
"""
seen: dict[str, object] = {}
class Recorder:
def submit(self, name, **kwargs):
seen.update(kwargs)
return Run(
id="cause-1",
flow_id=name,
cause=str(kwargs["cause"]),
created_at=datetime.now(UTC),
)
monkeypatch.setattr(client.app.state, "run_service", Recorder())
url = f"{settings.API_V1_STR}/runs/flows/demo"
answer = client.post(url, headers=superuser_token_headers, json={"cause": "cli"})
assert answer.status_code == 202
assert (seen["cause"], answer.json()["cause"]) == ("cli", "cli")
# Nothing said still means the dashboard, which is the only caller that
# does not name itself.
client.post(url, headers=superuser_token_headers, json={})
assert seen["cause"] == "api"
refused = client.post(
url, headers=superuser_token_headers, json={"cause": "somewhere else"}
)
assert refused.status_code == 422
class _Unusable:
"""Anything reaching this is something a deduplicated submit should not do."""
def __getattr__(self, name):
raise AssertionError(f"a repeated submit must not reach {name}")
def test_a_repeated_submit_returns_the_run_it_already_made():
"""The key is the answer to "did my first attempt land?".
Answered before the flow is even read: a caller retrying a submit it never
got a reply to is owed that run, whatever has been published since.
"""
service = RunService(controller=_Unusable(), queue=_Unusable())
with Session(db_engine) as session:
session.add(
Run(
id="dedup-1",
flow_id="study",
status="running",
idempotency_key="key-abc",
created_at=datetime.now(UTC),
)
)
session.commit()
again = service.submit("study", {"lr": 0.1}, idempotency_key="key-abc")
assert again.id == "dedup-1"
class _OneFlow:
"""A controller that has exactly one flow and no engine behind it."""
def __init__(self, flow):
self.store = self
# As the store reads one from before ids: its key is its id.
self._flow = flow.model_copy(update={"id": flow.id or flow.name})
def read_flow(self, name, draft=False):
return self._flow
def key_index(self):
return {self._flow.name: self._flow.id}
def head(self):
return ""
class _Collect:
def __init__(self):
self.items = []
def add(self, item):
self.items.append(item)
def test_a_run_records_the_inputs_it_actually_starts_from():
"""An input left out takes its declared value, and the row says so.
`params = {}` could not tell a run that took every default from one
submitted with those same numbers spelled out — and an export of the
first had a blank cell where its `lr` should be.
"""
flow = FlowDef(
name="study",
mode="batch",
inputs=[
FlowInput(spec=MessageSpec(name="lr", dtype=DType.FLOAT), initial=0.01),
FlowInput(spec=MessageSpec(name="epochs", dtype=DType.INT)),
],
)
service = RunService(controller=_OneFlow(flow), queue=_Collect())
made = []
try:
defaulted = service.submit("study", {"epochs": 5})
made.append(defaulted.id)
assert defaulted.params == {"lr": 0.01, "epochs": 5}
# Spelling out the declared value is the same run, and now reads as it.
spelled = service.submit("study", {"lr": 0.01, "epochs": 5})
made.append(spelled.id)
assert spelled.params_digest == defaulted.params_digest
# Only what the caller chose says which value was the flow's own.
assert (defaulted.passed, spelled.passed) == (["epochs"], ["epochs", "lr"])
# And what the flow declared then, so an input it adds later is not
# drawn on this run as a default it never received.
assert defaulted.declared == {"lr": 0.01, "epochs": None}
finally:
with Session(db_engine) as session:
for run in session.exec(select(Run).where(col(Run.id).in_(made))).all():
session.delete(run)
session.commit()
def test_the_seed_is_recorded_the_same_way_however_it_arrived():
"""One field an export should not have to coalesce two columns for.
`--seed 1` fills the run's own column; a flow declaring a `seed` input
fills the parameter. Both are the seed the run used, so both are written.
"""
flow = FlowDef(
name="seeded",
mode="batch",
inputs=[FlowInput(spec=MessageSpec(name="seed", dtype=DType.INT), initial=42)],
)
service = RunService(controller=_OneFlow(flow), queue=_Collect())
made = []
try:
passed = service.submit("seeded", {}, seed=1)
made.append(passed.id)
assert (passed.seed, passed.params) == (1, {"seed": 1})
# Nothing passed: the declared value is the seed it ran with, and the
# run-level column says so rather than staying empty.
defaulted = service.submit("seeded", {})
made.append(defaulted.id)
assert (defaulted.seed, defaulted.params) == (42, {"seed": 42})
# A parameter still outranks the run's own seed, as it always has —
# and the column follows it rather than reporting the one that lost.
both = service.submit("seeded", {"seed": 7}, seed=1)
made.append(both.id)
assert (both.seed, both.params) == (7, {"seed": 7})
finally:
with Session(db_engine) as session:
for run in session.exec(select(Run).where(col(Run.id).in_(made))).all():
session.delete(run)
session.commit()
def test_a_run_made_elsewhere_is_imported_as_it_ended(tmp_path):
"""History from before the move, completed like a submit and borrowing
nothing from today: no commit, no code digest, nothing queued."""
flow = FlowDef(
name="history",
mode="batch",
inputs=[
FlowInput(spec=MessageSpec(name="lr", dtype=DType.FLOAT), initial=0.01),
FlowInput(spec=MessageSpec(name="epochs", dtype=DType.INT)),
],
)
store = ArtifactStore(tmp_path)
queue = _Collect()
service = RunService(controller=_OneFlow(flow), queue=queue, artifacts=store)
started = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC)
entry = {
"flow": "history",
"external_id": "mlflow:1/abc",
"params": {"epochs": 3},
"seed": 1001,
"status": "ok",
"created_at": started,
"started_at": started,
"finished_at": started + timedelta(seconds=90),
"result": {"final": {"mse": 7.0}},
"metrics": [
{"name": "history.mse", "step": 0, "ts": 1.0, "value": 9.0},
{"name": "history.mse", "step": 1, "ts": 2.0, "value": float("nan")},
{"name": "history.mse", "step": 1, "ts": 3.0, "value": 7.0},
],
"artifacts": [
{
"name": "model",
"filename": "model.txt",
"media_type": "text/plain",
"content_base64": base64.b64encode(b"weights").decode(),
}
],
"labels": ["imported"],
"group_id": "mlflow:1",
}
made: list[str] = []
try:
[answer] = service.import_runs([entry], actor="me@example.com")
made.append(answer["id"])
assert answer["created"] is True
with Session(db_engine) as session:
run = session.get(Run, answer["id"])
assert (run.cause, run.status, run.external_id) == (
"import",
"ok",
"mlflow:1/abc",
)
# The declared default is folded in, as a submit would have.
assert run.params == {"lr": 0.01, "epochs": 3}
assert run.params_digest
assert run.duration_ms == 90_000
assert (run.commit, run.origin_commit, run.code_digest) == ("", "", "")
points = session.exec(
select(RunMetric).where(col(RunMetric.run_id) == run.id)
).all()
assert sorted((point.step, point.value) for point in points) == [
(0, 9.0),
(1, 7.0),
]
[artifact] = session.exec(
select(RunArtifact).where(col(RunArtifact.run_id) == run.id)
).all()
assert artifact.filename == "model.txt"
assert b"".join(store.read(artifact.digest)) == b"weights"
assert queue.items == []
# The same history again adds nothing, and says so.
assert service.import_runs([entry]) == [{**answer, "created": False}]
# One refused entry takes the good one beside it down with it.
good = {**entry, "external_id": "mlflow:1/def", "artifacts": []}
bad = {**good, "external_id": "mlflow:1/ghi", "params": {"epochs": "three"}}
with pytest.raises(RunRejected, match="mlflow:1/ghi"):
service.import_runs([good, bad])
assert service._by_external("mlflow:1/def") is None
finally:
with Session(db_engine) as session:
for table in (RunMetric, RunArtifact, Run):
column = Run.id if table is Run else table.run_id
for row in session.exec(select(table).where(col(column).in_(made))):
session.delete(row)
session.commit()
def test_an_import_answers_what_it_added(client, superuser_token_headers, monkeypatch):
"""So a client can report what it actually added, and pages the rest."""
seen: dict[str, object] = {}
class Recorder:
def import_runs(self, entries, actor=""):
seen.update(entries=entries, actor=actor)
return [
{
"external_id": one["external_id"],
"id": f"run-{index}",
"created": True,
}
for index, one in enumerate(entries)
]
monkeypatch.setattr(client.app.state, "run_service", Recorder())
url = f"{settings.API_V1_STR}/runs/import"
entry = {
"flow": "history",
"external_id": "mlflow:1/abc",
"created_at": "2026-01-02T03:04:05Z",
}
answer = client.post(url, headers=superuser_token_headers, json={"runs": [entry]})
assert answer.status_code == 202
assert answer.json() == [
{"external_id": "mlflow:1/abc", "id": "run-0", "created": True}
]
assert seen["entries"][0]["status"] == "ok"
assert seen["actor"] == settings.FIRST_SUPERUSER
too_many = client.post(
url, headers=superuser_token_headers, json={"runs": [entry] * 201}
)
assert too_many.status_code == 422
DRAFT_SOURCE = "def process():\n return {'answer': 'draft'}\n"
PUBLISHED_SOURCE = "def process():\n return {'answer': 'published'}\n"
def test_a_draft_run_executes_the_draft(tmp_path):
"""The flag was checked at submit and forgotten by the time it ran.
Which made trying an edit before publishing it impossible from anywhere:
the submit validated the working copy and the driver then executed what
was published, and the two only agree when there is no draft.
"""
flow = FlowDef(
name="study",
mode="batch",
outputs=["answer"],
nodes=[
NodeDef(
id="answer",
provides=[MessageSpec(name="answer", dtype=DType.STR)],
)
],
)
store = FlowStore(tmp_path / "flows")
store.write_flow(flow)
store.write_node_source("study", "answer", PUBLISHED_SOURCE)
store.write_node_source("study", "answer", DRAFT_SOURCE, draft=True)
service = RunService(controller=FlowController(store), queue=_Collect())
made = []
try:
for draft, expected in ((True, "draft"), (False, "published")):
run = service.submit("study", {}, draft=draft)
made.append(run.id)
assert run.draft is draft
service._drive(run.id)
with Session(db_engine) as session:
stored = session.get(Run, run.id)
assert stored.status == "ok", stored.status_reason
assert stored.result == {"answer": expected}
finally:
with Session(db_engine) as session:
for run in session.exec(select(Run).where(col(Run.id).in_(made))).all():
session.delete(run)
for node in session.exec(
select(RunNode).where(col(RunNode.run_id).in_(made))
).all():
session.delete(node)
session.commit()
def test_a_failed_stage_skips_what_waits_on_it_and_fails_the_run(tmp_path):
"""Stages after a failure were called without their input and raised
TypeError, and a run whose only node could not load finished `ok`."""
model, score = (MessageSpec(name=n, dtype=DType.FLOAT) for n in ("model", "score"))
staged = FlowDef(
name="staged",
mode="batch",
nodes=[
NodeDef(id="fit", provides=[model]),
NodeDef(id="readout", requires=[model], provides=[score]),
NodeDef(id="evaluate", requires=[score]),
],
)
broken = FlowDef(name="broken", mode="batch", nodes=[NodeDef(id="fit")])
store = FlowStore(tmp_path / "flows")
store.write_flow(staged)
store.write_flow(broken)
store.write_node_source("staged", "fit", "def process():\n raise ValueError\n")
store.write_node_source(
"staged", "readout", "def process(model):\n return {'score': model}\n"
)
store.write_node_source("staged", "evaluate", "def process(score):\n pass\n")
store.write_node_source("broken", "fit", "def process(:\n")
service = RunService(controller=FlowController(store), queue=_Collect())
made = [service.submit(name, {}).id for name in ("staged", "broken")]
try:
for run_id in made:
service._drive(run_id)
with Session(db_engine) as session:
ran, failed = (session.get(Run, run_id) for run_id in made)
assert ran.status == failed.status == "error"
assert failed.status_reason.startswith("Node 'broken.fit' could not")
rows = session.exec(select(RunNode).where(col(RunNode.run_id).in_(made)))
assert {row.node: row.status for row in rows} == {
"staged.fit": "error",
"staged.readout": "skipped",
"staged.evaluate": "skipped",
"broken.fit": "error",
}
finally:
with Session(db_engine) as session:
for node in session.exec(
select(RunNode).where(col(RunNode.run_id).in_(made))
).all():
session.delete(node)
for run in session.exec(select(Run).where(col(Run.id).in_(made))).all():
session.delete(run)
session.commit()
def test_a_stage_with_nothing_on_an_input_fails_instead_of_being_called(tmp_path):
"""The producer succeeded but published nothing on the port it waits on."""
model, score, report = (
MessageSpec(name=n, dtype=DType.FLOAT) for n in ("model", "score", "report")
)
flow = FlowDef(
name="split",
mode="batch",
nodes=[
NodeDef(id="fit", provides=[model, score]),
NodeDef(id="readout", requires=[score], provides=[report]),
NodeDef(id="evaluate", requires=[report]),
],
)
store = FlowStore(tmp_path / "flows")
store.write_flow(flow)
store.write_node_source(
"split", "fit", "def process():\n return {'model': 1.0}\n"
)
store.write_node_source(
"split", "readout", "def process(score):\n return {'report': score}\n"
)
store.write_node_source("split", "evaluate", "def process(report):\n pass\n")
service = RunService(controller=FlowController(store), queue=_Collect())
run_id = service.submit("split", {}).id
try:
service._drive(run_id)
with Session(db_engine) as session:
assert session.get(Run, run_id).status == "error"
rows = {
row.node: row
for row in session.exec(select(RunNode).where(RunNode.run_id == run_id))
}
assert {node: row.status for node, row in rows.items()} == {
"split.fit": "ok",
"split.readout": "error",
"split.evaluate": "skipped",
}
assert rows["split.readout"].error == "No value on input 'score' from 'fit'"
finally:
with Session(db_engine) as session:
for node in session.exec(select(RunNode).where(RunNode.run_id == run_id)):
session.delete(node)
session.delete(session.get(Run, run_id))
session.commit()
def test_a_retry_is_a_new_run_that_names_the_one_it_repeats():
"""The way back from a run an engine restart interrupted.
Re-issuing the whole sweep is the blunt version; this keeps the group, so
what was missing is filled in rather than run twice.
"""
flow = FlowDef(
name="study",
mode="batch",
inputs=[
FlowInput(spec=MessageSpec(name="lr", dtype=DType.FLOAT), initial=0.01)
],
)
queue = _Collect()
service = RunService(controller=_OneFlow(flow), queue=queue)
made = []
with Session(db_engine) as session:
session.add(
Run(
id="abandoned-1",
flow_id="study",
status="abandoned",
params={"lr": 0.3},
group_id="sweep-9",
draft=True,
created_at=datetime.now(UTC),
)
)
session.add(
Run(
id="going-1",
flow_id="study",
status="running",
created_at=datetime.now(UTC),
)
)
session.commit()
try:
again = service.retry("abandoned-1", actor="someone@example.com")
made.append(again.id)
assert again.parent_id == "abandoned-1"
assert (again.params, again.group_id, again.cause) == (
{"lr": 0.3},
"sweep-9",
"retry",
)
# What it was a run of comes with it: retrying a run of the working
# copy that silently ran the published one would say nothing at all.
assert again.draft is True
assert queue.items[-1].run_id == again.id
# A run that has not finished is cancelled, not retried.
with pytest.raises(RunRejected):
service.retry("going-1")
with pytest.raises(RunRejected):
service.retry("never-existed")
finally:
with Session(db_engine) as session:
for run in session.exec(
select(Run).where(col(Run.id).in_([*made, "abandoned-1", "going-1"]))
).all():
session.delete(run)
session.commit()
def test_runs_left_queued_are_woken_by_the_next_engine():
"""The row is the record, and the work item is not part of its transaction.
An in-memory queue loses the item with the process and a stream item
nobody claimed is nobody's, so without this a run submitted just before a
restart waits for an engine that will never be told about it.
"""
queue = _Collect()
service = RunService(controller=_Unusable(), queue=queue)
with Session(db_engine) as session:
session.add(
Run(
id="orphan-1",
flow_id="study",
status="queued",
created_at=datetime.now(UTC),
)
)
session.add(
Run(
id="finished-1",
flow_id="study",
status="ok",
created_at=datetime.now(UTC),
)
)
session.commit()
try:
service._requeue_queued()
assert [item.run_id for item in queue.items] == ["orphan-1"]
assert queue.items[0].flow == "study"
finally:
with Session(db_engine) as session:
for run in session.exec(
select(Run).where(col(Run.id).in_(["orphan-1", "finished-1"]))
).all():
session.delete(run)
session.commit()
class _BusyWorkers(_OneFlow):
"""Builds a run's pipeline the way a full worker pool does: eventually."""
events = None
def __init__(self, flow):
super().__init__(flow)
self.building = threading.Event()
self.free = threading.Event()
def build_run_pipeline(self, flow, **_kwargs):
self.building.set()
self.free.wait(10)
return object() # never run: the test cancels it first
def test_a_run_waiting_for_a_worker_keeps_its_lease_and_can_be_cancelled(
monkeypatch,
):
"""Only a run whose pipeline was built had its lease kept.
Building one loads every python node in a worker, so a run claimed while
all of them were busy was swept as `abandoned` and finished `ok` minutes
later, and a cancel in that window was refused.
"""
controller = _BusyWorkers(FlowDef(name="study", mode="batch"))
service = RunService(controller=controller, queue=_Collect())
run = service.submit("study", {})
driver = threading.Thread(target=service._drive, args=(run.id,))
driver.start()
try:
assert controller.building.wait(5)
# As if it had waited past the cutoff, then one pass of the keeper:
# its lease is renewed before the sweep looks at it.
with Session(db_engine) as session:
session.get(Run, run.id).lease_at = datetime.now(UTC) - timedelta(
seconds=2 * LEASE_STALE_S
)
session.commit()
monkeypatch.setattr("fluksio.flow.runs.LEASE_INTERVAL_S", 0)
monkeypatch.setattr("fluksio.flow.runs.SWEEP_INTERVAL_S", 0)
sweep = service._sweep_abandoned
def sweep_once(now):
sweep(now)
service._stop.set()
monkeypatch.setattr(service, "_sweep_abandoned", sweep_once)
service._keep_leases()
with Session(db_engine) as session:
assert session.get(Run, run.id).status == "running"
assert service.cancel(run.id)
with Session(db_engine) as session:
assert session.get(Run, run.id).status == "cancelled"
# A worker frees up: the build returns and the run must not start.
controller.free.set()
driver.join(5)
with Session(db_engine) as session:
stored = session.get(Run, run.id)
assert stored.status == "cancelled"
assert stored.status_reason == "Cancelled before it started"
finally:
controller.free.set()
driver.join(5)
with Session(db_engine) as session:
session.delete(session.get(Run, run.id))
session.commit()
class _Bus:
def __init__(self):
self.events = []
def publish(self, event):
self.events.append(event)
def test_cancelling_a_run_does_not_pause_the_live_flow_it_shares_a_key_with():
"""A run's pipeline publishes on the engine's bus under the flow's key, so
its pause marked the live flow paused in every open editor."""
class _Idle(_OneFlow):
workers = None
remote = None
service = RunService(
controller=_Idle(FlowDef(name="study", mode="batch")), queue=_Collect()
)
run = service.submit("study", {})
bus = _Bus()
pipeline = Pipeline(events=bus)
service._active[run.id] = pipeline
try:
assert service.cancel(run.id)
assert pipeline.is_disabled("study")
assert not [e for e in bus.events if e["type"] == "flow_paused"]
finally:
with Session(db_engine) as session:
session.delete(session.get(Run, run.id))
session.commit()
def test_a_key_nobody_used_submits_normally(
client, superuser_token_headers, monkeypatch
):
"""The route carries the key through; without one nothing changes."""
seen: dict[str, object] = {}
class Recorder:
def submit(self, name, **kwargs):
seen.update(kwargs)
return Run(id="keyed-1", flow_id=name, created_at=datetime.now(UTC))
monkeypatch.setattr(client.app.state, "run_service", Recorder())
answer = client.post(
f"{settings.API_V1_STR}/runs/flows/demo",
headers=superuser_token_headers,
json={"idempotency_key": "key-xyz"},
)
assert answer.status_code == 202
assert seen["idempotency_key"] == "key-xyz"
def test_overview_is_not_read_as_a_run_id(client, superuser_token_headers):
"""`/overview` is declared before `/{run_id}`, which would swallow it."""
answer = client.get(
f"{settings.API_V1_STR}/runs/overview", headers=superuser_token_headers
)
assert answer.status_code == 200
assert isinstance(answer.json(), list)
# -----------------------------------------------------------------------------
# Deleting a run
#
# The foreign keys cascade; what these guard is that deleting a run takes the
# children with it and refuses a run the driver is still writing to.
# -----------------------------------------------------------------------------
@pytest.fixture
def deletable_run():
"""One finished run with a node, a number and an artifact row hanging off it."""
run_id = "del-1"
with Session(db_engine) as session:
session.add(
Run(id=run_id, flow_id="deleted", status="ok", created_at=datetime.now(UTC))
)
session.add(RunNode(run_id=run_id, node="deleted.a", status="ok"))
session.add(RunMetric(run_id=run_id, name="deleted.loss", step=0, value=1.0))
session.add(
RunArtifact(
run_id=run_id,
name="deleted.out",
filename="out.bin",
node="a",
digest="d" * 64,
size=7,
)
)
session.commit()
yield run_id
with Session(db_engine) as session:
run = session.get(Run, run_id)
if run is not None:
session.delete(run)
session.commit()
def test_deleting_a_run_takes_its_children_with_it(
client, superuser_token_headers, deletable_run
):
"""The foreign keys cascade, which needs them on for every connection."""
answer = client.delete(
f"{settings.API_V1_STR}/runs/{deletable_run}", headers=superuser_token_headers
)
assert answer.status_code == 204
with Session(db_engine) as session:
assert session.get(Run, deletable_run) is None
for table in (RunNode, RunMetric, RunArtifact):
left = session.exec(
select(table).where(col(table.run_id) == deletable_run)
).all()
assert left == [], f"{table.__name__} rows outlived the run"
def test_a_node_finishing_after_its_run_was_deleted_is_dropped_quietly(caplog):
"""The constraint refuses its row, and that is a no-op rather than an error."""
service = RunService(controller=_OneFlow(FlowDef(name="gone")), queue=_Collect())
with caplog.at_level(logging.INFO, logger="fluksio.flow.runs"):
service._record_node(
Run(id="del-gone", flow_id="gone"),
"gone",
NodeOutcome(node="gone.a", ok=True),
)
with Session(db_engine) as session:
assert session.get(RunNode, {"run_id": "del-gone", "node": "gone.a"}) is None
assert "is gone" in caplog.text
assert not [record for record in caplog.records if record.levelno >= logging.ERROR]
def test_deleting_a_run_that_is_not_there_is_a_404(client, superuser_token_headers):
answer = client.delete(
f"{settings.API_V1_STR}/runs/nope-1", headers=superuser_token_headers
)
assert answer.status_code == 404
def test_a_running_run_is_refused_rather_than_raced(client, superuser_token_headers):
"""The driver writes its nodes back at the end; they would have no run."""
run_id = "del-live"
with Session(db_engine) as session:
session.add(
Run(
id=run_id,
flow_id="deleted",
status="running",
created_at=datetime.now(UTC),
)
)
session.commit()
try:
answer = client.delete(
f"{settings.API_V1_STR}/runs/{run_id}", headers=superuser_token_headers
)
assert answer.status_code == 409
assert "Cancel it" in answer.json()["detail"]
with Session(db_engine) as session:
assert session.get(Run, run_id) is not None
finally:
with Session(db_engine) as session:
run = session.get(Run, run_id)
if run is not None:
session.delete(run)
session.commit()
def test_deleting_a_flows_runs_clears_that_flow_only(client, superuser_token_headers):
"""The counterpart to the list's `flow` filter — what a reseed needs.
Deleting the flow itself leaves the history standing, so this is the tool
that clears one. It refuses while a run of that flow is still going, for
the same reason deleting a single run does.
"""
made = datetime.now(UTC)
url = f"{settings.API_V1_STR}/runs"
with Session(db_engine) as session:
session.add(Run(id="bulk-1", flow_id="swept", status="ok", created_at=made))
session.add(RunNode(run_id="bulk-1", node="swept.a", status="ok"))
session.add(RunMetric(run_id="bulk-1", name="swept.loss", step=0, value=1.0))
session.add(
Run(id="bulk-2", flow_id="swept", status="running", created_at=made)
)
session.add(Run(id="keep-1", flow_id="other", status="ok", created_at=made))
session.commit()
busy = client.delete(url, params={"flow": "swept"}, headers=superuser_token_headers)
assert busy.status_code == 409
assert "bulk-2" in busy.json()["detail"]
with Session(db_engine) as session:
session.get(Run, "bulk-2").status = "cancelled"
session.commit()
answer = client.delete(
url, params={"flow": "swept"}, headers=superuser_token_headers
)
assert answer.status_code == 200
assert "2" in answer.json()["message"]
with Session(db_engine) as session:
assert session.exec(select(Run).where(col(Run.flow_id) == "swept")).all() == []
for table in (RunNode, RunMetric):
assert (
session.exec(select(table).where(col(table.run_id) == "bulk-1")).all()
== []
)
assert session.get(Run, "keep-1") is not None
session.delete(session.get(Run, "keep-1"))
session.commit()
def test_history_also_clears_a_flows_cascades(client, superuser_token_headers):
"""What deleting a flow together with its runs asks for."""
made = datetime.now(UTC)
url = f"{settings.API_V1_STR}/runs"
with Session(db_engine) as session:
session.add(Run(id="hist-1", flow_id="gone", status="ok", created_at=made))
session.add(FlowRun(id="hist-1", flow_id="gone", source="run", started_at=made))
session.add(FlowRun(id="hist-2", flow_id="gone", started_at=made))
session.add(FlowRun(id="hist-3", flow_id="other", started_at=made))
session.commit()
plain = client.delete(url, params={"flow": "gone"}, headers=superuser_token_headers)
assert plain.status_code == 200
with Session(db_engine) as session:
assert session.get(FlowRun, "hist-2") is not None
answer = client.delete(
url, params={"flow": "gone", "history": True}, headers=superuser_token_headers
)
assert answer.status_code == 200
with Session(db_engine) as session:
left = session.exec(select(FlowRun).where(col(FlowRun.flow_id) == "gone")).all()
assert left == []
session.delete(session.get(FlowRun, "hist-3"))
session.commit()
# -----------------------------------------------------------------------------
# A cached node's curve
#
# A hit replays no emissions, so the series stays in the run that recorded it
# and the run reusing it points there. Reading either one answers the same.
# -----------------------------------------------------------------------------
@pytest.fixture
def reused_run():
"""A run of `quick` whose node was restored from a run of `train`."""
with Session(db_engine) as session:
made = datetime.now(UTC)
session.add(Run(id="src-1", flow_id="train", status="ok", created_at=made))
session.add(Run(id="reuse-1", flow_id="quick", status="ok", created_at=made))
session.add(RunNode(run_id="src-1", node="train.fit", status="ok"))
session.add(
RunNode(
run_id="reuse-1",
node="quick.fit",
status="cached",
cached_from="src-1",
)
)
for step, value in enumerate([3.0, 2.0, 1.0]):
session.add(
RunMetric(
run_id="src-1",
name="train.loss",
step=step,
node="train.fit",
value=value,
)
)
session.commit()
yield
with Session(db_engine) as session:
for row in session.exec(select(RunMetric)).all():
session.delete(row)
for row in session.exec(select(RunNode)).all():
session.delete(row)
for run_id in ("src-1", "reuse-1"):
run = session.get(Run, run_id)
if run is not None:
session.delete(run)
session.commit()
@pytest.mark.usefixtures("reused_run")
def test_a_cached_node_answers_with_the_curve_it_was_restored_from(
client, superuser_token_headers
):
points = client.get(
f"{settings.API_V1_STR}/runs/reuse-1/metrics",
headers=superuser_token_headers,
).json()
# Named for the flow that asked, not the one that recorded it.
assert [point["name"] for point in points] == ["quick.loss"] * 3
assert [point["value"] for point in points] == [3.0, 2.0, 1.0]
named = client.get(
f"{settings.API_V1_STR}/runs/reuse-1/metrics",
params={"name": "quick.loss"},
headers=superuser_token_headers,
).json()
assert len(named) == 3
@pytest.mark.usefixtures("reused_run")
def test_a_curve_whose_run_is_gone_is_empty_rather_than_an_error(
client, superuser_token_headers
):
"""Delete the run a cache hit points at and the reusing run holds nothing."""
with Session(db_engine) as session:
session.delete(session.get(Run, "src-1"))
session.commit()
answer = client.get(
f"{settings.API_V1_STR}/runs/reuse-1/metrics",
headers=superuser_token_headers,
)
assert answer.status_code == 200
assert answer.json() == []
# And the run says why, rather than leaving an empty chart to explain itself.
(node,) = client.get(
f"{settings.API_V1_STR}/runs/reuse-1", headers=superuser_token_headers
).json()["nodes"]
assert node["curve_lost"] is True
def test_a_node_answers_with_what_it_returned(client, superuser_token_headers):
"""What the stage cache stored, and a 404 that says why for a node without."""
run_id = new_run_id()
with Session(db_engine) as session:
session.add(
Run(id=run_id, flow_id="study", status="ok", created_at=datetime.now(UTC))
)
session.add(RunNode(run_id=run_id, node="fit", outputs='{"study.loss":0.5}'))
session.add(RunNode(run_id=run_id, node="plot", status="error"))
session.commit()
url = f"{settings.API_V1_STR}/runs/{run_id}/nodes"
try:
kept = client.get(f"{url}/fit/outputs", headers=superuser_token_headers)
assert kept.json() == {"study.loss": 0.5}
none = client.get(f"{url}/plot/outputs", headers=superuser_token_headers)
assert none.status_code == 404
assert "kept no outputs" in none.json()["detail"]
absent = client.get(f"{url}/nope/outputs", headers=superuser_token_headers)
assert absent.status_code == 404
finally:
with Session(db_engine) as session:
for row in session.exec(
select(RunNode).where(col(RunNode.run_id) == run_id)
).all():
session.delete(row)
session.delete(session.get(Run, run_id))
session.commit()
def _metric(run_id: str, name: str, step: int, value: float, ts: float) -> RunMetric:
return RunMetric(run_id=run_id, name=name, step=step, value=value, ts=ts)
@pytest.fixture
def plotted():
"""A run with a loss curve and an epoch counter beside it."""
run_id = new_run_id()
with Session(db_engine) as session:
session.add(
Run(id=run_id, flow_id="study", status="ok", created_at=datetime.now(UTC))
)
for step, (loss, epoch) in enumerate([(1.0, 10.0), (0.5, 20.0), (0.25, 30.0)]):
session.add(_metric(run_id, "study.loss", step, loss, 100.0 + step * 5))
session.add(_metric(run_id, "study.epoch", step, epoch, 100.0 + step * 5))
session.commit()
yield run_id
with Session(db_engine) as session:
for row in session.exec(
select(RunMetric).where(col(RunMetric.run_id) == run_id)
).all():
session.delete(row)
session.delete(session.get(Run, run_id))
session.commit()
def test_the_metric_names_of_a_selection_are_exact(
client, superuser_token_headers, plotted
):
"""Names are flow-qualified, so this is what says which spellings exist.
Read over the whole selection rather than off the newest run that measured
anything, which is what missed a name only an older run ever wrote — here,
the older run's `study.grad` against the newer one's two.
"""
older = new_run_id()
with Session(db_engine) as session:
session.add(
Run(
id=older,
flow_id="study",
status="ok",
created_at=datetime.now(UTC) - timedelta(hours=1),
)
)
session.add(_metric(older, "study.grad", 0, 0.1, 50.0))
session.commit()
try:
both = client.get(
f"{settings.API_V1_STR}/runs/metrics/names",
params={"ids": f"{plotted},{older}"},
headers=superuser_token_headers,
)
assert both.status_code == 200
assert both.json() == ["study.epoch", "study.grad", "study.loss"]
# The same question by filter rather than by name.
assert client.get(
f"{settings.API_V1_STR}/runs/metrics/names",
params={"flow": "study"},
headers=superuser_token_headers,
).json() == ["study.epoch", "study.grad", "study.loss"]
# A selection that recorded nothing has nothing to offer, which is not
# an error: a run opened before its first reading is the ordinary case.
assert (
client.get(
f"{settings.API_V1_STR}/runs/metrics/names",
params={"ids": "no-such-run"},
headers=superuser_token_headers,
).json()
== []
)
finally:
with Session(db_engine) as session:
for row in session.exec(
select(RunMetric).where(col(RunMetric.run_id) == older)
).all():
session.delete(row)
session.delete(session.get(Run, older))
session.commit()
def test_a_comparison_is_plotted_against_the_step_by_default(
client, superuser_token_headers, plotted
):
answer = client.get(
f"{settings.API_V1_STR}/runs/series/compare",
params={"ids": plotted, "metric": "study.loss"},
headers=superuser_token_headers,
).json()
assert answer["x"] == "step"
assert answer["lines"][0]["points"] == [[0.0, 1.0], [1.0, 0.5], [2.0, 0.25]]
def test_time_is_measured_from_this_runs_own_first_reading(
client, superuser_token_headers, plotted
):
"""Runs started hours apart still lie on top of each other."""
answer = client.get(
f"{settings.API_V1_STR}/runs/series/compare",
params={"ids": plotted, "metric": "study.loss", "x": "time"},
headers=superuser_token_headers,
).json()
assert answer["x"] == "time"
assert answer["lines"][0]["points"] == [[0.0, 1.0], [5.0, 0.5], [10.0, 0.25]]
def test_one_metric_can_be_plotted_against_another(
client, superuser_token_headers, plotted
):
answer = client.get(
f"{settings.API_V1_STR}/runs/series/compare",
params={"ids": plotted, "metric": "study.loss", "x": "study.epoch"},
headers=superuser_token_headers,
).json()
assert answer["lines"][0]["points"] == [[10.0, 1.0], [20.0, 0.5], [30.0, 0.25]]
def test_a_step_the_x_metric_never_reached_is_left_out(
client, superuser_token_headers, plotted
):
"""The join is on the step, which is the only thing two series share."""
with Session(db_engine) as session:
session.add(_metric(plotted, "study.loss", 3, 0.1, 120.0))
session.commit()
answer = client.get(
f"{settings.API_V1_STR}/runs/series/compare",
params={"ids": plotted, "metric": "study.loss", "x": "study.epoch"},
headers=superuser_token_headers,
).json()
assert [point[0] for point in answer["lines"][0]["points"]] == [10.0, 20.0, 30.0]
def test_a_run_records_the_code_it_started_with_not_the_code_it_was_queued_with():
"""A sweep queues every run at once and the tree moves while it waits.
`_restamp` is what runs at claim time, so the digest the record keeps is
the one the run actually executed.
"""
service = RunService(controller=_Unusable(), queue=_Unusable())
with Session(db_engine) as session:
session.add(
Run(
id="stamp-1",
flow_id="study",
status="queued",
code_digest="at-submit",
created_at=datetime.now(UTC),
)
)
session.commit()
run = session.get(Run, "stamp-1")
assert service._restamp(run, "at-claim") == "at-claim"
with Session(db_engine) as session:
assert session.get(Run, "stamp-1").code_digest == "at-claim"
def test_a_tree_that_did_not_move_is_not_written_again():
service = RunService(controller=_Unusable(), queue=_Unusable())
with Session(db_engine) as session:
session.add(
Run(
id="stamp-2",
flow_id="study",
status="queued",
code_digest="same",
created_at=datetime.now(UTC),
)
)
session.commit()
run = session.get(Run, "stamp-2")
assert service._restamp(run, "same") == "same"
# -----------------------------------------------------------------------------
# Export
#
# Two tables an analysis reads: the long one a curve is plotted from, and the
# wide one arms are compared in. Both carry the run id on every row.
# -----------------------------------------------------------------------------
@pytest.fixture
def exported():
"""Two runs of one flow: two curves each, and records on both sides.
The numbers a node returns are usually inside a record rather than at the
top of the result, so the fixture is shaped the way a real one is.
"""
made = datetime.now(UTC)
with Session(db_engine) as session:
for index, (lr, acc) in enumerate([(0.1, 0.5), (0.01, 0.25)]):
run_id = f"exp-{index}"
session.add(
Run(
id=run_id,
flow_id="export-study",
status="ok",
params={
"lr": lr,
"epochs": 10,
"config": {"model": "mlp", "depth": index + 1},
},
result={
"acc": acc,
"note": "n/a",
"final_metrics": {"train_loss": acc * 2},
"test_metrics": {"known": {"perfect": 1.0}},
# Keyed by a number written with its decimal point,
# which no dotted path can address.
"spectrum": {"-1.0": acc, "0.25": {"mean": acc}},
},
created_at=made + timedelta(seconds=index),
)
)
for name in ("study.loss", "study.val"):
for step in range(4):
session.add(_metric(run_id, name, step, float(step), 100.0 + step))
session.commit()
yield ["exp-0", "exp-1"]
with Session(db_engine) as session:
for run_id in ("exp-0", "exp-1"):
for row in session.exec(
select(RunMetric).where(col(RunMetric.run_id) == run_id)
).all():
session.delete(row)
run = session.get(Run, run_id)
if run is not None:
session.delete(run)
session.commit()
def _lines(answer) -> list[dict]:
return [json.loads(line) for line in answer.text.splitlines() if line]
def test_an_export_strides_each_series_and_names_its_run(
client, superuser_token_headers, exported
):
"""Every second point of every curve — not every second row of all of them."""
answer = client.get(
f"{settings.API_V1_STR}/runs/export/metrics",
params={"ids": ",".join(exported), "stride": 2, "format": "jsonl"},
headers=superuser_token_headers,
)
assert answer.status_code == 200
rows = _lines(answer)
assert {row["run"] for row in rows} == set(exported)
curves: dict[tuple[str, str], list[int]] = {}
for row in rows:
curves.setdefault((row["run"], row["name"]), []).append(row["step"])
assert len(curves) == 4
assert all(steps == [0, 2] for steps in curves.values())
def test_an_exported_run_row_carries_every_recorded_input(
client, superuser_token_headers, exported
):
"""Every input is a column, so the schema does not move with the selection."""
def export(**extra):
answer = client.get(
f"{settings.API_V1_STR}/runs/export/runs",
params={"ids": ",".join(exported), **extra},
headers=superuser_token_headers,
)
assert answer.status_code == 200
return answer
rows = _lines(export(format="jsonl"))
assert [row["id"] for row in rows] == ["exp-1", "exp-0"]
assert {row["param.lr"] for row in rows} == {0.1, 0.01}
# `epochs` is the same on both runs and stays a column anyway: which runs
# were asked for is not something a downstream filter should have to know.
assert rows[0]["param.epochs"] == 10
assert rows[0]["param.config.model"] == "mlp"
assert {row["param.config.depth"] for row in rows} == {1, 2}
narrowed = _lines(export(format="jsonl", params="epochs"))[0]
assert "param.epochs" in narrowed
assert "param.lr" not in narrowed
# A number inside a record is a column of its own, however deep; a string
# is not one of the run's numbers wherever it sits.
assert rows[0]["metric.acc"] == 0.25
assert rows[0]["metric.final_metrics.train_loss"] == 0.5
assert rows[0]["metric.test_metrics.known.perfect"] == 1.0
assert "metric.note" not in rows[0]
# A key holding the separator cannot be read back, so it is not a column.
assert not any(key.startswith("metric.spectrum") for key in rows[0])
named = _lines(export(format="jsonl", metrics="test_metrics.known.perfect"))
assert list(named[0])[-1] == "metric.test_metrics.known.perfect"
assert "metric.acc" not in named[0]
# csv is the default, and the columns are in the order the header names.
header = export().text.splitlines()[0]
assert header.startswith("id,flow,flow_name,status,")
assert header.endswith(
"param.config.depth,param.config.model,param.epochs,param.lr,"
"metric.acc,metric.final_metrics.train_loss,"
"metric.test_metrics.known.perfect"
)
# `final_metrics` alone, short of `.train_loss`, is a record rather than a
# leaf — the shape a large nested blob is. jsonl keeps it a json value;
# csv still has nowhere to put it but a quoted string, unchanged.
nested = _lines(export(format="jsonl", metrics="final_metrics"))
assert nested[0]["metric.final_metrics"] == {"train_loss": 0.5}
rows = list(csv.reader(export(metrics="final_metrics").text.splitlines()))
assert rows[1][-1] == '{"train_loss": 0.5}'
@pytest.fixture
def paged():
"""Three runs of one flow, a minute apart."""
made = datetime.now(UTC).replace(microsecond=0)
with Session(db_engine) as session:
for index in range(3):
session.add(
Run(
id=f"page-{index}",
flow_id="paged-study",
status="ok",
created_at=made + timedelta(minutes=index),
)
)
session.commit()
yield made
with Session(db_engine) as session:
for index in range(3):
run = session.get(Run, f"page-{index}")
if run is not None:
session.delete(run)
session.commit()
def test_runs_page_by_when_they_were_created(client, superuser_token_headers, paged):
"""`before` is the cursor: the last row's own timestamp reads the next page."""
def listed(**params):
answer = client.get(
f"{settings.API_V1_STR}/runs",
params={"flow": "paged-study", **params},
headers=superuser_token_headers,
)
assert answer.status_code == 200
return [row["id"] for row in answer.json()]
assert listed() == ["page-2", "page-1", "page-0"]
assert listed(before=(paged + timedelta(minutes=2)).isoformat()) == [
"page-1",
"page-0",
]
assert listed(since=(paged + timedelta(minutes=1)).isoformat()) == [
"page-2",
"page-1",
]
@pytest.mark.usefixtures("paged")
def test_a_truncated_export_says_how_many_runs_it_left_out(
client, superuser_token_headers, monkeypatch
):
"""The cap is in one header, the selection's size in another, and the SDK
turns the two into a warning naming what is missing."""
from fluksio.sdk.client import Client
monkeypatch.setattr("fluksio.api.routes.runs.EXPORT_CAP", 2)
answer = client.get(
f"{settings.API_V1_STR}/runs/export/runs",
params={"flow": "paged-study", "format": "jsonl"},
headers=superuser_token_headers,
)
assert answer.headers["X-Truncated"] == "2"
assert answer.headers["X-Total-Count"] == "3"
assert [row["id"] for row in _lines(answer)] == ["page-2", "page-1"]
sdk = Client(http=client, token=superuser_token_headers["Authorization"][7:])
with pytest.warns(UserWarning, match="1 older runs"):
assert len(sdk.export_runs(flow="paged-study")) == 2
Reference in New Issue View Git Blame Copy Permalink