History is kept by flow id: runs, cascades, rollups and events file the id in their flow column and nodes as <flow id>.<node>, the API answers flow plus flow_name and shows and takes nodes as <key>.<node>; events carry flow_id from the moment they are published; run queue items, the stage cache and cached curves keep ids and keys apart; endpoints, the brain graph, device bindings and search name the id to open

This commit is contained in:
2026-11-02 19:01:19 +01:00
parent e6027af8fe
commit d4a1b1db47
34 changed files with 1374 additions and 329 deletions
+51 -36
View File
@@ -45,7 +45,12 @@ def test_a_run_cache_finds_what_an_earlier_run_recorded(tmp_path):
with Session(db_engine) as session:
for run_id in ("cache-1", "cache-2", "cache-3"):
session.add(
Run(id=run_id, flow="study", status="ok", created_at=datetime.now(UTC))
Run(
id=run_id,
flow_id="study",
status="ok",
created_at=datetime.now(UTC),
)
)
session.add(
RunNode(
@@ -115,7 +120,7 @@ def test_a_row_keyed_before_the_current_shape_is_unkeyed():
("shape-old", CACHE_SHAPE_SINCE - timedelta(days=1)),
("shape-new", CACHE_SHAPE_SINCE + timedelta(seconds=1)),
):
session.add(Run(id=run_id, flow="study", status="ok", created_at=made))
session.add(Run(id=run_id, flow_id="study", status="ok", created_at=made))
session.add(
RunNode(
run_id=run_id,
@@ -174,7 +179,7 @@ def made_artifact():
session.add(
Run(
id=run_id,
flow="prepare",
flow_id="prepare",
status="ok",
result={"dataset": reference},
created_at=datetime.now(UTC),
@@ -288,7 +293,7 @@ def made_config():
session.add(
Run(
id=run_id,
flow="generate",
flow_id="generate",
status="ok",
result={"meta": meta, "label": "run-7"},
created_at=datetime.now(UTC),
@@ -335,7 +340,7 @@ def test_the_overview_counts_a_flow_the_list_page_would_not_reach(
session.add(
Run(
id=f"ov-{index}",
flow="overviewed",
flow_id="overviewed",
status="ok" if index else "running",
created_at=datetime(2026, 1, 1 + index, tzinfo=UTC),
)
@@ -358,7 +363,7 @@ def test_a_running_run_reports_how_long_it_has_been_going(
session.add(
Run(
id="in-flight",
flow="timed",
flow_id="timed",
status="running",
created_at=datetime.now(UTC),
started_at=datetime.now(UTC) - timedelta(seconds=30),
@@ -390,7 +395,7 @@ def test_a_run_records_which_caller_asked_for_it(
seen.update(kwargs)
return Run(
id="cause-1",
flow=name,
flow_id=name,
cause=str(kwargs["cause"]),
created_at=datetime.now(UTC),
)
@@ -431,7 +436,7 @@ def test_a_repeated_submit_returns_the_run_it_already_made():
session.add(
Run(
id="dedup-1",
flow="study",
flow_id="study",
status="running",
idempotency_key="key-abc",
created_at=datetime.now(UTC),
@@ -449,11 +454,15 @@ class _OneFlow:
def __init__(self, flow):
self.store = self
self._flow = flow
# As the store reads one from before ids: its key is its id.
self._flow = flow.model_copy(update={"id": flow.id or flow.name})
def read_flow(self, name, draft=False):
return self._flow
def key_index(self):
return {self._flow.name: self._flow.id}
def head(self):
return ""
@@ -738,7 +747,7 @@ def test_a_retry_is_a_new_run_that_names_the_one_it_repeats():
session.add(
Run(
id="abandoned-1",
flow="study",
flow_id="study",
status="abandoned",
params={"lr": 0.3},
group_id="sweep-9",
@@ -749,7 +758,7 @@ def test_a_retry_is_a_new_run_that_names_the_one_it_repeats():
session.add(
Run(
id="going-1",
flow="study",
flow_id="study",
status="running",
created_at=datetime.now(UTC),
)
@@ -796,7 +805,7 @@ def test_runs_left_queued_are_woken_by_the_next_engine():
session.add(
Run(
id="orphan-1",
flow="study",
flow_id="study",
status="queued",
created_at=datetime.now(UTC),
)
@@ -804,7 +813,7 @@ def test_runs_left_queued_are_woken_by_the_next_engine():
session.add(
Run(
id="finished-1",
flow="study",
flow_id="study",
status="ok",
created_at=datetime.now(UTC),
)
@@ -903,7 +912,7 @@ def test_a_key_nobody_used_submits_normally(
class Recorder:
def submit(self, name, **kwargs):
seen.update(kwargs)
return Run(id="keyed-1", flow=name, created_at=datetime.now(UTC))
return Run(id="keyed-1", flow_id=name, created_at=datetime.now(UTC))
monkeypatch.setattr(client.app.state, "run_service", Recorder())
answer = client.post(
@@ -940,7 +949,7 @@ def deletable_run():
run_id = "del-1"
with Session(db_engine) as session:
session.add(
Run(id=run_id, flow="deleted", status="ok", created_at=datetime.now(UTC))
Run(id=run_id, flow_id="deleted", status="ok", created_at=datetime.now(UTC))
)
session.add(RunNode(run_id=run_id, node="deleted.a", status="ok"))
session.add(RunMetric(run_id=run_id, name="deleted.loss", step=0, value=1.0))
@@ -985,7 +994,11 @@ def test_a_node_finishing_after_its_run_was_deleted_is_dropped_quietly(caplog):
"""The constraint refuses its row, and that is a no-op rather than an error."""
service = RunService(controller=_OneFlow(FlowDef(name="gone")), queue=_Collect())
with caplog.at_level(logging.INFO, logger="fluksio.flow.runs"):
service._record_node("del-gone", NodeOutcome(node="gone.a", ok=True))
service._record_node(
Run(id="del-gone", flow_id="gone"),
"gone",
NodeOutcome(node="gone.a", ok=True),
)
with Session(db_engine) as session:
assert session.get(RunNode, {"run_id": "del-gone", "node": "gone.a"}) is None
@@ -1008,7 +1021,7 @@ def test_a_running_run_is_refused_rather_than_raced(client, superuser_token_head
session.add(
Run(
id=run_id,
flow="deleted",
flow_id="deleted",
status="running",
created_at=datetime.now(UTC),
)
@@ -1041,11 +1054,13 @@ def test_deleting_a_flows_runs_clears_that_flow_only(client, superuser_token_hea
made = datetime.now(UTC)
url = f"{settings.API_V1_STR}/runs"
with Session(db_engine) as session:
session.add(Run(id="bulk-1", flow="swept", status="ok", created_at=made))
session.add(Run(id="bulk-1", flow_id="swept", status="ok", created_at=made))
session.add(RunNode(run_id="bulk-1", node="swept.a", status="ok"))
session.add(RunMetric(run_id="bulk-1", name="swept.loss", step=0, value=1.0))
session.add(Run(id="bulk-2", flow="swept", status="running", created_at=made))
session.add(Run(id="keep-1", flow="other", status="ok", created_at=made))
session.add(
Run(id="bulk-2", flow_id="swept", status="running", created_at=made)
)
session.add(Run(id="keep-1", flow_id="other", status="ok", created_at=made))
session.commit()
busy = client.delete(url, params={"flow": "swept"}, headers=superuser_token_headers)
@@ -1063,7 +1078,7 @@ def test_deleting_a_flows_runs_clears_that_flow_only(client, superuser_token_hea
assert "2" in answer.json()["message"]
with Session(db_engine) as session:
assert session.exec(select(Run).where(col(Run.flow) == "swept")).all() == []
assert session.exec(select(Run).where(col(Run.flow_id) == "swept")).all() == []
for table in (RunNode, RunMetric):
assert (
session.exec(select(table).where(col(table.run_id) == "bulk-1")).all()
@@ -1079,10 +1094,10 @@ def test_history_also_clears_a_flows_cascades(client, superuser_token_headers):
made = datetime.now(UTC)
url = f"{settings.API_V1_STR}/runs"
with Session(db_engine) as session:
session.add(Run(id="hist-1", flow="gone", status="ok", created_at=made))
session.add(FlowRun(id="hist-1", flow="gone", source="run", started_at=made))
session.add(FlowRun(id="hist-2", flow="gone", started_at=made))
session.add(FlowRun(id="hist-3", flow="other", started_at=made))
session.add(Run(id="hist-1", flow_id="gone", status="ok", created_at=made))
session.add(FlowRun(id="hist-1", flow_id="gone", source="run", started_at=made))
session.add(FlowRun(id="hist-2", flow_id="gone", started_at=made))
session.add(FlowRun(id="hist-3", flow_id="other", started_at=made))
session.commit()
plain = client.delete(url, params={"flow": "gone"}, headers=superuser_token_headers)
@@ -1095,7 +1110,7 @@ def test_history_also_clears_a_flows_cascades(client, superuser_token_headers):
)
assert answer.status_code == 200
with Session(db_engine) as session:
left = session.exec(select(FlowRun).where(col(FlowRun.flow) == "gone")).all()
left = session.exec(select(FlowRun).where(col(FlowRun.flow_id) == "gone")).all()
assert left == []
session.delete(session.get(FlowRun, "hist-3"))
session.commit()
@@ -1114,8 +1129,8 @@ def reused_run():
"""A run of `quick` whose node was restored from a run of `train`."""
with Session(db_engine) as session:
made = datetime.now(UTC)
session.add(Run(id="src-1", flow="train", status="ok", created_at=made))
session.add(Run(id="reuse-1", flow="quick", status="ok", created_at=made))
session.add(Run(id="src-1", flow_id="train", status="ok", created_at=made))
session.add(Run(id="reuse-1", flow_id="quick", status="ok", created_at=made))
session.add(RunNode(run_id="src-1", node="train.fit", status="ok"))
session.add(
RunNode(
@@ -1197,7 +1212,7 @@ def test_a_node_answers_with_what_it_returned(client, superuser_token_headers):
run_id = new_run_id()
with Session(db_engine) as session:
session.add(
Run(id=run_id, flow="study", status="ok", created_at=datetime.now(UTC))
Run(id=run_id, flow_id="study", status="ok", created_at=datetime.now(UTC))
)
session.add(RunNode(run_id=run_id, node="fit", outputs='{"study.loss":0.5}'))
session.add(RunNode(run_id=run_id, node="plot", status="error"))
@@ -1233,7 +1248,7 @@ def plotted():
run_id = new_run_id()
with Session(db_engine) as session:
session.add(
Run(id=run_id, flow="study", status="ok", created_at=datetime.now(UTC))
Run(id=run_id, flow_id="study", status="ok", created_at=datetime.now(UTC))
)
for step, (loss, epoch) in enumerate([(1.0, 10.0), (0.5, 20.0), (0.25, 30.0)]):
session.add(_metric(run_id, "study.loss", step, loss, 100.0 + step * 5))
@@ -1263,7 +1278,7 @@ def test_the_metric_names_of_a_selection_are_exact(
session.add(
Run(
id=older,
flow="study",
flow_id="study",
status="ok",
created_at=datetime.now(UTC) - timedelta(hours=1),
)
@@ -1374,7 +1389,7 @@ def test_a_run_records_the_code_it_started_with_not_the_code_it_was_queued_with(
session.add(
Run(
id="stamp-1",
flow="study",
flow_id="study",
status="queued",
code_digest="at-submit",
created_at=datetime.now(UTC),
@@ -1395,7 +1410,7 @@ def test_a_tree_that_did_not_move_is_not_written_again():
session.add(
Run(
id="stamp-2",
flow="study",
flow_id="study",
status="queued",
code_digest="same",
created_at=datetime.now(UTC),
@@ -1429,7 +1444,7 @@ def exported():
session.add(
Run(
id=run_id,
flow="export-study",
flow_id="export-study",
status="ok",
params={
"lr": lr,
@@ -1531,7 +1546,7 @@ def test_an_exported_run_row_carries_every_recorded_input(
# csv is the default, and the columns are in the order the header names.
header = export().text.splitlines()[0]
assert header.startswith("id,flow,status,")
assert header.startswith("id,flow,flow_name,status,")
assert header.endswith(
"param.config.depth,param.config.model,param.epochs,param.lr,"
"metric.acc,metric.final_metrics.train_loss,"
@@ -1557,7 +1572,7 @@ def paged():
session.add(
Run(
id=f"page-{index}",
flow="paged-study",
flow_id="paged-study",
status="ok",
created_at=made + timedelta(minutes=index),
)