Flows and dashboards are created by POST under an issued or superuser-pinned id; PUT only updates; deleted flows leave a tombstone with their key; the controller stores by id and runs by key, hooks live under /hooks/<flow id>; a key with state or queued work behind it is refused until cleared

This commit is contained in:
2026-11-02 17:41:23 +01:00
parent fae85f5ecb
commit 08a6376836
63 changed files with 2832 additions and 936 deletions
+244 -36
View File
@@ -8,6 +8,7 @@ is why the migration is driven through crashes, refusals and stores without git.
import json
import logging
import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import pytest
@@ -18,17 +19,23 @@ from fluksio.flow.dashboards import (
DASHBOARD_DIR,
DashboardDef,
DashboardExists,
DashboardNotFound,
DashboardStore,
default_dashboard,
results_dashboard,
)
from fluksio.flow.identity import COMMIT_MESSAGE, IdsRefused, assign_ids
from fluksio.flow.schemas import FlowDef
from fluksio.flow.schemas import FlowCreate, FlowDef
from fluksio.flow.store import (
ISSUED_ID,
FlowExists,
FlowNotFound,
FlowStore,
IdentityChange,
IdTaken,
RecordedKey,
ReservedKey,
StaleVersion,
new_id,
)
@@ -37,23 +44,51 @@ from fluksio.flow.store import (
# -----------------------------------------------------------------------------
class Graves:
"""Tombstones and history, as `fluksio.flow.tombstones.Tombstones` keeps
them in the database, which store tests have none of."""
def __init__(self) -> None:
self.stones: dict[str, str] = {}
#: Ids that history rows name.
self.history: set[str] = set()
def bury(self, flow_id: str, name: str) -> None:
self.stones[flow_id] = name
def lift(self, flow_id: str) -> None:
self.stones.pop(flow_id, None)
def recorded(self, flow_id: str) -> str | None:
if flow_id in self.stones:
return self.stones[flow_id]
return flow_id if flow_id in self.history else None
@pytest.fixture
def store(tmp_path: Path) -> FlowStore:
return FlowStore(tmp_path / "flows")
def graves() -> Graves:
return Graves()
def test_a_flow_is_given_its_key_as_id_and_keeps_both(store: FlowStore):
created = store.write_draft(FlowDef(name="heating"), 0)
assert (created.id, created.name) == ("heating", "heating")
@pytest.fixture
def store(tmp_path: Path, graves: Graves) -> FlowStore:
return FlowStore(tmp_path / "flows", tombstones=graves) # type: ignore[arg-type]
published = store.publish_flow("heating", created.version)
def test_a_new_flow_gets_an_issued_id_and_keeps_it(store: FlowStore):
created = store.create(FlowDef(name="heating"))
flow_id = created.id
assert ISSUED_ID.fullmatch(flow_id) and flow_id.startswith("f_")
assert created.name == "heating" and created.version == 1
assert store.key_index() == {"heating": flow_id}
published = store.publish_flow(flow_id, created.version)
store.write_draft(published.model_copy(update={"title": "Warm"}), 1)
discarded = store.discard_draft("heating")
discarded = store.discard_draft(flow_id)
assert json.loads((store.root / "heating" / "flow.json").read_text())["id"] == (
"heating"
)
assert (discarded.id, discarded.name) == ("heating", "heating")
on_disk = json.loads((store.root / flow_id / "flow.json").read_text())
assert (on_disk["id"], on_disk["name"]) == (flow_id, "heating")
assert (discarded.id, discarded.name) == (flow_id, "heating")
def test_a_document_from_before_ids_reads_with_its_directory_as_id(store: FlowStore):
@@ -70,9 +105,13 @@ def test_a_document_from_before_ids_reads_with_its_directory_as_id(store: FlowSt
assert store.write_draft(FlowDef(name="heating", title="Warm"), 1).version == 2
def test_a_save_cannot_change_an_id_or_a_key(store: FlowStore):
store.write_flow(FlowDef(id="x_heating", name="heating"))
def test_a_save_neither_creates_nor_changes_an_id_or_a_key(store: FlowStore):
with pytest.raises(FlowNotFound):
store.write_draft(FlowDef(name="nobody"), 0)
with pytest.raises(DashboardNotFound):
DashboardStore(store).write_draft(default_dashboard("nobody"), 0)
store.write_flow(FlowDef(id="x_heating", name="heating"))
with pytest.raises(IdentityChange):
store.write_draft(FlowDef(id="x_heating", name="warmth"), 1)
with pytest.raises(IdentityChange):
@@ -87,56 +126,218 @@ def test_a_save_cannot_change_an_id_or_a_key(store: FlowStore):
def test_a_key_belongs_to_one_flow_drafts_included(store: FlowStore):
"""A draft nobody published holds its key, and a delete lets it go."""
store.write_draft(FlowDef(id="x_heating", name="heating"), 0)
assert store.key_index() == {"heating": "x_heating"}
first = store.create(FlowDef(name="heating"))
assert store.key_index() == {"heating": first.id}
with pytest.raises(FlowExists):
store.write_draft(FlowDef(name="heating"), 0)
store.create(FlowDef(name="heating"))
with pytest.raises(FlowExists):
store.create(FlowDef(name="heating"), "x_heating")
with pytest.raises(FlowExists):
store.write_flow(FlowDef(id="y_heating", name="heating"))
store.delete_flow("x_heating")
assert store.write_draft(FlowDef(name="heating"), 0).id == "heating"
assert store.key_index() == {"heating": "heating"}
store.delete_flow(first.id)
second = store.create(FlowDef(name="heating"))
assert second.id != first.id
assert store.key_index() == {"heating": second.id}
def test_a_key_belongs_to_one_dashboard_drafts_included(store: FlowStore):
dashboards = DashboardStore(store)
dashboards.write_draft(DashboardDef(id="x_home", name="home"), 0)
first = dashboards.create(default_dashboard("home"))
assert ISSUED_ID.fullmatch(first.id) and first.id.startswith("d_")
# Its own id space: a flow may share the key.
store.write_draft(FlowDef(name="home"), 0)
store.create(FlowDef(name="home"))
with pytest.raises(DashboardExists):
dashboards.write_draft(default_dashboard("home"), 0)
dashboards.create(default_dashboard("home"))
with pytest.raises(DashboardExists):
dashboards.write(DashboardDef(id="y_home", name="home"))
dashboards.delete("x_home")
assert dashboards.write(default_dashboard("home")).id == "home"
dashboards.delete(first.id)
assert dashboards.create(default_dashboard("home"), "home").id == "home"
def test_a_copy_takes_a_key_nobody_holds(store: FlowStore):
def test_a_pin_is_an_id_nothing_holds(store: FlowStore):
store.create(FlowDef(name="heating"), "x_heating")
with pytest.raises(IdTaken):
store.create(FlowDef(name="cooling"), "x_heating")
# Another flow's key would make naming either ambiguous.
with pytest.raises(IdTaken):
store.create(FlowDef(name="cooling"), "heating")
# And a directory left behind would hand the new flow what is in it.
(store.root / "leftover" / "nodes").mkdir(parents=True)
with pytest.raises(IdTaken):
store.create(FlowDef(name="leftover"), "leftover")
dashboards = DashboardStore(store)
dashboards.create(default_dashboard("home"), "x_home")
with pytest.raises(IdTaken):
dashboards.create(default_dashboard("house"), "x_home")
def test_an_issued_id_avoids_whatever_history_or_a_directory_holds(
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
):
candidates = iter(["f_aaaaaaaa", "f_bbbbbbbb", "f_cccccccc", "f_dddddddd"])
monkeypatch.setattr(
"fluksio.flow.store.new_id",
lambda kind, taken: next(c for c in candidates if not taken(c)),
)
graves.bury("f_aaaaaaaa", "gone")
graves.history.add("f_bbbbbbbb")
(store.root / "f_cccccccc").mkdir()
assert store.create(FlowDef(name="heating")).id == "f_dddddddd"
def test_a_deleted_flow_is_buried_before_its_files_go(
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
):
flow_id = store.create(FlowDef(name="heating")).id
def crash(path: Path) -> None:
raise OSError("disk gone")
monkeypatch.setattr("fluksio.flow.store.shutil.rmtree", crash)
with pytest.raises(OSError):
store.delete_flow(flow_id)
# The name is safe, and the live document still wins over it.
assert graves.stones == {flow_id: "heating"}
assert store.read_flow(flow_id, draft=True).name == "heating"
monkeypatch.undo()
assert store.delete_flow(flow_id) == "heating"
assert graves.stones == {flow_id: "heating"}
assert not store.exists(flow_id)
def test_a_delete_that_stops_after_the_files_has_kept_the_name(
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
):
flow_id = store.create(FlowDef(name="heating")).id
def killed(message: str) -> None:
raise OSError("killed before the commit")
monkeypatch.setattr(store, "_commit", killed)
with pytest.raises(OSError):
store.delete_flow(flow_id)
monkeypatch.undo()
assert not store.exists(flow_id)
assert graves.stones == {flow_id: "heating"}
with pytest.raises(FlowNotFound):
store.delete_flow(flow_id)
def test_a_pinned_restore_takes_up_its_history_under_its_own_key(
store: FlowStore, graves: Graves
):
flow_id = store.create(FlowDef(name="heating")).id
store.delete_flow(flow_id)
with pytest.raises(RecordedKey):
store.create(FlowDef(name="warmth"), flow_id)
assert graves.stones == {flow_id: "heating"}
restored = store.create(FlowDef(name="heating"), flow_id)
assert restored.id == flow_id and graves.stones == {}
# History from before tombstones is filed under an id that was its key.
graves.history.add("legacy")
with pytest.raises(RecordedKey):
store.create(FlowDef(name="other"), "legacy")
assert store.create(FlowDef(name="legacy"), "legacy").id == "legacy"
def test_a_key_reused_after_a_delete_starts_a_new_history(
store: FlowStore, graves: Graves
):
old = store.create(FlowDef(name="heating")).id
store.delete_flow(old)
new = store.create(FlowDef(name="heating")).id
assert new != old
assert graves.stones == {old: "heating"}
def test_two_creates_of_one_key_make_one_flow(store: FlowStore):
def create(_: int) -> str:
try:
return store.create(FlowDef(name="heating")).id
except FlowExists:
return ""
with ThreadPoolExecutor(max_workers=8) as pool:
made = [flow_id for flow_id in pool.map(create, range(8)) if flow_id]
assert len(made) == 1
assert store.list_flows() == made
def test_two_results_dashboards_of_one_flow_make_one(store: FlowStore):
dashboards = DashboardStore(store)
flow = FlowDef(id="x_lab", name="lab", mode="batch")
def generate(_: int) -> str:
try:
return dashboards.create(results_dashboard(flow), "", True).id
except DashboardExists:
return ""
with ThreadPoolExecutor(max_workers=4) as pool:
made = [board for board in pool.map(generate, range(4)) if board]
assert len(made) == 1
assert dashboards.read(made[0]).name == "lab_results"
def test_a_discard_leaves_no_old_draft_version_to_match(store: FlowStore):
flow_id = store.create(FlowDef(name="heating")).id
published = store.publish_flow(flow_id, 1)
held = store.write_draft(published.model_copy(update={"title": "A"}), 1)
store.write_draft(held.model_copy(update={"title": "B"}), held.version)
restored = store.discard_draft(flow_id)
# Above every draft version handed out, so an editor holding one is stale.
assert restored.version == 4 and restored.name == "heating"
with pytest.raises(StaleVersion):
store.write_draft(held.model_copy(update={"title": "late"}), held.version)
assert store.write_draft(restored.model_copy(update={"title": "C"}), 4).version == 5
dashboards = DashboardStore(store)
board = dashboards.create(default_dashboard("home"))
dashboards.publish(board.id, 1)
dashboards.write_draft(dashboards.read(board.id).model_copy(), 1)
assert dashboards.discard_draft(board.id).version == 3
# A direct write goes above a draft too.
dashboards.write_draft(dashboards.read(board.id).model_copy(), 3)
assert dashboards.write(dashboards.read(board.id)).version == 5
def test_a_copy_gets_an_id_of_its_own_and_a_key_nobody_holds(store: FlowStore):
store.write_flow(FlowDef(name="heating"))
store.write_draft(FlowDef(id="x_other", name="heating_copy"), 0)
store.create(FlowDef(name="heating_copy"))
dashboards = DashboardStore(store)
dashboards.write(default_dashboard("home"))
dashboards.write_draft(DashboardDef(id="x_other", name="home_copy"), 0)
dashboards.create(default_dashboard("home_copy"))
copy = store.duplicate_flow("heating")
copy = store.duplicate_flow("heating", busy=lambda key: key == "heating_copy_2")
board = dashboards.duplicate("home")
assert (copy.id, copy.name) == ("heating_copy_2", "heating_copy_2")
assert (board.id, board.name) == ("home_copy_2", "home_copy_2")
# `_copy_2` still had work behind it, so the copy went past it.
assert copy.name == "heating_copy_3" and ISSUED_ID.fullmatch(copy.id)
assert board.name == "home_copy_2" and ISSUED_ID.fullmatch(board.id)
def test_a_new_key_shaped_like_an_issued_id_is_refused(store: FlowStore):
with pytest.raises(ReservedKey):
store.write_draft(FlowDef(name="f_k3q7m2xa"), 0)
store.create(FlowDef(name="f_k3q7m2xa"))
with pytest.raises(ReservedKey):
DashboardStore(store).write_draft(default_dashboard("d_k3q7m2xa"), 0)
assert store.write_draft(FlowDef(name="f_k3q7m2xa_lab"), 0).name == (
"f_k3q7m2xa_lab"
)
DashboardStore(store).create(default_dashboard("d_k3q7m2xa"))
assert store.create(FlowDef(name="f_k3q7m2xa_lab")).name == "f_k3q7m2xa_lab"
def test_an_existing_key_shaped_like_an_issued_id_stays_saveable(store: FlowStore):
@@ -515,3 +716,10 @@ def test_ls_lists_ids_keys_and_kinds(tmp_path: Path, capsys):
assert rows[0] == ["ID", "KEY", "KIND"]
assert ["home", "home", "dashboard"] in rows and ["home", "home", "flow"] in rows
assert ["warmth", "radiators", "flow"] in rows
def test_what_a_create_takes_is_a_document_but_for_its_version():
"""`FlowCreate` lists the fields again; one added to `FlowDef` belongs there."""
assert set(FlowCreate.model_fields) == set(FlowDef.model_fields) - {"version"}
with pytest.raises(ValueError):
FlowCreate(name="Not A Key")