Flows and dashboards are created by POST under an issued or superuser-pinned id; PUT only updates; deleted flows leave a tombstone with their key; the controller stores by id and runs by key, hooks live under /hooks/<flow id>; a key with state or queued work behind it is refused until cleared
This commit is contained in:
@@ -8,6 +8,7 @@ is why the migration is driven through crashes, refusals and stores without git.
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -18,17 +19,23 @@ from fluksio.flow.dashboards import (
|
||||
DASHBOARD_DIR,
|
||||
DashboardDef,
|
||||
DashboardExists,
|
||||
DashboardNotFound,
|
||||
DashboardStore,
|
||||
default_dashboard,
|
||||
results_dashboard,
|
||||
)
|
||||
from fluksio.flow.identity import COMMIT_MESSAGE, IdsRefused, assign_ids
|
||||
from fluksio.flow.schemas import FlowDef
|
||||
from fluksio.flow.schemas import FlowCreate, FlowDef
|
||||
from fluksio.flow.store import (
|
||||
ISSUED_ID,
|
||||
FlowExists,
|
||||
FlowNotFound,
|
||||
FlowStore,
|
||||
IdentityChange,
|
||||
IdTaken,
|
||||
RecordedKey,
|
||||
ReservedKey,
|
||||
StaleVersion,
|
||||
new_id,
|
||||
)
|
||||
|
||||
@@ -37,23 +44,51 @@ from fluksio.flow.store import (
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Graves:
|
||||
"""Tombstones and history, as `fluksio.flow.tombstones.Tombstones` keeps
|
||||
them in the database, which store tests have none of."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.stones: dict[str, str] = {}
|
||||
#: Ids that history rows name.
|
||||
self.history: set[str] = set()
|
||||
|
||||
def bury(self, flow_id: str, name: str) -> None:
|
||||
self.stones[flow_id] = name
|
||||
|
||||
def lift(self, flow_id: str) -> None:
|
||||
self.stones.pop(flow_id, None)
|
||||
|
||||
def recorded(self, flow_id: str) -> str | None:
|
||||
if flow_id in self.stones:
|
||||
return self.stones[flow_id]
|
||||
return flow_id if flow_id in self.history else None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> FlowStore:
|
||||
return FlowStore(tmp_path / "flows")
|
||||
def graves() -> Graves:
|
||||
return Graves()
|
||||
|
||||
|
||||
def test_a_flow_is_given_its_key_as_id_and_keeps_both(store: FlowStore):
|
||||
created = store.write_draft(FlowDef(name="heating"), 0)
|
||||
assert (created.id, created.name) == ("heating", "heating")
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path, graves: Graves) -> FlowStore:
|
||||
return FlowStore(tmp_path / "flows", tombstones=graves) # type: ignore[arg-type]
|
||||
|
||||
published = store.publish_flow("heating", created.version)
|
||||
|
||||
def test_a_new_flow_gets_an_issued_id_and_keeps_it(store: FlowStore):
|
||||
created = store.create(FlowDef(name="heating"))
|
||||
flow_id = created.id
|
||||
assert ISSUED_ID.fullmatch(flow_id) and flow_id.startswith("f_")
|
||||
assert created.name == "heating" and created.version == 1
|
||||
assert store.key_index() == {"heating": flow_id}
|
||||
|
||||
published = store.publish_flow(flow_id, created.version)
|
||||
store.write_draft(published.model_copy(update={"title": "Warm"}), 1)
|
||||
discarded = store.discard_draft("heating")
|
||||
discarded = store.discard_draft(flow_id)
|
||||
|
||||
assert json.loads((store.root / "heating" / "flow.json").read_text())["id"] == (
|
||||
"heating"
|
||||
)
|
||||
assert (discarded.id, discarded.name) == ("heating", "heating")
|
||||
on_disk = json.loads((store.root / flow_id / "flow.json").read_text())
|
||||
assert (on_disk["id"], on_disk["name"]) == (flow_id, "heating")
|
||||
assert (discarded.id, discarded.name) == (flow_id, "heating")
|
||||
|
||||
|
||||
def test_a_document_from_before_ids_reads_with_its_directory_as_id(store: FlowStore):
|
||||
@@ -70,9 +105,13 @@ def test_a_document_from_before_ids_reads_with_its_directory_as_id(store: FlowSt
|
||||
assert store.write_draft(FlowDef(name="heating", title="Warm"), 1).version == 2
|
||||
|
||||
|
||||
def test_a_save_cannot_change_an_id_or_a_key(store: FlowStore):
|
||||
store.write_flow(FlowDef(id="x_heating", name="heating"))
|
||||
def test_a_save_neither_creates_nor_changes_an_id_or_a_key(store: FlowStore):
|
||||
with pytest.raises(FlowNotFound):
|
||||
store.write_draft(FlowDef(name="nobody"), 0)
|
||||
with pytest.raises(DashboardNotFound):
|
||||
DashboardStore(store).write_draft(default_dashboard("nobody"), 0)
|
||||
|
||||
store.write_flow(FlowDef(id="x_heating", name="heating"))
|
||||
with pytest.raises(IdentityChange):
|
||||
store.write_draft(FlowDef(id="x_heating", name="warmth"), 1)
|
||||
with pytest.raises(IdentityChange):
|
||||
@@ -87,56 +126,218 @@ def test_a_save_cannot_change_an_id_or_a_key(store: FlowStore):
|
||||
|
||||
def test_a_key_belongs_to_one_flow_drafts_included(store: FlowStore):
|
||||
"""A draft nobody published holds its key, and a delete lets it go."""
|
||||
store.write_draft(FlowDef(id="x_heating", name="heating"), 0)
|
||||
assert store.key_index() == {"heating": "x_heating"}
|
||||
first = store.create(FlowDef(name="heating"))
|
||||
assert store.key_index() == {"heating": first.id}
|
||||
|
||||
with pytest.raises(FlowExists):
|
||||
store.write_draft(FlowDef(name="heating"), 0)
|
||||
store.create(FlowDef(name="heating"))
|
||||
with pytest.raises(FlowExists):
|
||||
store.create(FlowDef(name="heating"), "x_heating")
|
||||
with pytest.raises(FlowExists):
|
||||
store.write_flow(FlowDef(id="y_heating", name="heating"))
|
||||
|
||||
store.delete_flow("x_heating")
|
||||
assert store.write_draft(FlowDef(name="heating"), 0).id == "heating"
|
||||
assert store.key_index() == {"heating": "heating"}
|
||||
store.delete_flow(first.id)
|
||||
second = store.create(FlowDef(name="heating"))
|
||||
assert second.id != first.id
|
||||
assert store.key_index() == {"heating": second.id}
|
||||
|
||||
|
||||
def test_a_key_belongs_to_one_dashboard_drafts_included(store: FlowStore):
|
||||
dashboards = DashboardStore(store)
|
||||
dashboards.write_draft(DashboardDef(id="x_home", name="home"), 0)
|
||||
first = dashboards.create(default_dashboard("home"))
|
||||
assert ISSUED_ID.fullmatch(first.id) and first.id.startswith("d_")
|
||||
# Its own id space: a flow may share the key.
|
||||
store.write_draft(FlowDef(name="home"), 0)
|
||||
store.create(FlowDef(name="home"))
|
||||
|
||||
with pytest.raises(DashboardExists):
|
||||
dashboards.write_draft(default_dashboard("home"), 0)
|
||||
dashboards.create(default_dashboard("home"))
|
||||
with pytest.raises(DashboardExists):
|
||||
dashboards.write(DashboardDef(id="y_home", name="home"))
|
||||
|
||||
dashboards.delete("x_home")
|
||||
assert dashboards.write(default_dashboard("home")).id == "home"
|
||||
dashboards.delete(first.id)
|
||||
assert dashboards.create(default_dashboard("home"), "home").id == "home"
|
||||
|
||||
|
||||
def test_a_copy_takes_a_key_nobody_holds(store: FlowStore):
|
||||
def test_a_pin_is_an_id_nothing_holds(store: FlowStore):
|
||||
store.create(FlowDef(name="heating"), "x_heating")
|
||||
with pytest.raises(IdTaken):
|
||||
store.create(FlowDef(name="cooling"), "x_heating")
|
||||
# Another flow's key would make naming either ambiguous.
|
||||
with pytest.raises(IdTaken):
|
||||
store.create(FlowDef(name="cooling"), "heating")
|
||||
# And a directory left behind would hand the new flow what is in it.
|
||||
(store.root / "leftover" / "nodes").mkdir(parents=True)
|
||||
with pytest.raises(IdTaken):
|
||||
store.create(FlowDef(name="leftover"), "leftover")
|
||||
dashboards = DashboardStore(store)
|
||||
dashboards.create(default_dashboard("home"), "x_home")
|
||||
with pytest.raises(IdTaken):
|
||||
dashboards.create(default_dashboard("house"), "x_home")
|
||||
|
||||
|
||||
def test_an_issued_id_avoids_whatever_history_or_a_directory_holds(
|
||||
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
candidates = iter(["f_aaaaaaaa", "f_bbbbbbbb", "f_cccccccc", "f_dddddddd"])
|
||||
monkeypatch.setattr(
|
||||
"fluksio.flow.store.new_id",
|
||||
lambda kind, taken: next(c for c in candidates if not taken(c)),
|
||||
)
|
||||
graves.bury("f_aaaaaaaa", "gone")
|
||||
graves.history.add("f_bbbbbbbb")
|
||||
(store.root / "f_cccccccc").mkdir()
|
||||
|
||||
assert store.create(FlowDef(name="heating")).id == "f_dddddddd"
|
||||
|
||||
|
||||
def test_a_deleted_flow_is_buried_before_its_files_go(
|
||||
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
flow_id = store.create(FlowDef(name="heating")).id
|
||||
|
||||
def crash(path: Path) -> None:
|
||||
raise OSError("disk gone")
|
||||
|
||||
monkeypatch.setattr("fluksio.flow.store.shutil.rmtree", crash)
|
||||
with pytest.raises(OSError):
|
||||
store.delete_flow(flow_id)
|
||||
# The name is safe, and the live document still wins over it.
|
||||
assert graves.stones == {flow_id: "heating"}
|
||||
assert store.read_flow(flow_id, draft=True).name == "heating"
|
||||
|
||||
monkeypatch.undo()
|
||||
assert store.delete_flow(flow_id) == "heating"
|
||||
assert graves.stones == {flow_id: "heating"}
|
||||
assert not store.exists(flow_id)
|
||||
|
||||
|
||||
def test_a_delete_that_stops_after_the_files_has_kept_the_name(
|
||||
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
flow_id = store.create(FlowDef(name="heating")).id
|
||||
|
||||
def killed(message: str) -> None:
|
||||
raise OSError("killed before the commit")
|
||||
|
||||
monkeypatch.setattr(store, "_commit", killed)
|
||||
with pytest.raises(OSError):
|
||||
store.delete_flow(flow_id)
|
||||
monkeypatch.undo()
|
||||
|
||||
assert not store.exists(flow_id)
|
||||
assert graves.stones == {flow_id: "heating"}
|
||||
with pytest.raises(FlowNotFound):
|
||||
store.delete_flow(flow_id)
|
||||
|
||||
|
||||
def test_a_pinned_restore_takes_up_its_history_under_its_own_key(
|
||||
store: FlowStore, graves: Graves
|
||||
):
|
||||
flow_id = store.create(FlowDef(name="heating")).id
|
||||
store.delete_flow(flow_id)
|
||||
|
||||
with pytest.raises(RecordedKey):
|
||||
store.create(FlowDef(name="warmth"), flow_id)
|
||||
assert graves.stones == {flow_id: "heating"}
|
||||
|
||||
restored = store.create(FlowDef(name="heating"), flow_id)
|
||||
assert restored.id == flow_id and graves.stones == {}
|
||||
|
||||
# History from before tombstones is filed under an id that was its key.
|
||||
graves.history.add("legacy")
|
||||
with pytest.raises(RecordedKey):
|
||||
store.create(FlowDef(name="other"), "legacy")
|
||||
assert store.create(FlowDef(name="legacy"), "legacy").id == "legacy"
|
||||
|
||||
|
||||
def test_a_key_reused_after_a_delete_starts_a_new_history(
|
||||
store: FlowStore, graves: Graves
|
||||
):
|
||||
old = store.create(FlowDef(name="heating")).id
|
||||
store.delete_flow(old)
|
||||
|
||||
new = store.create(FlowDef(name="heating")).id
|
||||
|
||||
assert new != old
|
||||
assert graves.stones == {old: "heating"}
|
||||
|
||||
|
||||
def test_two_creates_of_one_key_make_one_flow(store: FlowStore):
|
||||
def create(_: int) -> str:
|
||||
try:
|
||||
return store.create(FlowDef(name="heating")).id
|
||||
except FlowExists:
|
||||
return ""
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
made = [flow_id for flow_id in pool.map(create, range(8)) if flow_id]
|
||||
|
||||
assert len(made) == 1
|
||||
assert store.list_flows() == made
|
||||
|
||||
|
||||
def test_two_results_dashboards_of_one_flow_make_one(store: FlowStore):
|
||||
dashboards = DashboardStore(store)
|
||||
flow = FlowDef(id="x_lab", name="lab", mode="batch")
|
||||
|
||||
def generate(_: int) -> str:
|
||||
try:
|
||||
return dashboards.create(results_dashboard(flow), "", True).id
|
||||
except DashboardExists:
|
||||
return ""
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
made = [board for board in pool.map(generate, range(4)) if board]
|
||||
|
||||
assert len(made) == 1
|
||||
assert dashboards.read(made[0]).name == "lab_results"
|
||||
|
||||
|
||||
def test_a_discard_leaves_no_old_draft_version_to_match(store: FlowStore):
|
||||
flow_id = store.create(FlowDef(name="heating")).id
|
||||
published = store.publish_flow(flow_id, 1)
|
||||
held = store.write_draft(published.model_copy(update={"title": "A"}), 1)
|
||||
store.write_draft(held.model_copy(update={"title": "B"}), held.version)
|
||||
|
||||
restored = store.discard_draft(flow_id)
|
||||
|
||||
# Above every draft version handed out, so an editor holding one is stale.
|
||||
assert restored.version == 4 and restored.name == "heating"
|
||||
with pytest.raises(StaleVersion):
|
||||
store.write_draft(held.model_copy(update={"title": "late"}), held.version)
|
||||
assert store.write_draft(restored.model_copy(update={"title": "C"}), 4).version == 5
|
||||
|
||||
dashboards = DashboardStore(store)
|
||||
board = dashboards.create(default_dashboard("home"))
|
||||
dashboards.publish(board.id, 1)
|
||||
dashboards.write_draft(dashboards.read(board.id).model_copy(), 1)
|
||||
assert dashboards.discard_draft(board.id).version == 3
|
||||
# A direct write goes above a draft too.
|
||||
dashboards.write_draft(dashboards.read(board.id).model_copy(), 3)
|
||||
assert dashboards.write(dashboards.read(board.id)).version == 5
|
||||
|
||||
|
||||
def test_a_copy_gets_an_id_of_its_own_and_a_key_nobody_holds(store: FlowStore):
|
||||
store.write_flow(FlowDef(name="heating"))
|
||||
store.write_draft(FlowDef(id="x_other", name="heating_copy"), 0)
|
||||
store.create(FlowDef(name="heating_copy"))
|
||||
dashboards = DashboardStore(store)
|
||||
dashboards.write(default_dashboard("home"))
|
||||
dashboards.write_draft(DashboardDef(id="x_other", name="home_copy"), 0)
|
||||
dashboards.create(default_dashboard("home_copy"))
|
||||
|
||||
copy = store.duplicate_flow("heating")
|
||||
copy = store.duplicate_flow("heating", busy=lambda key: key == "heating_copy_2")
|
||||
board = dashboards.duplicate("home")
|
||||
|
||||
assert (copy.id, copy.name) == ("heating_copy_2", "heating_copy_2")
|
||||
assert (board.id, board.name) == ("home_copy_2", "home_copy_2")
|
||||
# `_copy_2` still had work behind it, so the copy went past it.
|
||||
assert copy.name == "heating_copy_3" and ISSUED_ID.fullmatch(copy.id)
|
||||
assert board.name == "home_copy_2" and ISSUED_ID.fullmatch(board.id)
|
||||
|
||||
|
||||
def test_a_new_key_shaped_like_an_issued_id_is_refused(store: FlowStore):
|
||||
with pytest.raises(ReservedKey):
|
||||
store.write_draft(FlowDef(name="f_k3q7m2xa"), 0)
|
||||
store.create(FlowDef(name="f_k3q7m2xa"))
|
||||
with pytest.raises(ReservedKey):
|
||||
DashboardStore(store).write_draft(default_dashboard("d_k3q7m2xa"), 0)
|
||||
assert store.write_draft(FlowDef(name="f_k3q7m2xa_lab"), 0).name == (
|
||||
"f_k3q7m2xa_lab"
|
||||
)
|
||||
DashboardStore(store).create(default_dashboard("d_k3q7m2xa"))
|
||||
assert store.create(FlowDef(name="f_k3q7m2xa_lab")).name == "f_k3q7m2xa_lab"
|
||||
|
||||
|
||||
def test_an_existing_key_shaped_like_an_issued_id_stays_saveable(store: FlowStore):
|
||||
@@ -515,3 +716,10 @@ def test_ls_lists_ids_keys_and_kinds(tmp_path: Path, capsys):
|
||||
assert rows[0] == ["ID", "KEY", "KIND"]
|
||||
assert ["home", "home", "dashboard"] in rows and ["home", "home", "flow"] in rows
|
||||
assert ["warmth", "radiators", "flow"] in rows
|
||||
|
||||
|
||||
def test_what_a_create_takes_is_a_document_but_for_its_version():
|
||||
"""`FlowCreate` lists the fields again; one added to `FlowDef` belongs there."""
|
||||
assert set(FlowCreate.model_fields) == set(FlowDef.model_fields) - {"version"}
|
||||
with pytest.raises(ValueError):
|
||||
FlowCreate(name="Not A Key")
|
||||
|
||||
Reference in New Issue
Block a user