Flows and dashboards are created by POST under an issued or superuser-pinned id; PUT only updates; deleted flows leave a tombstone with their key; the controller stores by id and runs by key, hooks live under /hooks/<flow id>; a key with state or queued work behind it is refused until cleared
This commit is contained in:
@@ -19,7 +19,8 @@ from fluksio.flow.nodes import Node
|
||||
from fluksio.flow.pipeline import Pipeline
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||
from fluksio.flow.state import MemoryState
|
||||
from fluksio.flow.store import FlowStore, StaleVersion
|
||||
from fluksio.flow.store import ISSUED_ID, FlowStore, StaleVersion
|
||||
from tests.utils.stores import created
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -93,25 +94,25 @@ def test_discarding_leaves_what_is_published(store: DashboardStore):
|
||||
|
||||
def test_a_new_dashboard_is_a_draft_until_it_is_published(store: DashboardStore):
|
||||
"""Creating one does not put it on a wall; publishing is what does."""
|
||||
created = store.write_draft(default_dashboard("house"), 0)
|
||||
made = created(store, default_dashboard("house"))
|
||||
|
||||
assert not store.is_published("house")
|
||||
with pytest.raises(DashboardNotFound):
|
||||
store.read("house")
|
||||
assert store.read("house", draft=True).title == created.title
|
||||
assert store.read("house", draft=True).title == made.title
|
||||
# Listed all the same, so the editor can find what it just made.
|
||||
assert [(d.name, d.has_draft, d.version) for d in store.list()] == [
|
||||
("house", True, created.version)
|
||||
("house", True, made.version)
|
||||
]
|
||||
|
||||
store.publish("house", created.version)
|
||||
store.publish("house", made.version)
|
||||
|
||||
assert store.is_published("house")
|
||||
assert store.read("house").title == created.title
|
||||
assert store.read("house").title == made.title
|
||||
|
||||
|
||||
def test_an_unpublished_dashboard_can_be_deleted(store: DashboardStore):
|
||||
store.write_draft(default_dashboard("house"), 0)
|
||||
created(store, default_dashboard("house"))
|
||||
|
||||
store.delete("house")
|
||||
assert not store.exists("house")
|
||||
@@ -128,7 +129,8 @@ def test_a_duplicate_is_an_unpublished_copy_of_the_working_copy(
|
||||
copy = store.duplicate("house")
|
||||
|
||||
assert (copy.name, copy.title, copy.version) == ("house_copy", "Edited (copy)", 1)
|
||||
assert store.has_draft("house_copy") and not store.is_published("house_copy")
|
||||
assert ISSUED_ID.fullmatch(copy.id)
|
||||
assert store.has_draft(copy.id) and not store.is_published(copy.id)
|
||||
assert store.read("house", draft=True).version == edited.version
|
||||
assert store.duplicate("house").name == "house_copy_2"
|
||||
|
||||
|
||||
@@ -8,7 +8,8 @@ import pytest
|
||||
from fluksio.flow.controller import FlowController
|
||||
from fluksio.flow.messages import MessageSpec
|
||||
from fluksio.flow.schemas import FlowDef, FlowOrigin, NodeDef
|
||||
from fluksio.flow.store import FlowStore, StaleVersion
|
||||
from fluksio.flow.store import ISSUED_ID, FlowStore, StaleVersion
|
||||
from tests.utils.stores import created
|
||||
|
||||
SOURCE = "def process():\n return {}\n"
|
||||
EDITED = "def process():\n return {'temp': 1}\n"
|
||||
@@ -28,7 +29,7 @@ def a_flow(title: str = "") -> FlowDef:
|
||||
|
||||
|
||||
def test_a_draft_is_invisible_to_the_engine(store: FlowStore):
|
||||
store.write_draft(a_flow(), 0)
|
||||
created(store, a_flow())
|
||||
|
||||
assert store.has_draft("heating")
|
||||
assert store.list_flows() == ["heating"] # the editor sees it
|
||||
@@ -36,7 +37,7 @@ def test_a_draft_is_invisible_to_the_engine(store: FlowStore):
|
||||
|
||||
|
||||
def test_the_version_counter_advances_per_save(store: FlowStore):
|
||||
first = store.write_draft(a_flow(), 0)
|
||||
first = created(store, a_flow())
|
||||
assert first.version == 1
|
||||
|
||||
second = store.write_draft(a_flow(title="Warm"), first.version)
|
||||
@@ -44,14 +45,14 @@ def test_the_version_counter_advances_per_save(store: FlowStore):
|
||||
|
||||
|
||||
def test_resaving_identical_content_neither_bumps_nor_commits(store: FlowStore):
|
||||
stored = store.write_draft(a_flow(), 0)
|
||||
stored = created(store, a_flow())
|
||||
|
||||
again = store.write_draft(a_flow(), stored.version)
|
||||
assert again.version == stored.version
|
||||
|
||||
|
||||
def test_a_stale_write_is_refused(store: FlowStore):
|
||||
store.write_draft(a_flow(), 0)
|
||||
created(store, a_flow())
|
||||
store.write_draft(a_flow(title="Mine"), 1)
|
||||
|
||||
# A second client still holding version 1 must not overwrite version 2.
|
||||
@@ -61,7 +62,7 @@ def test_a_stale_write_is_refused(store: FlowStore):
|
||||
|
||||
|
||||
def test_publishing_promotes_the_draft_and_its_sources(store: FlowStore):
|
||||
stored = store.write_draft(a_flow(title="Warm"), 0)
|
||||
stored = created(store, a_flow(title="Warm"))
|
||||
store.write_node_source("heating", "sensor", EDITED, draft=True)
|
||||
|
||||
published = store.publish_flow("heating", stored.version)
|
||||
@@ -74,7 +75,7 @@ def test_publishing_promotes_the_draft_and_its_sources(store: FlowStore):
|
||||
|
||||
|
||||
def test_publishing_a_stale_draft_is_refused(store: FlowStore):
|
||||
store.write_draft(a_flow(), 0)
|
||||
created(store, a_flow())
|
||||
|
||||
with pytest.raises(StaleVersion):
|
||||
store.publish_flow("heating", 99)
|
||||
@@ -150,7 +151,7 @@ def test_a_worker_on_a_node_without_code_is_reported(store: FlowStore):
|
||||
|
||||
def test_a_node_being_written_in_the_editor_is_not_reported(store: FlowStore):
|
||||
"""A draft legitimately has no published body yet — that is what a draft is."""
|
||||
store.write_draft(a_flow(), 0)
|
||||
created(store, a_flow())
|
||||
controller = FlowController(store)
|
||||
|
||||
assert controller.preview("heating").issues == []
|
||||
@@ -175,12 +176,14 @@ def test_a_duplicate_is_a_local_draft_of_what_the_editor_shows(store: FlowStore)
|
||||
copy = store.duplicate_flow("heating")
|
||||
|
||||
assert (copy.name, copy.title, copy.version) == ("heating_copy", "Warm (copy)", 1)
|
||||
# An id of its own, which its key is not.
|
||||
assert ISSUED_ID.fullmatch(copy.id)
|
||||
assert copy.origin is None
|
||||
assert copy.nodes[1].requires[0].name == "heating_copy.temp"
|
||||
assert store.has_draft("heating_copy") and not store.is_published("heating_copy")
|
||||
assert store.read_node_source("heating_copy", "sensor", draft=True) == EDITED
|
||||
assert not (store.root / "heating_copy" / "runtime.json").exists()
|
||||
assert store.has_draft(copy.id) and not store.is_published(copy.id)
|
||||
assert store.read_node_source(copy.id, "sensor", draft=True) == EDITED
|
||||
assert not (store.root / copy.id / "runtime.json").exists()
|
||||
assert store.duplicate_flow("heating").name == "heating_copy_2"
|
||||
|
||||
store.publish_flow("heating_copy", copy.version)
|
||||
assert store.read_node_source("heating_copy", "sensor") == EDITED
|
||||
store.publish_flow(copy.id, copy.version)
|
||||
assert store.read_node_source(copy.id, "sensor") == EDITED
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
"""The engine runs a flow by its key and finds it by its id.
|
||||
|
||||
Every test runs twice, once with the id equal to the key the way a flow from
|
||||
before ids has it, and once with them apart. A mix-up of the two is invisible
|
||||
in the first run, which is why the second one exists.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from fluksio.flow.controller import HOOK_PREFIX, FlowController
|
||||
from fluksio.flow.messages import DType, MessageSpec
|
||||
from fluksio.flow.queue import MemoryWorkQueue, WorkItem
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||
from fluksio.flow.store import FlowStore
|
||||
|
||||
SENSOR = "def process():\n return {'reading': 21.5}\n"
|
||||
DOUBLE = "def process(reading):\n return {'doubled': reading * 2}\n"
|
||||
|
||||
|
||||
@pytest.fixture(params=["house", "x_house"], ids=["id=key", "id!=key"])
|
||||
def flow_id(request: pytest.FixtureRequest) -> str:
|
||||
return str(request.param)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path, flow_id: str) -> FlowStore:
|
||||
store = FlowStore(tmp_path / "flows")
|
||||
store.write_flow(
|
||||
FlowDef(
|
||||
id=flow_id,
|
||||
name="house",
|
||||
nodes=[
|
||||
NodeDef(
|
||||
id="sensor",
|
||||
provides=[MessageSpec(name="reading", dtype=DType.FLOAT)],
|
||||
),
|
||||
NodeDef(
|
||||
id="double",
|
||||
requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
|
||||
provides=[MessageSpec(name="doubled", dtype=DType.FLOAT)],
|
||||
),
|
||||
NodeDef(
|
||||
id="hook",
|
||||
type="http",
|
||||
params={"url": "/tick", "method": "POST"},
|
||||
provides=[MessageSpec(name="tick", dtype=DType.FLOAT)],
|
||||
),
|
||||
],
|
||||
)
|
||||
)
|
||||
store.write_node_source(flow_id, "sensor", SENSOR)
|
||||
store.write_node_source(flow_id, "double", DOUBLE)
|
||||
return store
|
||||
|
||||
|
||||
def test_a_flow_runs_under_its_key_from_the_code_its_id_holds(
|
||||
store: FlowStore, flow_id: str
|
||||
):
|
||||
engine = FlowController(store)
|
||||
asyncio.run(engine.reload())
|
||||
|
||||
entry = engine.loaded["house.double"]
|
||||
assert (entry.flow, entry.flow_id) == ("house", flow_id)
|
||||
# Read by the id: by the key it would be the template, and missing.
|
||||
assert not entry.missing_source and entry.status.value == "active"
|
||||
assert engine.loaded["house.hook"].node.url == f"{HOOK_PREFIX}/{flow_id}/tick"
|
||||
|
||||
engine.run_flow("house")
|
||||
|
||||
assert engine.values()["house.doubled"]["value"] == 43.0
|
||||
assert {s.id for s in engine.node_statuses("house")} >= {"house.double"}
|
||||
|
||||
|
||||
def test_stopping_writes_under_the_id_and_gates_the_key(store: FlowStore, flow_id: str):
|
||||
engine = FlowController(store)
|
||||
|
||||
async def scenario() -> None:
|
||||
await engine.reload()
|
||||
await engine.set_enabled(flow_id, "house", False)
|
||||
assert engine.disabled == {"house"}
|
||||
assert not store.read_enabled(flow_id)
|
||||
# Read back by id when everything is built again.
|
||||
await engine.reload()
|
||||
assert engine.disabled == {"house"}
|
||||
|
||||
asyncio.run(scenario())
|
||||
assert (store.root / flow_id / "runtime.json").exists()
|
||||
|
||||
|
||||
def test_a_flow_rebuilt_by_id_is_taken_out_by_its_key_once_gone(
|
||||
store: FlowStore, flow_id: str
|
||||
):
|
||||
engine = FlowController(store)
|
||||
|
||||
async def scenario() -> None:
|
||||
await engine.reload()
|
||||
assert "house.sensor" in engine.loaded
|
||||
await engine.reload_flow(flow_id)
|
||||
assert "house.sensor" in engine.loaded
|
||||
|
||||
store.delete_flow(flow_id)
|
||||
await engine.reload_flow(flow_id)
|
||||
|
||||
asyncio.run(scenario())
|
||||
assert not [node for node in engine.loaded if node.startswith("house.")]
|
||||
assert engine.pipeline is not None and not engine.pipeline.flow_nodes("house")
|
||||
|
||||
|
||||
def test_a_draft_previews_by_id_and_reports_by_key(store: FlowStore, flow_id: str):
|
||||
draft = store.read_flow(flow_id).model_copy(update={"title": "Edited"})
|
||||
store.write_draft(draft, draft.version)
|
||||
engine = FlowController(store)
|
||||
|
||||
preview = engine.preview(flow_id)
|
||||
|
||||
assert {node.id for node in preview.nodes} == {
|
||||
"house.sensor",
|
||||
"house.double",
|
||||
"house.hook",
|
||||
}
|
||||
assert all(node.status == "active" for node in preview.nodes)
|
||||
engine.run_preview(flow_id)
|
||||
assert engine.state.get("house.doubled") == 43.0
|
||||
|
||||
|
||||
def test_what_a_key_leaves_behind_is_found_and_forgotten(store: FlowStore):
|
||||
engine = FlowController(store)
|
||||
queue = MemoryWorkQueue()
|
||||
engine.execution = SimpleNamespace(queue=queue) # type: ignore[assignment]
|
||||
engine.state.set("house.reading", 1.0)
|
||||
engine.state.set("__node__:house.double:last", 2)
|
||||
engine.state.set("housework.reading", 3.0)
|
||||
queue.add_delayed(WorkItem(kind="flush", node="house.double", flow="house"), 9e9)
|
||||
queue.park("house", WorkItem(kind="cascade", node="house.sensor", flow="house"))
|
||||
# Another flow's work, carrying one of this key's messages.
|
||||
queue.add(
|
||||
WorkItem(
|
||||
kind="cascade", node="garden.pump", flow="garden", outputs={"house.x": 1}
|
||||
)
|
||||
)
|
||||
|
||||
found = engine.key_residue("house")
|
||||
|
||||
assert "house.reading" in found and "__node__:house.double:last" in found
|
||||
assert "housework.reading" not in found
|
||||
assert sum("queued work" in what for what in found) == 3
|
||||
assert engine.key_residue("garden") == ["queued work of 'garden.pump'"]
|
||||
|
||||
engine.forget_flow("house")
|
||||
|
||||
# What the other flow queued is its own, and runs as it would have.
|
||||
assert engine.key_residue("house") == ["queued work of 'garden.pump'"]
|
||||
assert engine.state.get("housework.reading") == 3.0
|
||||
@@ -8,6 +8,7 @@ is why the migration is driven through crashes, refusals and stores without git.
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -18,17 +19,23 @@ from fluksio.flow.dashboards import (
|
||||
DASHBOARD_DIR,
|
||||
DashboardDef,
|
||||
DashboardExists,
|
||||
DashboardNotFound,
|
||||
DashboardStore,
|
||||
default_dashboard,
|
||||
results_dashboard,
|
||||
)
|
||||
from fluksio.flow.identity import COMMIT_MESSAGE, IdsRefused, assign_ids
|
||||
from fluksio.flow.schemas import FlowDef
|
||||
from fluksio.flow.schemas import FlowCreate, FlowDef
|
||||
from fluksio.flow.store import (
|
||||
ISSUED_ID,
|
||||
FlowExists,
|
||||
FlowNotFound,
|
||||
FlowStore,
|
||||
IdentityChange,
|
||||
IdTaken,
|
||||
RecordedKey,
|
||||
ReservedKey,
|
||||
StaleVersion,
|
||||
new_id,
|
||||
)
|
||||
|
||||
@@ -37,23 +44,51 @@ from fluksio.flow.store import (
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Graves:
|
||||
"""Tombstones and history, as `fluksio.flow.tombstones.Tombstones` keeps
|
||||
them in the database, which store tests have none of."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.stones: dict[str, str] = {}
|
||||
#: Ids that history rows name.
|
||||
self.history: set[str] = set()
|
||||
|
||||
def bury(self, flow_id: str, name: str) -> None:
|
||||
self.stones[flow_id] = name
|
||||
|
||||
def lift(self, flow_id: str) -> None:
|
||||
self.stones.pop(flow_id, None)
|
||||
|
||||
def recorded(self, flow_id: str) -> str | None:
|
||||
if flow_id in self.stones:
|
||||
return self.stones[flow_id]
|
||||
return flow_id if flow_id in self.history else None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> FlowStore:
|
||||
return FlowStore(tmp_path / "flows")
|
||||
def graves() -> Graves:
|
||||
return Graves()
|
||||
|
||||
|
||||
def test_a_flow_is_given_its_key_as_id_and_keeps_both(store: FlowStore):
|
||||
created = store.write_draft(FlowDef(name="heating"), 0)
|
||||
assert (created.id, created.name) == ("heating", "heating")
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path, graves: Graves) -> FlowStore:
|
||||
return FlowStore(tmp_path / "flows", tombstones=graves) # type: ignore[arg-type]
|
||||
|
||||
published = store.publish_flow("heating", created.version)
|
||||
|
||||
def test_a_new_flow_gets_an_issued_id_and_keeps_it(store: FlowStore):
|
||||
created = store.create(FlowDef(name="heating"))
|
||||
flow_id = created.id
|
||||
assert ISSUED_ID.fullmatch(flow_id) and flow_id.startswith("f_")
|
||||
assert created.name == "heating" and created.version == 1
|
||||
assert store.key_index() == {"heating": flow_id}
|
||||
|
||||
published = store.publish_flow(flow_id, created.version)
|
||||
store.write_draft(published.model_copy(update={"title": "Warm"}), 1)
|
||||
discarded = store.discard_draft("heating")
|
||||
discarded = store.discard_draft(flow_id)
|
||||
|
||||
assert json.loads((store.root / "heating" / "flow.json").read_text())["id"] == (
|
||||
"heating"
|
||||
)
|
||||
assert (discarded.id, discarded.name) == ("heating", "heating")
|
||||
on_disk = json.loads((store.root / flow_id / "flow.json").read_text())
|
||||
assert (on_disk["id"], on_disk["name"]) == (flow_id, "heating")
|
||||
assert (discarded.id, discarded.name) == (flow_id, "heating")
|
||||
|
||||
|
||||
def test_a_document_from_before_ids_reads_with_its_directory_as_id(store: FlowStore):
|
||||
@@ -70,9 +105,13 @@ def test_a_document_from_before_ids_reads_with_its_directory_as_id(store: FlowSt
|
||||
assert store.write_draft(FlowDef(name="heating", title="Warm"), 1).version == 2
|
||||
|
||||
|
||||
def test_a_save_cannot_change_an_id_or_a_key(store: FlowStore):
|
||||
store.write_flow(FlowDef(id="x_heating", name="heating"))
|
||||
def test_a_save_neither_creates_nor_changes_an_id_or_a_key(store: FlowStore):
|
||||
with pytest.raises(FlowNotFound):
|
||||
store.write_draft(FlowDef(name="nobody"), 0)
|
||||
with pytest.raises(DashboardNotFound):
|
||||
DashboardStore(store).write_draft(default_dashboard("nobody"), 0)
|
||||
|
||||
store.write_flow(FlowDef(id="x_heating", name="heating"))
|
||||
with pytest.raises(IdentityChange):
|
||||
store.write_draft(FlowDef(id="x_heating", name="warmth"), 1)
|
||||
with pytest.raises(IdentityChange):
|
||||
@@ -87,56 +126,218 @@ def test_a_save_cannot_change_an_id_or_a_key(store: FlowStore):
|
||||
|
||||
def test_a_key_belongs_to_one_flow_drafts_included(store: FlowStore):
|
||||
"""A draft nobody published holds its key, and a delete lets it go."""
|
||||
store.write_draft(FlowDef(id="x_heating", name="heating"), 0)
|
||||
assert store.key_index() == {"heating": "x_heating"}
|
||||
first = store.create(FlowDef(name="heating"))
|
||||
assert store.key_index() == {"heating": first.id}
|
||||
|
||||
with pytest.raises(FlowExists):
|
||||
store.write_draft(FlowDef(name="heating"), 0)
|
||||
store.create(FlowDef(name="heating"))
|
||||
with pytest.raises(FlowExists):
|
||||
store.create(FlowDef(name="heating"), "x_heating")
|
||||
with pytest.raises(FlowExists):
|
||||
store.write_flow(FlowDef(id="y_heating", name="heating"))
|
||||
|
||||
store.delete_flow("x_heating")
|
||||
assert store.write_draft(FlowDef(name="heating"), 0).id == "heating"
|
||||
assert store.key_index() == {"heating": "heating"}
|
||||
store.delete_flow(first.id)
|
||||
second = store.create(FlowDef(name="heating"))
|
||||
assert second.id != first.id
|
||||
assert store.key_index() == {"heating": second.id}
|
||||
|
||||
|
||||
def test_a_key_belongs_to_one_dashboard_drafts_included(store: FlowStore):
|
||||
dashboards = DashboardStore(store)
|
||||
dashboards.write_draft(DashboardDef(id="x_home", name="home"), 0)
|
||||
first = dashboards.create(default_dashboard("home"))
|
||||
assert ISSUED_ID.fullmatch(first.id) and first.id.startswith("d_")
|
||||
# Its own id space: a flow may share the key.
|
||||
store.write_draft(FlowDef(name="home"), 0)
|
||||
store.create(FlowDef(name="home"))
|
||||
|
||||
with pytest.raises(DashboardExists):
|
||||
dashboards.write_draft(default_dashboard("home"), 0)
|
||||
dashboards.create(default_dashboard("home"))
|
||||
with pytest.raises(DashboardExists):
|
||||
dashboards.write(DashboardDef(id="y_home", name="home"))
|
||||
|
||||
dashboards.delete("x_home")
|
||||
assert dashboards.write(default_dashboard("home")).id == "home"
|
||||
dashboards.delete(first.id)
|
||||
assert dashboards.create(default_dashboard("home"), "home").id == "home"
|
||||
|
||||
|
||||
def test_a_copy_takes_a_key_nobody_holds(store: FlowStore):
|
||||
def test_a_pin_is_an_id_nothing_holds(store: FlowStore):
|
||||
store.create(FlowDef(name="heating"), "x_heating")
|
||||
with pytest.raises(IdTaken):
|
||||
store.create(FlowDef(name="cooling"), "x_heating")
|
||||
# Another flow's key would make naming either ambiguous.
|
||||
with pytest.raises(IdTaken):
|
||||
store.create(FlowDef(name="cooling"), "heating")
|
||||
# And a directory left behind would hand the new flow what is in it.
|
||||
(store.root / "leftover" / "nodes").mkdir(parents=True)
|
||||
with pytest.raises(IdTaken):
|
||||
store.create(FlowDef(name="leftover"), "leftover")
|
||||
dashboards = DashboardStore(store)
|
||||
dashboards.create(default_dashboard("home"), "x_home")
|
||||
with pytest.raises(IdTaken):
|
||||
dashboards.create(default_dashboard("house"), "x_home")
|
||||
|
||||
|
||||
def test_an_issued_id_avoids_whatever_history_or_a_directory_holds(
|
||||
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
candidates = iter(["f_aaaaaaaa", "f_bbbbbbbb", "f_cccccccc", "f_dddddddd"])
|
||||
monkeypatch.setattr(
|
||||
"fluksio.flow.store.new_id",
|
||||
lambda kind, taken: next(c for c in candidates if not taken(c)),
|
||||
)
|
||||
graves.bury("f_aaaaaaaa", "gone")
|
||||
graves.history.add("f_bbbbbbbb")
|
||||
(store.root / "f_cccccccc").mkdir()
|
||||
|
||||
assert store.create(FlowDef(name="heating")).id == "f_dddddddd"
|
||||
|
||||
|
||||
def test_a_deleted_flow_is_buried_before_its_files_go(
|
||||
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
flow_id = store.create(FlowDef(name="heating")).id
|
||||
|
||||
def crash(path: Path) -> None:
|
||||
raise OSError("disk gone")
|
||||
|
||||
monkeypatch.setattr("fluksio.flow.store.shutil.rmtree", crash)
|
||||
with pytest.raises(OSError):
|
||||
store.delete_flow(flow_id)
|
||||
# The name is safe, and the live document still wins over it.
|
||||
assert graves.stones == {flow_id: "heating"}
|
||||
assert store.read_flow(flow_id, draft=True).name == "heating"
|
||||
|
||||
monkeypatch.undo()
|
||||
assert store.delete_flow(flow_id) == "heating"
|
||||
assert graves.stones == {flow_id: "heating"}
|
||||
assert not store.exists(flow_id)
|
||||
|
||||
|
||||
def test_a_delete_that_stops_after_the_files_has_kept_the_name(
|
||||
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
flow_id = store.create(FlowDef(name="heating")).id
|
||||
|
||||
def killed(message: str) -> None:
|
||||
raise OSError("killed before the commit")
|
||||
|
||||
monkeypatch.setattr(store, "_commit", killed)
|
||||
with pytest.raises(OSError):
|
||||
store.delete_flow(flow_id)
|
||||
monkeypatch.undo()
|
||||
|
||||
assert not store.exists(flow_id)
|
||||
assert graves.stones == {flow_id: "heating"}
|
||||
with pytest.raises(FlowNotFound):
|
||||
store.delete_flow(flow_id)
|
||||
|
||||
|
||||
def test_a_pinned_restore_takes_up_its_history_under_its_own_key(
|
||||
store: FlowStore, graves: Graves
|
||||
):
|
||||
flow_id = store.create(FlowDef(name="heating")).id
|
||||
store.delete_flow(flow_id)
|
||||
|
||||
with pytest.raises(RecordedKey):
|
||||
store.create(FlowDef(name="warmth"), flow_id)
|
||||
assert graves.stones == {flow_id: "heating"}
|
||||
|
||||
restored = store.create(FlowDef(name="heating"), flow_id)
|
||||
assert restored.id == flow_id and graves.stones == {}
|
||||
|
||||
# History from before tombstones is filed under an id that was its key.
|
||||
graves.history.add("legacy")
|
||||
with pytest.raises(RecordedKey):
|
||||
store.create(FlowDef(name="other"), "legacy")
|
||||
assert store.create(FlowDef(name="legacy"), "legacy").id == "legacy"
|
||||
|
||||
|
||||
def test_a_key_reused_after_a_delete_starts_a_new_history(
|
||||
store: FlowStore, graves: Graves
|
||||
):
|
||||
old = store.create(FlowDef(name="heating")).id
|
||||
store.delete_flow(old)
|
||||
|
||||
new = store.create(FlowDef(name="heating")).id
|
||||
|
||||
assert new != old
|
||||
assert graves.stones == {old: "heating"}
|
||||
|
||||
|
||||
def test_two_creates_of_one_key_make_one_flow(store: FlowStore):
|
||||
def create(_: int) -> str:
|
||||
try:
|
||||
return store.create(FlowDef(name="heating")).id
|
||||
except FlowExists:
|
||||
return ""
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
made = [flow_id for flow_id in pool.map(create, range(8)) if flow_id]
|
||||
|
||||
assert len(made) == 1
|
||||
assert store.list_flows() == made
|
||||
|
||||
|
||||
def test_two_results_dashboards_of_one_flow_make_one(store: FlowStore):
|
||||
dashboards = DashboardStore(store)
|
||||
flow = FlowDef(id="x_lab", name="lab", mode="batch")
|
||||
|
||||
def generate(_: int) -> str:
|
||||
try:
|
||||
return dashboards.create(results_dashboard(flow), "", True).id
|
||||
except DashboardExists:
|
||||
return ""
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
made = [board for board in pool.map(generate, range(4)) if board]
|
||||
|
||||
assert len(made) == 1
|
||||
assert dashboards.read(made[0]).name == "lab_results"
|
||||
|
||||
|
||||
def test_a_discard_leaves_no_old_draft_version_to_match(store: FlowStore):
|
||||
flow_id = store.create(FlowDef(name="heating")).id
|
||||
published = store.publish_flow(flow_id, 1)
|
||||
held = store.write_draft(published.model_copy(update={"title": "A"}), 1)
|
||||
store.write_draft(held.model_copy(update={"title": "B"}), held.version)
|
||||
|
||||
restored = store.discard_draft(flow_id)
|
||||
|
||||
# Above every draft version handed out, so an editor holding one is stale.
|
||||
assert restored.version == 4 and restored.name == "heating"
|
||||
with pytest.raises(StaleVersion):
|
||||
store.write_draft(held.model_copy(update={"title": "late"}), held.version)
|
||||
assert store.write_draft(restored.model_copy(update={"title": "C"}), 4).version == 5
|
||||
|
||||
dashboards = DashboardStore(store)
|
||||
board = dashboards.create(default_dashboard("home"))
|
||||
dashboards.publish(board.id, 1)
|
||||
dashboards.write_draft(dashboards.read(board.id).model_copy(), 1)
|
||||
assert dashboards.discard_draft(board.id).version == 3
|
||||
# A direct write goes above a draft too.
|
||||
dashboards.write_draft(dashboards.read(board.id).model_copy(), 3)
|
||||
assert dashboards.write(dashboards.read(board.id)).version == 5
|
||||
|
||||
|
||||
def test_a_copy_gets_an_id_of_its_own_and_a_key_nobody_holds(store: FlowStore):
|
||||
store.write_flow(FlowDef(name="heating"))
|
||||
store.write_draft(FlowDef(id="x_other", name="heating_copy"), 0)
|
||||
store.create(FlowDef(name="heating_copy"))
|
||||
dashboards = DashboardStore(store)
|
||||
dashboards.write(default_dashboard("home"))
|
||||
dashboards.write_draft(DashboardDef(id="x_other", name="home_copy"), 0)
|
||||
dashboards.create(default_dashboard("home_copy"))
|
||||
|
||||
copy = store.duplicate_flow("heating")
|
||||
copy = store.duplicate_flow("heating", busy=lambda key: key == "heating_copy_2")
|
||||
board = dashboards.duplicate("home")
|
||||
|
||||
assert (copy.id, copy.name) == ("heating_copy_2", "heating_copy_2")
|
||||
assert (board.id, board.name) == ("home_copy_2", "home_copy_2")
|
||||
# `_copy_2` still had work behind it, so the copy went past it.
|
||||
assert copy.name == "heating_copy_3" and ISSUED_ID.fullmatch(copy.id)
|
||||
assert board.name == "home_copy_2" and ISSUED_ID.fullmatch(board.id)
|
||||
|
||||
|
||||
def test_a_new_key_shaped_like_an_issued_id_is_refused(store: FlowStore):
|
||||
with pytest.raises(ReservedKey):
|
||||
store.write_draft(FlowDef(name="f_k3q7m2xa"), 0)
|
||||
store.create(FlowDef(name="f_k3q7m2xa"))
|
||||
with pytest.raises(ReservedKey):
|
||||
DashboardStore(store).write_draft(default_dashboard("d_k3q7m2xa"), 0)
|
||||
assert store.write_draft(FlowDef(name="f_k3q7m2xa_lab"), 0).name == (
|
||||
"f_k3q7m2xa_lab"
|
||||
)
|
||||
DashboardStore(store).create(default_dashboard("d_k3q7m2xa"))
|
||||
assert store.create(FlowDef(name="f_k3q7m2xa_lab")).name == "f_k3q7m2xa_lab"
|
||||
|
||||
|
||||
def test_an_existing_key_shaped_like_an_issued_id_stays_saveable(store: FlowStore):
|
||||
@@ -515,3 +716,10 @@ def test_ls_lists_ids_keys_and_kinds(tmp_path: Path, capsys):
|
||||
assert rows[0] == ["ID", "KEY", "KIND"]
|
||||
assert ["home", "home", "dashboard"] in rows and ["home", "home", "flow"] in rows
|
||||
assert ["warmth", "radiators", "flow"] in rows
|
||||
|
||||
|
||||
def test_what_a_create_takes_is_a_document_but_for_its_version():
|
||||
"""`FlowCreate` lists the fields again; one added to `FlowDef` belongs there."""
|
||||
assert set(FlowCreate.model_fields) == set(FlowDef.model_fields) - {"version"}
|
||||
with pytest.raises(ValueError):
|
||||
FlowCreate(name="Not A Key")
|
||||
|
||||
@@ -9,6 +9,7 @@ from fluksio.flow.controller import FlowController
|
||||
from fluksio.flow.messages import MessageSpec
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||
from fluksio.flow.store import FlowStore, LibExists, LibNotFound
|
||||
from tests.utils.stores import created
|
||||
|
||||
SOURCE = "def process():\n return {'temp': 1}\n"
|
||||
|
||||
@@ -79,7 +80,7 @@ def test_a_shared_source_change_rebuilds_only_what_is_published(store: FlowStore
|
||||
# Shared in a draft that was never published.
|
||||
cooling = a_flow("cooling")
|
||||
cooling.nodes[0].source_ref = "read_temp"
|
||||
store.write_draft(cooling, 0)
|
||||
created(store, cooling)
|
||||
|
||||
controller = FlowController(store)
|
||||
rebuilt: list[str] = []
|
||||
@@ -131,7 +132,7 @@ def test_retiring_a_shared_source_leaves_each_user_a_copy(store: FlowStore):
|
||||
# Shared in a draft that was never published.
|
||||
cooling = a_flow("cooling")
|
||||
cooling.nodes[0].source_ref = "read_temp"
|
||||
store.write_draft(cooling, 0)
|
||||
created(store, cooling)
|
||||
|
||||
store.detach_lib("read_temp")
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ from fluksio.flow.pipeline import Pipeline
|
||||
from fluksio.flow.schemas import FlowDef, NodeDef
|
||||
from fluksio.flow.state import MemoryState
|
||||
from fluksio.flow.store import FlowStore
|
||||
from tests.utils.stores import created
|
||||
|
||||
SOURCE = "def process(reading, factor):\n return {'scaled': reading * factor}\n"
|
||||
|
||||
@@ -53,7 +54,7 @@ def test_a_setting_reaches_the_function_as_a_keyword_argument(store: FlowStore):
|
||||
|
||||
|
||||
def test_a_setting_named_after_a_port_is_refused(store: FlowStore):
|
||||
store.write_draft(a_flow(reading=3), 0)
|
||||
created(store, a_flow(reading=3))
|
||||
store.write_node_source("house", "scale", SOURCE, draft=True)
|
||||
controller = FlowController(store)
|
||||
|
||||
@@ -81,7 +82,7 @@ READING = [MessageSpec(name="reading", dtype=DType.FLOAT)]
|
||||
|
||||
|
||||
def _preview(store: FlowStore, flow: FlowDef, source: str | None):
|
||||
store.write_draft(flow, 0)
|
||||
created(store, flow)
|
||||
if source is not None:
|
||||
store.write_node_source("house", "scale", source, draft=True)
|
||||
return FlowController(store).preview("house").nodes[0]
|
||||
|
||||
@@ -680,3 +680,28 @@ def test_done_markers_do_not_grow_without_bound():
|
||||
# The newest are what redelivery would ask about.
|
||||
assert queue.was_done(f"entry-{queue_module.DONE_MARKERS + 499}", "f.node")
|
||||
assert not queue.was_done("entry-0", "f.node")
|
||||
|
||||
|
||||
def test_clearing_a_flow_drops_its_work_and_nothing_else():
|
||||
"""A new flow under the same key must never run what the old one queued."""
|
||||
queue = MemoryWorkQueue()
|
||||
queue.add(WorkItem(kind="cascade", node="gone.a", flow="gone"))
|
||||
# Another flow's work carrying one of its messages, and a run, which is
|
||||
# filed under a flow's id rather than its key: neither is the flow's.
|
||||
queue.add(
|
||||
WorkItem(kind="cascade", node="kept.a", flow="kept", outputs={"gone.x": 1})
|
||||
)
|
||||
queue.add(WorkItem(kind="run", node="", flow="gone", run_id="r1"))
|
||||
queue.add_delayed(WorkItem(kind="flush", node="gone.b", flow="gone"), 9e9)
|
||||
queue.add_delayed(WorkItem(kind="flush", node="kept.b", flow="kept"), 9e9)
|
||||
queue.park("gone", WorkItem(kind="cascade", node="gone.c", flow="gone"))
|
||||
assert len(queue.live_items()) == 6
|
||||
|
||||
queue.clear_flow("gone")
|
||||
|
||||
assert sorted((i.kind, i.node) for i in queue.live_items()) == [
|
||||
("cascade", "kept.a"),
|
||||
("flush", "kept.b"),
|
||||
("run", ""),
|
||||
]
|
||||
assert queue.next_due() == 9e9
|
||||
|
||||
@@ -66,3 +66,30 @@ def test_a_refused_stream_write_keeps_the_timer_for_retry(queue):
|
||||
assert queue.move_due(200.0) == 1
|
||||
assert queue.move_due(200.0) == 0
|
||||
assert [item.node for item in queue.claim(10, 10, due_only=True)] == ["f.stop"]
|
||||
|
||||
|
||||
def test_clearing_a_flow_drops_its_queued_claimed_and_held_work(queue):
|
||||
queue.add(WorkItem(kind="cascade", node="gone.p", flow="gone"))
|
||||
(claimed,) = queue.claim(1, 10)
|
||||
assert claimed.node == "gone.p"
|
||||
queue.add(WorkItem(kind="cascade", node="gone.a", flow="gone"))
|
||||
queue.add(
|
||||
WorkItem(kind="cascade", node="kept.a", flow="kept", outputs={"gone.x": 1})
|
||||
)
|
||||
queue.add(WorkItem(kind="run", node="", flow="gone", run_id="r1"))
|
||||
queue.add_delayed(WorkItem(kind="flush", node="gone.b", flow="gone"), 9e9)
|
||||
queue.add_delayed(WorkItem(kind="flush", node="kept.b", flow="kept"), 9e9)
|
||||
queue.park("gone", WorkItem(kind="cascade", node="gone.c", flow="gone"))
|
||||
assert len(queue.live_items()) == 7
|
||||
|
||||
queue.clear_flow("gone")
|
||||
|
||||
assert sorted((i.kind, i.node) for i in queue.live_items()) == [
|
||||
("cascade", "kept.a"),
|
||||
("flush", "kept.b"),
|
||||
("run", ""),
|
||||
]
|
||||
# Nothing comes back to be run again: not redelivered, not reclaimed.
|
||||
assert queue.reclaim_stale(0) == []
|
||||
queue.ack(claimed)
|
||||
assert [i.node for i in queue.claim(10, 10)] == ["kept.a", ""]
|
||||
|
||||
@@ -318,7 +318,7 @@ def test_stopping_one_flow_does_not_touch_the_others(
|
||||
before = {name: engine.loaded[name].node for name in ("a.io", "b.io")}
|
||||
lifecycle.clear()
|
||||
|
||||
await engine.set_enabled("a", False)
|
||||
await engine.set_enabled("a", "a", False)
|
||||
|
||||
assert lifecycle == ["stop a.io"]
|
||||
# Still built, still the same objects: only the lifecycle changed.
|
||||
@@ -326,14 +326,14 @@ def test_stopping_one_flow_does_not_touch_the_others(
|
||||
assert engine.disabled == {"a"}
|
||||
|
||||
lifecycle.clear()
|
||||
await engine.set_enabled("a", True)
|
||||
await engine.set_enabled("a", "a", True)
|
||||
|
||||
assert lifecycle == ["start a.io"]
|
||||
assert {name: engine.loaded[name].node for name in ("a.io", "b.io")} == before
|
||||
|
||||
# Started while running is a restart: the cancelled loops come back.
|
||||
lifecycle.clear()
|
||||
await engine.set_enabled("a", True)
|
||||
await engine.set_enabled("a", "a", True)
|
||||
|
||||
assert lifecycle == ["stop a.io", "start a.io"]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user