Flows and dashboards are created by POST under an issued or superuser-pinned id; PUT only updates; deleted flows leave a tombstone with their key; the controller stores by id and runs by key, hooks live under /hooks/<flow id>; a key with state or queued work behind it is refused until cleared

This commit is contained in:
2026-11-02 17:41:23 +01:00
parent fae85f5ecb
commit 08a6376836
63 changed files with 2832 additions and 936 deletions
+3 -3
View File
@@ -36,10 +36,10 @@ def save(
) -> Callable[..., Any]:
"""Seed the flow as a superuser, then save a change as someone else."""
client.delete(f"{PREFIX}/{FLOW}", headers=superuser_token_headers)
seeded = client.put(
f"{PREFIX}/{FLOW}", headers=superuser_token_headers, json=a_flow(0)
seeded = client.post(
f"{PREFIX}/", headers=superuser_token_headers, json={**a_flow(0), "id": FLOW}
)
assert seeded.status_code == 200, seeded.text
assert seeded.status_code == 201, seeded.text
version = seeded.json()["definition"]["version"]
def put(headers: dict[str, str], **changes: Any) -> Any:
+83 -39
View File
@@ -1,8 +1,10 @@
"""Dashboards over HTTP: saving one, which is also how the first one is made."""
"""Dashboards over HTTP: creating one under an issued id, and saving it."""
from fastapi.testclient import TestClient
from fluksio.core.config import settings
from fluksio.flow.store import ISSUED_ID
from tests.utils.flows import put_dashboard
PREFIX = f"{settings.API_V1_STR}/dashboards"
@@ -26,37 +28,78 @@ def a_dashboard(name: str) -> dict:
}
def test_a_save_creates_a_dashboard_that_does_not_exist_yet(
def test_a_dashboard_is_created_with_an_id_and_saved_by_it(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
"""A first draft, not a 404: creating one *is* saving it at version 0."""
body = a_dashboard("put_creates")
response = client.put(
f"{PREFIX}/put_creates", headers=superuser_token_headers, json=body
"""A first draft under an id the server issues, which every path then names."""
created = client.post(
f"{PREFIX}/", headers=superuser_token_headers, json={"name": "made_here"}
)
assert response.status_code == 200, response.text
saved = response.json()
assert saved["version"] == 1 and saved["has_draft"] is True
assert created.status_code == 201, created.text
made = created.json()
assert ISSUED_ID.fullmatch(made["id"]) and made["name"] == "made_here"
assert made["version"] == 1 and made["has_draft"] is True
# A draft alone: nothing was published, so no panel can be shown it.
assert (
client.get(f"{PREFIX}/put_creates", headers=superuser_token_headers).status_code
client.get(
f"{PREFIX}/{made['id']}", headers=superuser_token_headers
).status_code
== 404
)
body = {**a_dashboard("made_here"), "version": made["version"]}
saved = client.put(
f"{PREFIX}/{made['id']}", headers=superuser_token_headers, json=body
)
assert saved.status_code == 200, saved.text
draft = client.get(
f"{PREFIX}/put_creates?draft=true", headers=superuser_token_headers
f"{PREFIX}/{made['id']}?draft=true", headers=superuser_token_headers
)
assert draft.json()["widgets"] == body["widgets"]
(listed,) = client.get(
f"{PREFIX}/?name=made_here", headers=superuser_token_headers
).json()["data"]
assert listed["id"] == made["id"]
copy = client.post(
f"{PREFIX}/{made['id']}/duplicate", headers=superuser_token_headers
).json()
assert copy["name"] == "made_here_copy" and copy["id"] != made["id"]
for dashboard_id in (made["id"], copy["id"]):
client.delete(f"{PREFIX}/{dashboard_id}", headers=superuser_token_headers)
def test_a_save_neither_creates_nor_renames(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
missing = client.put(
f"{PREFIX}/nowhere",
headers=superuser_token_headers,
json={**a_dashboard("nowhere"), "version": 0},
)
assert missing.status_code == 404
saved = put_dashboard(
client, superuser_token_headers, a_dashboard("put_fixed")
).json()
renamed = client.put(
f"{PREFIX}/put_fixed",
headers=superuser_token_headers,
json={**saved, "name": "renamed"},
)
assert renamed.status_code == 409
elsewhere = client.put(
f"{PREFIX}/put_fixed",
headers=superuser_token_headers,
json={**saved, "id": "elsewhere"},
)
assert elsewhere.status_code == 422
client.delete(f"{PREFIX}/put_fixed", headers=superuser_token_headers)
def test_a_save_of_an_existing_dashboard_keeps_what_it_did_not_change(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
seeded = a_dashboard("put_updates")
first = client.put(
f"{PREFIX}/put_updates", headers=superuser_token_headers, json=seeded
).json()
first = put_dashboard(client, superuser_token_headers, seeded).json()
renamed = {**first, "widgets": [{**first["widgets"][0], "title": "Outside"}]}
second = client.put(
@@ -78,37 +121,38 @@ def test_a_save_based_on_a_version_someone_moved_past_is_refused(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
body = a_dashboard("put_conflicts")
client.put(f"{PREFIX}/put_conflicts", headers=superuser_token_headers, json=body)
put_dashboard(client, superuser_token_headers, body)
stale = client.put(
f"{PREFIX}/put_conflicts", headers=superuser_token_headers, json=body
f"{PREFIX}/put_conflicts",
headers=superuser_token_headers,
json={**body, "version": 1},
)
assert stale.status_code == 409
assert stale.json()["detail"]["current_version"] == 1
assert stale.json()["detail"]["current_version"] == 2
def test_a_dashboard_is_known_by_an_id_no_save_changes(
client: TestClient, superuser_token_headers: dict[str, str]
def test_only_a_superuser_pins_a_dashboard_id(
client: TestClient,
superuser_token_headers: dict[str, str],
normal_user_token_headers: dict[str, str],
) -> None:
saved = client.put(
f"{PREFIX}/put_identified",
pinned = {"id": "pinned_hall", "name": "pinned_hall"}
refused = client.post(f"{PREFIX}/", headers=normal_user_token_headers, json=pinned)
assert refused.status_code == 403
made = client.post(f"{PREFIX}/", headers=superuser_token_headers, json=pinned)
assert made.status_code == 201 and made.json()["id"] == "pinned_hall"
again = client.post(
f"{PREFIX}/",
headers=superuser_token_headers,
json=a_dashboard("put_identified"),
).json()
assert saved["id"] == "put_identified"
elsewhere = client.put(
f"{PREFIX}/put_identified",
headers=superuser_token_headers,
json={**saved, "id": "elsewhere"},
json={"id": "pinned_hall", "name": "other_hall"},
)
assert elsewhere.status_code == 422
renamed = client.post(
f"{PREFIX}/put_identified/rename",
headers=superuser_token_headers,
json={"name": "renamed"},
assert again.status_code == 409
# Anyone may make one under an id of the server's.
issued = client.post(
f"{PREFIX}/", headers=normal_user_token_headers, json={"name": "their_hall"}
)
assert renamed.status_code in (404, 405)
client.delete(f"{PREFIX}/put_identified", headers=superuser_token_headers)
assert issued.status_code == 201
for dashboard_id in ("pinned_hall", issued.json()["id"]):
client.delete(f"{PREFIX}/{dashboard_id}", headers=superuser_token_headers)
+32 -19
View File
@@ -12,6 +12,7 @@ from fluksio.cloud import config as cloud_config
from fluksio.core.config import settings
from fluksio.flow.devices import DeviceState
from fluksio.main import app
from tests.utils.flows import put_flow
PREFIX = f"{settings.API_V1_STR}/devices"
#: What the openpanel firmware calls, kept as aliases.
@@ -54,8 +55,10 @@ def _dashboard(client: TestClient, headers: dict[str, str], name: str) -> dict:
A new one is a draft, and a panel is only ever shown what is published.
"""
created = client.post(f"{DASHBOARDS}/{name}", headers=headers)
assert created.status_code == 200, created.text
created = client.post(
f"{DASHBOARDS}/", headers=headers, json={"id": name, "name": name}
)
assert created.status_code == 201, created.text
published = client.post(
f"{DASHBOARDS}/{name}/publish",
headers=headers,
@@ -95,7 +98,11 @@ def test_assign_and_read_back(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
for name in ("hall_a", "hall_b"):
client.post(f"{DASHBOARDS}/{name}", headers=superuser_token_headers)
client.post(
f"{DASHBOARDS}/",
headers=superuser_token_headers,
json={"id": name, "name": name},
)
_panels(
client,
@@ -132,7 +139,11 @@ def test_touch_belongs_to_the_device(
The same dashboard may hang on a hallway tablet and on a desk browser, so
the flag rides on the panel and each one answers for itself.
"""
client.post(f"{DASHBOARDS}/shared", headers=superuser_token_headers)
client.post(
f"{DASHBOARDS}/",
headers=superuser_token_headers,
json={"id": "shared", "name": "shared"},
)
_panels(
client,
superuser_token_headers,
@@ -553,12 +564,8 @@ def _publish_flow(
) -> dict:
"""Write a flow and publish it, replacing whatever was there."""
client.delete(f"{FLOWS}/{name}", headers=headers)
saved = client.put(
f"{FLOWS}/{name}",
headers=headers,
json={"name": name, "nodes": nodes, "version": 0, **more},
)
assert saved.status_code == 200, saved.text
saved = put_flow(client, headers, {"name": name, "nodes": nodes, **more}, name)
assert saved.is_success, saved.text
published = client.post(
f"{FLOWS}/{name}/publish",
headers=headers,
@@ -590,10 +597,10 @@ def test_a_devices_io_comes_from_its_device_nodes(
_panels(client, superuser_token_headers, {"panels": [{"id": "porch_io"}]})
_publish_flow(client, superuser_token_headers, "porch", [PORCH_NODE])
# A draft wires nothing until it is published.
client.put(
f"{FLOWS}/porch_draft",
headers=superuser_token_headers,
json={
put_flow(
client,
superuser_token_headers,
{
"name": "porch_draft",
"nodes": [
_device_node(
@@ -603,6 +610,7 @@ def test_a_devices_io_comes_from_its_device_nodes(
],
"version": 0,
},
"porch_draft",
)
first = client.get(f"{PREFIX}/porch_io", headers=superuser_token_headers)
@@ -792,10 +800,10 @@ def test_a_port_wired_twice_or_an_unknown_device_is_an_issue(
"ports": [{"port": "touch", "direction": "in", "dtype": "bool"}],
}
store._write_states(states)
client.put(
f"{FLOWS}/porch_too",
headers=superuser_token_headers,
json={
put_flow(
client,
superuser_token_headers,
{
"name": "porch_too",
"version": client.get(
f"{FLOWS}/porch_too", headers=superuser_token_headers
@@ -816,6 +824,7 @@ def test_a_port_wired_twice_or_an_unknown_device_is_an_issue(
)
],
},
"porch_too",
)
assert codes("porch_too") == {
("device_port", "relay", True),
@@ -1440,7 +1449,11 @@ def test_a_panel_credential_cannot_move_artifacts(
the scope check has to be handed the request there too — without it a panel
would act as whoever approved it.
"""
client.post(f"{DASHBOARDS}/panel_art", headers=superuser_token_headers)
client.post(
f"{DASHBOARDS}/",
headers=superuser_token_headers,
json={"id": "panel_art", "name": "panel_art"},
)
_panels(
client,
superuser_token_headers,
+7 -8
View File
@@ -7,6 +7,7 @@ from sqlmodel import Session, select
from fluksio.core.config import settings
from fluksio.core.db import seed_flavors
from fluksio.models import Flavor
from tests.utils.flows import put_flow
def test_an_instance_starts_with_sizes_to_pick_from(
@@ -97,12 +98,11 @@ def test_a_flow_cannot_name_a_size_that_does_not_exist(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
"""Caught at the save, which covers the canvas and `fluksio sync` at once."""
response = client.put(
f"{settings.API_V1_STR}/flows/flavor_check",
response = client.post(
f"{settings.API_V1_STR}/flows/",
headers=superuser_token_headers,
json={
"name": "flavor_check",
"version": 1,
"nodes": [
{"id": "fit", "type": "python", "resources": {"flavor": "imaginary"}}
],
@@ -120,12 +120,11 @@ def test_a_size_a_node_still_asks_for_is_not_deleted(
headers=superuser_token_headers,
json={"name": "in-use", "cpus": 2},
)
client.put(
f"{settings.API_V1_STR}/flows/flavor_user",
headers=superuser_token_headers,
json={
put_flow(
client,
superuser_token_headers,
{
"name": "flavor_user",
"version": 1,
"nodes": [
{"id": "fit", "type": "python", "resources": {"flavor": "in-use"}}
],
+341
View File
@@ -0,0 +1,341 @@
"""Flows over HTTP by id, with keys apart from them.
A flow made through the API gets an opaque id unless a superuser pins one, so
most tests here run on both: a pinned ``x_<key>`` and an issued id. A path
that quietly read the key where it meant the id passes for a flow whose id is
its key, and nowhere else.
"""
import uuid
from collections.abc import Callable
from datetime import UTC, datetime
from typing import Any
import pytest
from fastapi.testclient import TestClient
from sqlmodel import Session, delete, select
from fluksio.core.config import settings
from fluksio.flow.store import ISSUED_ID
from fluksio.flow.tombstones import names_of
from fluksio.models import FlowTombstone, Run
FLOWS = f"{settings.API_V1_STR}/flows"
DASHBOARDS = f"{settings.API_V1_STR}/dashboards"
SENSOR = "def process():\n return {'reading': 21.5}\n"
DOUBLE = "def process(reading):\n return {'doubled': reading * 2}\n"
def a_flow(key: str, mode: str = "live") -> dict[str, Any]:
return {
"name": key,
"mode": mode,
"nodes": [
{
"id": "sensor",
"type": "python",
"provides": [{"name": "reading", "dtype": "float"}],
},
{
"id": "double",
"type": "python",
"requires": [{"name": "reading", "dtype": "float"}],
"provides": [{"name": "doubled", "dtype": "float"}],
},
{
"id": "hook",
"type": "http",
"params": {"url": "/tick", "method": "POST"},
"provides": [{"name": "tick", "dtype": "float"}],
},
],
"outputs": ["doubled"] if mode == "batch" else [],
}
def a_key() -> str:
return f"ids_{uuid.uuid4().hex[:8]}"
Make = Callable[..., dict[str, Any]]
@pytest.fixture(params=["pinned", "issued"])
def make(
request: pytest.FixtureRequest,
client: TestClient,
superuser_token_headers: dict[str, str],
) -> Make:
"""Create a flow, pinned to ``x_<key>`` or under an issued id; its detail."""
made: list[str] = []
def create(key: str, **document: Any) -> dict[str, Any]:
body = {**a_flow(key), **document}
if request.param == "pinned":
body["id"] = f"x_{key}"
response = client.post(f"{FLOWS}/", headers=superuser_token_headers, json=body)
assert response.status_code == 201, response.text
detail: dict[str, Any] = response.json()
made.append(detail["definition"]["id"])
return detail
yield create # type: ignore[misc]
for flow_id in made:
client.delete(f"{FLOWS}/{flow_id}", headers=superuser_token_headers)
def test_a_flow_is_made_found_and_saved_by_its_id(
make: Make, client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
key = a_key()
definition = make(key)["definition"]
flow_id = definition["id"]
assert flow_id != key and definition["name"] == key
assert flow_id == f"x_{key}" or ISSUED_ID.fullmatch(flow_id)
read = client.get(f"{FLOWS}/{flow_id}", headers=superuser_token_headers)
assert read.json()["definition"]["name"] == key
# By its key it is not there: only a client resolves a key.
assert client.get(
f"{FLOWS}/{key}", headers=superuser_token_headers
).status_code == (404)
(listed,) = client.get(
f"{FLOWS}/?name={key}", headers=superuser_token_headers
).json()["data"]
assert (listed["id"], listed["name"]) == (flow_id, key)
assert (
client.get(f"{FLOWS}/?name={key}_no", headers=superuser_token_headers).json()[
"data"
]
== []
)
saved = client.put(
f"{FLOWS}/{flow_id}",
headers=superuser_token_headers,
json={**definition, "title": "Retitled"},
)
assert saved.status_code == 200, saved.text
assert saved.json()["definition"]["title"] == "Retitled"
renamed = client.put(
f"{FLOWS}/{flow_id}",
headers=superuser_token_headers,
json={**saved.json()["definition"], "name": f"{key}_b"},
)
assert renamed.status_code == 409
copy = client.post(
f"{FLOWS}/{flow_id}/duplicate", headers=superuser_token_headers
).json()["definition"]
assert copy["name"] == f"{key}_copy" and ISSUED_ID.fullmatch(copy["id"])
client.delete(f"{FLOWS}/{copy['id']}", headers=superuser_token_headers)
def test_a_flow_runs_under_its_key_and_is_reached_by_its_id(
make: Make, client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
"""Publish, a message, the nodes running, the state under the key."""
key = a_key()
flow_id = make(key)["definition"]["id"]
for node, code in (("sensor", SENSOR), ("double", DOUBLE)):
saved = client.put(
f"{FLOWS}/{flow_id}/nodes/{node}/source",
headers=superuser_token_headers,
json={"code": code},
)
assert saved.json()["id"] == f"{key}.{node}"
version = client.get(f"{FLOWS}/{flow_id}", headers=superuser_token_headers).json()[
"definition"
]["version"]
published = client.post(
f"{FLOWS}/{flow_id}/publish",
headers=superuser_token_headers,
json={"version": version},
)
assert published.status_code == 200, published.text
assert {n["id"]: n["status"] for n in published.json()["nodes"]} == {
f"{key}.sensor": "active",
f"{key}.double": "active",
f"{key}.hook": "active",
}
ran = client.post(
f"{FLOWS}/{flow_id}/run", headers=superuser_token_headers, json={"inputs": {}}
)
assert ran.json()["values"][f"{key}.doubled"]["value"] == 43.0
triggered = client.post(
f"{FLOWS}/{flow_id}/nodes/double/trigger",
headers=superuser_token_headers,
json={"values": {f"{key}.reading": 1.0}},
)
assert triggered.json()["values"][f"{key}.doubled"]["value"] == 2.0
history = client.get(
f"{FLOWS}/{flow_id}/history/doubled", headers=superuser_token_headers
).json()
assert history["message"] == f"{key}.doubled" and history["points"]
# The webhook is under the id, which a later key change leaves alone.
hooked = client.post(f"/hooks/{flow_id}/tick", json={"tick": 7.0})
assert hooked.status_code == 200, hooked.text
assert client.post(f"/hooks/{key}/tick", json={"tick": 8.0}).status_code == 404
state = client.get(f"{FLOWS}/{flow_id}/state", headers=superuser_token_headers)
assert state.json()["values"][f"{key}.tick"]["value"] == 7.0
paused = client.post(f"{FLOWS}/{flow_id}/pause", headers=superuser_token_headers)
assert paused.status_code == 200
assert client.get(f"{FLOWS}/{flow_id}", headers=superuser_token_headers).json()[
"paused"
]
client.post(f"{FLOWS}/{flow_id}/resume", headers=superuser_token_headers)
stopped = client.post(f"{FLOWS}/{flow_id}/stop", headers=superuser_token_headers)
assert stopped.json()["enabled"] is False
assert (
client.post(
f"{FLOWS}/{flow_id}/run",
headers=superuser_token_headers,
json={"inputs": {}},
).status_code
== 409
)
started = client.post(f"{FLOWS}/{flow_id}/start", headers=superuser_token_headers)
assert started.json()["enabled"] is True
acknowledged = client.post(
f"{FLOWS}/{flow_id}/nodes/double/acknowledge", headers=superuser_token_headers
)
assert acknowledged.status_code == 200
def test_a_results_dashboard_is_made_from_a_flow_id_under_the_flows_key(
make: Make, client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
key = a_key()
flow_id = make(key, mode="batch")["definition"]["id"]
client.post(
f"{FLOWS}/{flow_id}/publish",
headers=superuser_token_headers,
json={"version": 1},
)
made = client.post(
f"{DASHBOARDS}/from-flow/{flow_id}", headers=superuser_token_headers
)
assert made.status_code == 201, made.text
board = made.json()
assert board["name"] == f"{key}_results" and ISSUED_ID.fullmatch(board["id"])
assert board["has_draft"] is False
again = client.post(
f"{DASHBOARDS}/from-flow/{flow_id}", headers=superuser_token_headers
)
assert again.status_code == 409
by_key = client.post(
f"{DASHBOARDS}/from-flow/{key}", headers=superuser_token_headers
)
assert by_key.status_code == 404
client.delete(f"{DASHBOARDS}/{board['id']}", headers=superuser_token_headers)
def test_who_may_create_what(
client: TestClient,
superuser_token_headers: dict[str, str],
normal_user_token_headers: dict[str, str],
) -> None:
key = a_key()
plain = {"name": key, "nodes": [{"id": "hold", "type": "delay"}]}
# Anyone makes a flow under an issued id, as anyone could save one before.
made = client.post(f"{FLOWS}/", headers=normal_user_token_headers, json=plain)
assert made.status_code == 201, made.text
flow_id = made.json()["definition"]["id"]
assert ISSUED_ID.fullmatch(flow_id)
# Code stays a superuser's to put there, and so does choosing an id.
coded = client.post(
f"{FLOWS}/",
headers=normal_user_token_headers,
json={**plain, "name": f"{key}_code", "nodes": [{"id": "fn"}]},
)
assert coded.status_code == 403
pinned = client.post(
f"{FLOWS}/",
headers=normal_user_token_headers,
json={**plain, "name": f"{key}_pin", "id": f"x_{key}"},
)
assert pinned.status_code == 403
def pin(flow: str, name: str) -> int:
return client.post(
f"{FLOWS}/",
headers=superuser_token_headers,
json={**plain, "name": name, "id": flow},
).status_code
assert pin("Not-An-Id", f"{key}_a") == 422
assert pin("graph", f"{key}_b") == 422 # behind `GET /flows/graph`
assert pin(flow_id, f"{key}_c") == 409 # a live flow's
assert pin(f"x_{key}", key) == 409 # the key is taken
assert pin(f"x_{key}", "f_k3q7m2xa") == 400 # shaped like an issued id
client.delete(f"{FLOWS}/{flow_id}", headers=superuser_token_headers)
def test_a_deleted_flow_keeps_its_name_and_frees_its_key(
make: Make,
client: TestClient,
superuser_token_headers: dict[str, str],
db: Session,
) -> None:
key = a_key()
old = make(key)["definition"]["id"]
db.add(Run(id=f"run-{key}", flow=old, status="ok", created_at=datetime.now(UTC)))
db.commit()
controller = client.app.state.flow_controller # type: ignore[attr-defined]
controller.state.set(f"{key}.doubled", 1.0)
deleted = client.delete(f"{FLOWS}/{old}", headers=superuser_token_headers)
assert deleted.status_code == 200
assert db.get(FlowTombstone, old).name == key
assert names_of([old], controller.store) == {old: key}
# What the old flow left under its key went with it.
assert controller.key_residue(key) == []
# Something still writing there keeps the key from a new flow.
controller.state.set(f"{key}.stray", 1.0)
refused = client.post(
f"{FLOWS}/", headers=superuser_token_headers, json=a_flow(key)
)
assert refused.status_code == 409 and f"{key}.stray" in refused.json()["detail"]
controller.forget_flow(key)
# Under an id of its own, not the one the runs are filed under.
made = client.post(f"{FLOWS}/", headers=superuser_token_headers, json=a_flow(key))
assert made.status_code == 201, made.text
new = made.json()["definition"]["id"]
assert new != old
assert db.exec(select(Run).where(Run.flow == new)).first() is None
client.delete(f"{FLOWS}/{new}", headers=superuser_token_headers)
db.exec(delete(Run).where(Run.id == f"run-{key}")) # type: ignore[call-overload]
db.commit()
def test_a_pinned_restore_takes_its_history_back(
client: TestClient, superuser_token_headers: dict[str, str], db: Session
) -> None:
key = a_key()
pin = f"x_{key}"
plain = {"name": key, "nodes": []}
client.post(f"{FLOWS}/", headers=superuser_token_headers, json={**plain, "id": pin})
client.delete(f"{FLOWS}/{pin}", headers=superuser_token_headers)
other = client.post(
f"{FLOWS}/",
headers=superuser_token_headers,
json={**plain, "name": f"{key}_b", "id": pin},
)
assert other.status_code == 409
restored = client.post(
f"{FLOWS}/", headers=superuser_token_headers, json={**plain, "id": pin}
)
assert restored.status_code == 201, restored.text
db.expire_all()
assert db.get(FlowTombstone, pin) is None
client.delete(f"{FLOWS}/{pin}", headers=superuser_token_headers)
+42 -81
View File
@@ -11,6 +11,7 @@ from fluksio.api.routes import flows as flows_route
from fluksio.core.config import settings
from fluksio.flow.events import event_bus
from fluksio.models import Run, RunArtifact, RunMetric, RunNode
from tests.utils.flows import put_flow
PREFIX = f"{settings.API_V1_STR}/flows"
@@ -51,10 +52,8 @@ def test_flows_require_authentication(client: TestClient) -> None:
def test_save_then_read_flow(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
response = client.put(
f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow()
)
assert response.status_code == 200
response = put_flow(client, superuser_token_headers, a_flow(), "demo")
assert response.status_code == 201
response = client.get(f"{PREFIX}/demo", headers=superuser_token_headers)
assert response.status_code == 200
@@ -68,58 +67,35 @@ def test_save_then_read_flow(
assert datetime.fromisoformat(summary["updated_at"]).tzinfo is not None
def test_name_mismatch_is_rejected(
def test_a_save_neither_creates_nor_renames(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
response = client.put(
f"{PREFIX}/other", headers=superuser_token_headers, json=a_flow("demo")
put_flow(client, superuser_token_headers, a_flow(), "demo")
current = client.get(f"{PREFIX}/demo", headers=superuser_token_headers).json()
document = current["definition"]
missing = client.put(
f"{PREFIX}/nowhere", headers=superuser_token_headers, json=a_flow("nowhere")
)
assert response.status_code == 400
def test_a_flow_is_known_by_an_id_no_save_changes(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
"""For now the id of a new flow is its key, and the URL names both."""
saved = client.put(
f"{PREFIX}/identified",
assert missing.status_code == 404
renamed = client.put(
f"{PREFIX}/demo",
headers=superuser_token_headers,
json=a_flow("identified"),
).json()["definition"]
assert saved["id"] == "identified"
listed = client.get(f"{PREFIX}/", headers=superuser_token_headers).json()["data"]
assert {"id": "identified", "name": "identified"}.items() <= next(
f for f in listed if f["name"] == "identified"
).items()
json={**document, "name": "other"},
)
assert renamed.status_code == 409
elsewhere = client.put(
f"{PREFIX}/identified",
f"{PREFIX}/demo",
headers=superuser_token_headers,
json={**saved, "id": "elsewhere"},
json={**document, "id": "elsewhere"},
)
assert elsewhere.status_code == 400
# A key changes by a rename, and that route is not there yet.
renamed = client.post(
f"{PREFIX}/identified/rename",
headers=superuser_token_headers,
json={"new_name": "renamed"},
)
assert renamed.status_code in (404, 405)
# Shaped like an id the server issues, which a new key may not be.
issued = client.put(
f"{PREFIX}/f_k3q7m2xa",
headers=superuser_token_headers,
json=a_flow("f_k3q7m2xa"),
)
assert issued.status_code == 400
client.delete(f"{PREFIX}/identified", headers=superuser_token_headers)
def test_broken_node_is_reported_and_siblings_stay_active(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
put_flow(client, superuser_token_headers, a_flow(), "demo")
client.put(
f"{PREFIX}/demo/nodes/sensor/source",
headers=superuser_token_headers,
@@ -143,7 +119,7 @@ def test_broken_node_is_reported_and_siblings_stay_active(
def test_running_a_flow_produces_values(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
put_flow(client, superuser_token_headers, a_flow(), "demo")
client.put(
f"{PREFIX}/demo/nodes/sensor/source",
headers=superuser_token_headers,
@@ -165,9 +141,7 @@ def test_running_a_flow_produces_values(
def test_editing_does_not_deploy_until_published(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
saved = client.put(
f"{PREFIX}/staged", headers=superuser_token_headers, json=a_flow("staged")
).json()
saved = put_flow(client, superuser_token_headers, a_flow("staged"), "staged").json()
assert saved["has_draft"] is True
# Nothing is running yet, so the engine knows no nodes of this flow.
assert (
@@ -197,8 +171,8 @@ def test_editing_does_not_deploy_until_published(
def test_a_stale_save_is_refused(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
saved = client.put(
f"{PREFIX}/contested", headers=superuser_token_headers, json=a_flow("contested")
saved = put_flow(
client, superuser_token_headers, a_flow("contested"), "contested"
).json()
stale = saved["definition"]
@@ -222,8 +196,8 @@ def test_a_stale_save_is_refused(
def test_discarding_a_draft_restores_what_is_running(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
saved = client.put(
f"{PREFIX}/reverted", headers=superuser_token_headers, json=a_flow("reverted")
saved = put_flow(
client, superuser_token_headers, a_flow("reverted"), "reverted"
).json()
client.post(
f"{PREFIX}/reverted/publish",
@@ -232,10 +206,11 @@ def test_discarding_a_draft_restores_what_is_running(
)
published = client.get(f"{PREFIX}/reverted", headers=superuser_token_headers).json()
client.put(
f"{PREFIX}/reverted",
headers=superuser_token_headers,
json={**published["definition"], "title": "Scratch that"},
put_flow(
client,
superuser_token_headers,
{**published["definition"], "title": "Scratch that"},
"reverted",
)
response = client.post(
@@ -251,9 +226,7 @@ def test_discarding_a_draft_restores_what_is_running(
def test_stopping_a_flow_takes_it_off_the_engine(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
saved = client.put(
f"{PREFIX}/halted", headers=superuser_token_headers, json=a_flow("halted")
).json()
saved = put_flow(client, superuser_token_headers, a_flow("halted"), "halted").json()
client.post(
f"{PREFIX}/halted/publish",
headers=superuser_token_headers,
@@ -286,9 +259,7 @@ def test_stopping_a_flow_takes_it_off_the_engine(
def test_pausing_is_reported_back(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
saved = client.put(
f"{PREFIX}/held", headers=superuser_token_headers, json=a_flow("held")
).json()
saved = put_flow(client, superuser_token_headers, a_flow("held"), "held").json()
client.post(
f"{PREFIX}/held/publish",
headers=superuser_token_headers,
@@ -318,7 +289,7 @@ def test_unconnected_input_is_surfaced(
) -> None:
flow = a_flow()
flow["nodes"][0]["provides"] = [] # nothing produces "reading" any more
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=flow)
put_flow(client, superuser_token_headers, flow, "demo")
issues = client.post(
f"{PREFIX}/demo/validate", headers=superuser_token_headers
@@ -329,7 +300,7 @@ def test_unconnected_input_is_surfaced(
def test_delete_flow(
client: TestClient, superuser_token_headers: dict[str, str], db: Session
) -> None:
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
put_flow(client, superuser_token_headers, a_flow(), "demo")
# Finished: a flow with a live run refuses to be deleted, which is its own
# test below.
db.add(Run(id="run-1", flow="demo", status="ok", created_at=datetime.now(UTC)))
@@ -362,7 +333,7 @@ def test_delete_flow(
def test_delete_flow_is_refused_while_a_run_is_live(
client: TestClient, superuser_token_headers: dict[str, str], db: Session
) -> None:
client.put(f"{PREFIX}/demo", headers=superuser_token_headers, json=a_flow())
put_flow(client, superuser_token_headers, a_flow(), "demo")
db.add(
Run(id="run-live", flow="demo", status="running", created_at=datetime.now(UTC))
)
@@ -390,8 +361,8 @@ def test_a_node_that_raises_answers_with_its_error(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
"""A manual run fails the way a scheduled one does, not as a 500."""
saved = client.put(
f"{PREFIX}/failing", headers=superuser_token_headers, json=a_flow("failing")
saved = put_flow(
client, superuser_token_headers, a_flow("failing"), "failing"
).json()
client.put(
f"{PREFIX}/failing/nodes/sensor/source",
@@ -422,9 +393,7 @@ def test_acknowledging_a_failure_is_announced_on_the_bus(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Another browser holds the same marker; the event is what clears it."""
saved = client.put(
f"{PREFIX}/acked", headers=superuser_token_headers, json=a_flow("acked")
).json()
saved = put_flow(client, superuser_token_headers, a_flow("acked"), "acked").json()
client.post(
f"{PREFIX}/acked/publish",
headers=superuser_token_headers,
@@ -505,9 +474,7 @@ async def test_the_live_socket_ends_when_its_client_goes(
def test_saving_a_header_moves_the_node_ports_in_the_draft(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
saved = client.put(
f"{PREFIX}/header", headers=superuser_token_headers, json=a_flow("header")
).json()
saved = put_flow(client, superuser_token_headers, a_flow("header"), "header").json()
response = client.put(
f"{PREFIX}/header/nodes/logger/source",
@@ -539,9 +506,7 @@ def test_saving_a_header_moves_the_node_ports_in_the_draft(
def test_a_shared_node_in_use_is_retired_only_with_detach(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
client.put(
f"{PREFIX}/shared", headers=superuser_token_headers, json=a_flow("shared")
)
put_flow(client, superuser_token_headers, a_flow("shared"), "shared")
client.put(
f"{PREFIX}/shared/nodes/sensor/source",
headers=superuser_token_headers,
@@ -571,9 +536,7 @@ def test_a_shared_node_only_the_published_flow_uses_is_kept(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
"""An unshare in the draft leaves the running flow on the source."""
client.put(
f"{PREFIX}/unshared", headers=superuser_token_headers, json=a_flow("unshared")
)
put_flow(client, superuser_token_headers, a_flow("unshared"), "unshared")
shared = client.post(
f"{PREFIX}/unshared/nodes/sensor/share",
headers=superuser_token_headers,
@@ -603,9 +566,7 @@ def test_a_shared_source_write_is_audited(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""It goes live at once, so it is recorded like a publish."""
client.put(
f"{PREFIX}/audited", headers=superuser_token_headers, json=a_flow("audited")
)
put_flow(client, superuser_token_headers, a_flow("audited"), "audited")
client.post(
f"{PREFIX}/audited/nodes/sensor/share",
headers=superuser_token_headers,
@@ -96,10 +96,10 @@ def test_a_flow_that_cannot_run_makes_the_summary_degraded(
controller = client.app.state.flow_controller
before = controller.issues
before_flows = controller.store.list_flows
before_flows = controller.store.key_index
# The looping flow has to be one the store lists, otherwise "running"
# counts nothing either way and the assertion below proves nothing.
controller.store.list_flows = lambda: ["looping"]
controller.store.key_index = lambda: {"looping": "f_looping"}
controller.issues = [
ValidationIssue(
code="cycle",
@@ -119,7 +119,7 @@ def test_a_flow_that_cannot_run_makes_the_summary_degraded(
body = client.get(f"{PREFIX}/summary", headers=superuser_token_headers).json()
finally:
controller.issues = before
controller.store.list_flows = before_flows
controller.store.key_index = before_flows
assert body["status"] == "degraded"
assert body["flows"]["invalid"] == 1
+21 -16
View File
@@ -7,6 +7,7 @@ from fastapi.testclient import TestClient
from fluksio.api.routes.search import MASK, setting_text
from fluksio.core.config import settings
from tests.utils.flows import put_dashboard, put_flow
PREFIX = f"{settings.API_V1_STR}/search"
FLOWS = f"{settings.API_V1_STR}/flows"
@@ -22,19 +23,20 @@ def test_index_reaches_inside_flows_and_dashboards(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
"""A node and a widget are the point: neither is on any list endpoint."""
client.put(
f"{FLOWS}/searchable",
headers=superuser_token_headers,
json={
put_flow(
client,
superuser_token_headers,
{
"name": "searchable",
"title": "Searchable",
"nodes": [{"id": "sensor", "type": "python", "title": "Hall sensor"}],
},
"searchable",
)
client.put(
f"{DASHBOARDS}/hall",
headers=superuser_token_headers,
json={
put_dashboard(
client,
superuser_token_headers,
{
"name": "hall",
"title": "Hall",
"widgets": [
@@ -48,6 +50,7 @@ def test_index_reaches_inside_flows_and_dashboards(
],
"version": 0,
},
"hall",
)
entries = client.get(f"{PREFIX}/", headers=superuser_token_headers).json()
@@ -69,15 +72,16 @@ def test_index_lists_a_flows_inputs_and_outputs(
client: TestClient, superuser_token_headers: dict[str, str]
) -> None:
"""Named by the qualified message, which is the label the canvas draws."""
client.put(
f"{FLOWS}/fit",
headers=superuser_token_headers,
json={
put_flow(
client,
superuser_token_headers,
{
"name": "fit",
"mode": "batch",
"inputs": [{"spec": {"name": "epochs", "dtype": "int"}}],
"outputs": ["accuracy"],
},
"fit",
)
entries = client.get(f"{PREFIX}/", headers=superuser_token_headers).json()
@@ -151,10 +155,10 @@ def test_index_lists_a_nodes_settings(
normal_user_token_headers: dict[str, str],
) -> None:
"""A setting is found by its node and name, with its value beside it."""
client.put(
f"{FLOWS}/tuned",
headers=superuser_token_headers,
json={
put_flow(
client,
superuser_token_headers,
{
"name": "tuned",
"nodes": [
{"id": "gain", "type": "python", "params": {"factor": 2.5}},
@@ -169,6 +173,7 @@ def test_index_lists_a_nodes_settings(
},
],
},
"tuned",
)
def settings_of(headers: dict[str, str]) -> tuple[dict[str, dict], str]:
+25 -15
View File
@@ -38,6 +38,13 @@ def api(client: TestClient, superuser_token_headers: dict[str, str]) -> Client:
return Client(http=client, token=superuser_token_headers["Authorization"][7:])
def flow_id(api: Client, key: str) -> str:
"""The id sync stored the flow with key *key* under."""
found = api.find_flow(key)
assert found is not None, key
return str(found["id"])
def commits() -> int:
"""How many commits the flow store has made."""
from fluksio.flow.store import FlowStore
@@ -62,14 +69,14 @@ def test_sync_stores_a_runnable_flow_with_its_origin(api, flows):
assert [r.flow for r in reports] == ["train"]
assert reports[0].created and reports[0].published
stored = api.get_flow("train")
stored = api.get_flow(flow_id(api, "train"))
definition = stored["definition"]
assert definition["origin"]["commit"] == "a1b2c3d4"
assert [n["id"] for n in definition["nodes"]] == ["prepare", "fit", "evaluate"]
assert not stored["has_draft"]
# The store holds a complete definition, and the bodies say where the real
# code is rather than being a copy of it.
source = api.get_source("train", "fit")
source = api.get_source(definition["id"], "fit")
assert source.startswith(MARKER)
assert "from myresearch.train import fit" in source
@@ -79,7 +86,7 @@ def test_a_flow_read_by_another_process_syncs_as_the_flow_does(api, flows):
carried = Declared(flows["train"].declaration())
sync([carried], api, origin=ORIGIN)
stored = api.get_flow("train")["definition"]
stored = api.get_flow(flow_id(api, "train"))["definition"]
assert stored["origin"]["declared_in"] == "myresearch/pipeline.py"
# Nothing differs from what the flow itself would have stored.
@@ -90,7 +97,8 @@ def test_a_node_carries_the_modules_its_code_reaches(api, flows):
"""The shim names one function; the cache has to key on what it calls."""
sync([flows["train"]], api, origin=ORIGIN)
nodes = {n["id"]: n for n in api.get_flow("train")["definition"]["nodes"]}
definition = api.get_flow(flow_id(api, "train"))["definition"]
nodes = {n["id"]: n for n in definition["nodes"]}
files = nodes["fit"]["code_files"]
# The module the function is in, and the package whose `__init__` runs on
@@ -156,14 +164,15 @@ def test_a_second_sync_changes_nothing_and_commits_nothing(api, flows):
def test_sync_refuses_to_overwrite_a_canvas_edit(api, flows):
sync([flows["train"]], api, origin=ORIGIN)
api.put_source("train", "evaluate", "def process(weights):\n return {}\n")
train = flow_id(api, "train")
api.put_source(train, "evaluate", "def process(weights):\n return {}\n")
with pytest.raises(SyncError, match="edited on the canvas"):
sync([flows["train"]], api, origin=ORIGIN)
reports = sync([flows["train"]], api, origin=ORIGIN, force=True)
assert "evaluate" in reports[0].changed
assert api.get_source("train", "evaluate").startswith(MARKER)
assert api.get_source(train, "evaluate").startswith(MARKER)
def test_an_interrupted_sync_finishes_on_the_next_one(api, flows):
@@ -177,23 +186,23 @@ def test_an_interrupted_sync_finishes_on_the_next_one(api, flows):
target = flows["train"]
target.name = "resumed"
try:
# Exactly what the document PUT leaves behind before any body is written.
api.put_flow(target.document(ORIGIN) | {"version": 1})
assert api.get_source_entry("resumed", "fit")["missing"]
# Exactly what creating the document leaves before any body is written.
resumed = api.create_flow(target.document(ORIGIN))["definition"]["id"]
assert api.get_source_entry(resumed, "fit")["missing"]
reports = sync([target], api, origin=ORIGIN)
assert "fit" in reports[0].changed
assert api.get_source("resumed", "fit").startswith(MARKER)
assert not api.get_source_entry("resumed", "fit")["missing"]
assert not api.get_flow("resumed")["has_draft"]
assert api.get_source(resumed, "fit").startswith(MARKER)
assert not api.get_source_entry(resumed, "fit")["missing"]
assert not api.get_flow(resumed)["has_draft"]
finally:
target.name = "train"
def test_sync_refuses_a_flow_it_did_not_create(api, flows):
api.put_flow({"name": "drawn", "version": 1, "nodes": [], "mode": "batch"})
api.publish("drawn", 1)
drawn = api.create_flow({"name": "drawn", "nodes": [], "mode": "batch"})
api.publish(drawn["definition"]["id"], 1)
theirs = flows["train"]
theirs.name = "drawn"
@@ -207,7 +216,8 @@ def test_sync_refuses_a_flow_it_did_not_create(api, flows):
def test_use_stores_the_rewiring_it_was_given(api, flows):
sync([flows["finetune"]], api, origin=ORIGIN)
nodes = {n["id"]: n for n in api.get_flow("finetune")["definition"]["nodes"]}
definition = api.get_flow(flow_id(api, "finetune"))["definition"]
nodes = {n["id"]: n for n in definition["nodes"]}
assert [p["name"] for p in nodes["fit"]["requires"]] == ["augmented", "lr"]
assert nodes["fit"]["params"] == {"epochs": 3}
+3 -4
View File
@@ -16,6 +16,7 @@ from fluksio.core.config import settings
from fluksio.core.db import engine as db_engine
from fluksio.models import Run, RunArtifact
from tests.flow.conftest import FakeWren
from tests.utils.flows import put_flow
FLOW = "wren_house"
PREFIX = f"{settings.API_V1_STR}/flows/{FLOW}/nodes"
@@ -70,10 +71,8 @@ def house(
{"id": "sensor", "type": "python"},
]
url = f"{settings.API_V1_STR}/flows/{FLOW}"
saved = client.put(
url, headers=superuser_token_headers, json={"name": FLOW, "nodes": nodes}
)
assert saved.status_code == 200, saved.text
saved = put_flow(client, superuser_token_headers, {"name": FLOW, "nodes": nodes})
assert saved.is_success, saved.text
yield
client.delete(url, headers=superuser_token_headers)
+10 -8
View File
@@ -19,7 +19,8 @@ from fluksio.flow.nodes import Node
from fluksio.flow.pipeline import Pipeline
from fluksio.flow.schemas import FlowDef, NodeDef
from fluksio.flow.state import MemoryState
from fluksio.flow.store import FlowStore, StaleVersion
from fluksio.flow.store import ISSUED_ID, FlowStore, StaleVersion
from tests.utils.stores import created
@pytest.fixture
@@ -93,25 +94,25 @@ def test_discarding_leaves_what_is_published(store: DashboardStore):
def test_a_new_dashboard_is_a_draft_until_it_is_published(store: DashboardStore):
"""Creating one does not put it on a wall; publishing is what does."""
created = store.write_draft(default_dashboard("house"), 0)
made = created(store, default_dashboard("house"))
assert not store.is_published("house")
with pytest.raises(DashboardNotFound):
store.read("house")
assert store.read("house", draft=True).title == created.title
assert store.read("house", draft=True).title == made.title
# Listed all the same, so the editor can find what it just made.
assert [(d.name, d.has_draft, d.version) for d in store.list()] == [
("house", True, created.version)
("house", True, made.version)
]
store.publish("house", created.version)
store.publish("house", made.version)
assert store.is_published("house")
assert store.read("house").title == created.title
assert store.read("house").title == made.title
def test_an_unpublished_dashboard_can_be_deleted(store: DashboardStore):
store.write_draft(default_dashboard("house"), 0)
created(store, default_dashboard("house"))
store.delete("house")
assert not store.exists("house")
@@ -128,7 +129,8 @@ def test_a_duplicate_is_an_unpublished_copy_of_the_working_copy(
copy = store.duplicate("house")
assert (copy.name, copy.title, copy.version) == ("house_copy", "Edited (copy)", 1)
assert store.has_draft("house_copy") and not store.is_published("house_copy")
assert ISSUED_ID.fullmatch(copy.id)
assert store.has_draft(copy.id) and not store.is_published(copy.id)
assert store.read("house", draft=True).version == edited.version
assert store.duplicate("house").name == "house_copy_2"
+16 -13
View File
@@ -8,7 +8,8 @@ import pytest
from fluksio.flow.controller import FlowController
from fluksio.flow.messages import MessageSpec
from fluksio.flow.schemas import FlowDef, FlowOrigin, NodeDef
from fluksio.flow.store import FlowStore, StaleVersion
from fluksio.flow.store import ISSUED_ID, FlowStore, StaleVersion
from tests.utils.stores import created
SOURCE = "def process():\n return {}\n"
EDITED = "def process():\n return {'temp': 1}\n"
@@ -28,7 +29,7 @@ def a_flow(title: str = "") -> FlowDef:
def test_a_draft_is_invisible_to_the_engine(store: FlowStore):
store.write_draft(a_flow(), 0)
created(store, a_flow())
assert store.has_draft("heating")
assert store.list_flows() == ["heating"] # the editor sees it
@@ -36,7 +37,7 @@ def test_a_draft_is_invisible_to_the_engine(store: FlowStore):
def test_the_version_counter_advances_per_save(store: FlowStore):
first = store.write_draft(a_flow(), 0)
first = created(store, a_flow())
assert first.version == 1
second = store.write_draft(a_flow(title="Warm"), first.version)
@@ -44,14 +45,14 @@ def test_the_version_counter_advances_per_save(store: FlowStore):
def test_resaving_identical_content_neither_bumps_nor_commits(store: FlowStore):
stored = store.write_draft(a_flow(), 0)
stored = created(store, a_flow())
again = store.write_draft(a_flow(), stored.version)
assert again.version == stored.version
def test_a_stale_write_is_refused(store: FlowStore):
store.write_draft(a_flow(), 0)
created(store, a_flow())
store.write_draft(a_flow(title="Mine"), 1)
# A second client still holding version 1 must not overwrite version 2.
@@ -61,7 +62,7 @@ def test_a_stale_write_is_refused(store: FlowStore):
def test_publishing_promotes_the_draft_and_its_sources(store: FlowStore):
stored = store.write_draft(a_flow(title="Warm"), 0)
stored = created(store, a_flow(title="Warm"))
store.write_node_source("heating", "sensor", EDITED, draft=True)
published = store.publish_flow("heating", stored.version)
@@ -74,7 +75,7 @@ def test_publishing_promotes_the_draft_and_its_sources(store: FlowStore):
def test_publishing_a_stale_draft_is_refused(store: FlowStore):
store.write_draft(a_flow(), 0)
created(store, a_flow())
with pytest.raises(StaleVersion):
store.publish_flow("heating", 99)
@@ -150,7 +151,7 @@ def test_a_worker_on_a_node_without_code_is_reported(store: FlowStore):
def test_a_node_being_written_in_the_editor_is_not_reported(store: FlowStore):
"""A draft legitimately has no published body yet — that is what a draft is."""
store.write_draft(a_flow(), 0)
created(store, a_flow())
controller = FlowController(store)
assert controller.preview("heating").issues == []
@@ -175,12 +176,14 @@ def test_a_duplicate_is_a_local_draft_of_what_the_editor_shows(store: FlowStore)
copy = store.duplicate_flow("heating")
assert (copy.name, copy.title, copy.version) == ("heating_copy", "Warm (copy)", 1)
# An id of its own, which its key is not.
assert ISSUED_ID.fullmatch(copy.id)
assert copy.origin is None
assert copy.nodes[1].requires[0].name == "heating_copy.temp"
assert store.has_draft("heating_copy") and not store.is_published("heating_copy")
assert store.read_node_source("heating_copy", "sensor", draft=True) == EDITED
assert not (store.root / "heating_copy" / "runtime.json").exists()
assert store.has_draft(copy.id) and not store.is_published(copy.id)
assert store.read_node_source(copy.id, "sensor", draft=True) == EDITED
assert not (store.root / copy.id / "runtime.json").exists()
assert store.duplicate_flow("heating").name == "heating_copy_2"
store.publish_flow("heating_copy", copy.version)
assert store.read_node_source("heating_copy", "sensor") == EDITED
store.publish_flow(copy.id, copy.version)
assert store.read_node_source(copy.id, "sensor") == EDITED
+157
View File
@@ -0,0 +1,157 @@
"""The engine runs a flow by its key and finds it by its id.
Every test runs twice, once with the id equal to the key the way a flow from
before ids has it, and once with them apart. A mix-up of the two is invisible
in the first run, which is why the second one exists.
"""
import asyncio
from pathlib import Path
from types import SimpleNamespace
import pytest
from fluksio.flow.controller import HOOK_PREFIX, FlowController
from fluksio.flow.messages import DType, MessageSpec
from fluksio.flow.queue import MemoryWorkQueue, WorkItem
from fluksio.flow.schemas import FlowDef, NodeDef
from fluksio.flow.store import FlowStore
SENSOR = "def process():\n return {'reading': 21.5}\n"
DOUBLE = "def process(reading):\n return {'doubled': reading * 2}\n"
@pytest.fixture(params=["house", "x_house"], ids=["id=key", "id!=key"])
def flow_id(request: pytest.FixtureRequest) -> str:
return str(request.param)
@pytest.fixture
def store(tmp_path: Path, flow_id: str) -> FlowStore:
store = FlowStore(tmp_path / "flows")
store.write_flow(
FlowDef(
id=flow_id,
name="house",
nodes=[
NodeDef(
id="sensor",
provides=[MessageSpec(name="reading", dtype=DType.FLOAT)],
),
NodeDef(
id="double",
requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
provides=[MessageSpec(name="doubled", dtype=DType.FLOAT)],
),
NodeDef(
id="hook",
type="http",
params={"url": "/tick", "method": "POST"},
provides=[MessageSpec(name="tick", dtype=DType.FLOAT)],
),
],
)
)
store.write_node_source(flow_id, "sensor", SENSOR)
store.write_node_source(flow_id, "double", DOUBLE)
return store
def test_a_flow_runs_under_its_key_from_the_code_its_id_holds(
store: FlowStore, flow_id: str
):
engine = FlowController(store)
asyncio.run(engine.reload())
entry = engine.loaded["house.double"]
assert (entry.flow, entry.flow_id) == ("house", flow_id)
# Read by the id: by the key it would be the template, and missing.
assert not entry.missing_source and entry.status.value == "active"
assert engine.loaded["house.hook"].node.url == f"{HOOK_PREFIX}/{flow_id}/tick"
engine.run_flow("house")
assert engine.values()["house.doubled"]["value"] == 43.0
assert {s.id for s in engine.node_statuses("house")} >= {"house.double"}
def test_stopping_writes_under_the_id_and_gates_the_key(store: FlowStore, flow_id: str):
engine = FlowController(store)
async def scenario() -> None:
await engine.reload()
await engine.set_enabled(flow_id, "house", False)
assert engine.disabled == {"house"}
assert not store.read_enabled(flow_id)
# Read back by id when everything is built again.
await engine.reload()
assert engine.disabled == {"house"}
asyncio.run(scenario())
assert (store.root / flow_id / "runtime.json").exists()
def test_a_flow_rebuilt_by_id_is_taken_out_by_its_key_once_gone(
store: FlowStore, flow_id: str
):
engine = FlowController(store)
async def scenario() -> None:
await engine.reload()
assert "house.sensor" in engine.loaded
await engine.reload_flow(flow_id)
assert "house.sensor" in engine.loaded
store.delete_flow(flow_id)
await engine.reload_flow(flow_id)
asyncio.run(scenario())
assert not [node for node in engine.loaded if node.startswith("house.")]
assert engine.pipeline is not None and not engine.pipeline.flow_nodes("house")
def test_a_draft_previews_by_id_and_reports_by_key(store: FlowStore, flow_id: str):
draft = store.read_flow(flow_id).model_copy(update={"title": "Edited"})
store.write_draft(draft, draft.version)
engine = FlowController(store)
preview = engine.preview(flow_id)
assert {node.id for node in preview.nodes} == {
"house.sensor",
"house.double",
"house.hook",
}
assert all(node.status == "active" for node in preview.nodes)
engine.run_preview(flow_id)
assert engine.state.get("house.doubled") == 43.0
def test_what_a_key_leaves_behind_is_found_and_forgotten(store: FlowStore):
engine = FlowController(store)
queue = MemoryWorkQueue()
engine.execution = SimpleNamespace(queue=queue) # type: ignore[assignment]
engine.state.set("house.reading", 1.0)
engine.state.set("__node__:house.double:last", 2)
engine.state.set("housework.reading", 3.0)
queue.add_delayed(WorkItem(kind="flush", node="house.double", flow="house"), 9e9)
queue.park("house", WorkItem(kind="cascade", node="house.sensor", flow="house"))
# Another flow's work, carrying one of this key's messages.
queue.add(
WorkItem(
kind="cascade", node="garden.pump", flow="garden", outputs={"house.x": 1}
)
)
found = engine.key_residue("house")
assert "house.reading" in found and "__node__:house.double:last" in found
assert "housework.reading" not in found
assert sum("queued work" in what for what in found) == 3
assert engine.key_residue("garden") == ["queued work of 'garden.pump'"]
engine.forget_flow("house")
# What the other flow queued is its own, and runs as it would have.
assert engine.key_residue("house") == ["queued work of 'garden.pump'"]
assert engine.state.get("housework.reading") == 3.0
+244 -36
View File
@@ -8,6 +8,7 @@ is why the migration is driven through crashes, refusals and stores without git.
import json
import logging
import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import pytest
@@ -18,17 +19,23 @@ from fluksio.flow.dashboards import (
DASHBOARD_DIR,
DashboardDef,
DashboardExists,
DashboardNotFound,
DashboardStore,
default_dashboard,
results_dashboard,
)
from fluksio.flow.identity import COMMIT_MESSAGE, IdsRefused, assign_ids
from fluksio.flow.schemas import FlowDef
from fluksio.flow.schemas import FlowCreate, FlowDef
from fluksio.flow.store import (
ISSUED_ID,
FlowExists,
FlowNotFound,
FlowStore,
IdentityChange,
IdTaken,
RecordedKey,
ReservedKey,
StaleVersion,
new_id,
)
@@ -37,23 +44,51 @@ from fluksio.flow.store import (
# -----------------------------------------------------------------------------
class Graves:
"""Tombstones and history, as `fluksio.flow.tombstones.Tombstones` keeps
them in the database, which store tests have none of."""
def __init__(self) -> None:
self.stones: dict[str, str] = {}
#: Ids that history rows name.
self.history: set[str] = set()
def bury(self, flow_id: str, name: str) -> None:
self.stones[flow_id] = name
def lift(self, flow_id: str) -> None:
self.stones.pop(flow_id, None)
def recorded(self, flow_id: str) -> str | None:
if flow_id in self.stones:
return self.stones[flow_id]
return flow_id if flow_id in self.history else None
@pytest.fixture
def store(tmp_path: Path) -> FlowStore:
return FlowStore(tmp_path / "flows")
def graves() -> Graves:
return Graves()
def test_a_flow_is_given_its_key_as_id_and_keeps_both(store: FlowStore):
created = store.write_draft(FlowDef(name="heating"), 0)
assert (created.id, created.name) == ("heating", "heating")
@pytest.fixture
def store(tmp_path: Path, graves: Graves) -> FlowStore:
return FlowStore(tmp_path / "flows", tombstones=graves) # type: ignore[arg-type]
published = store.publish_flow("heating", created.version)
def test_a_new_flow_gets_an_issued_id_and_keeps_it(store: FlowStore):
created = store.create(FlowDef(name="heating"))
flow_id = created.id
assert ISSUED_ID.fullmatch(flow_id) and flow_id.startswith("f_")
assert created.name == "heating" and created.version == 1
assert store.key_index() == {"heating": flow_id}
published = store.publish_flow(flow_id, created.version)
store.write_draft(published.model_copy(update={"title": "Warm"}), 1)
discarded = store.discard_draft("heating")
discarded = store.discard_draft(flow_id)
assert json.loads((store.root / "heating" / "flow.json").read_text())["id"] == (
"heating"
)
assert (discarded.id, discarded.name) == ("heating", "heating")
on_disk = json.loads((store.root / flow_id / "flow.json").read_text())
assert (on_disk["id"], on_disk["name"]) == (flow_id, "heating")
assert (discarded.id, discarded.name) == (flow_id, "heating")
def test_a_document_from_before_ids_reads_with_its_directory_as_id(store: FlowStore):
@@ -70,9 +105,13 @@ def test_a_document_from_before_ids_reads_with_its_directory_as_id(store: FlowSt
assert store.write_draft(FlowDef(name="heating", title="Warm"), 1).version == 2
def test_a_save_cannot_change_an_id_or_a_key(store: FlowStore):
store.write_flow(FlowDef(id="x_heating", name="heating"))
def test_a_save_neither_creates_nor_changes_an_id_or_a_key(store: FlowStore):
with pytest.raises(FlowNotFound):
store.write_draft(FlowDef(name="nobody"), 0)
with pytest.raises(DashboardNotFound):
DashboardStore(store).write_draft(default_dashboard("nobody"), 0)
store.write_flow(FlowDef(id="x_heating", name="heating"))
with pytest.raises(IdentityChange):
store.write_draft(FlowDef(id="x_heating", name="warmth"), 1)
with pytest.raises(IdentityChange):
@@ -87,56 +126,218 @@ def test_a_save_cannot_change_an_id_or_a_key(store: FlowStore):
def test_a_key_belongs_to_one_flow_drafts_included(store: FlowStore):
"""A draft nobody published holds its key, and a delete lets it go."""
store.write_draft(FlowDef(id="x_heating", name="heating"), 0)
assert store.key_index() == {"heating": "x_heating"}
first = store.create(FlowDef(name="heating"))
assert store.key_index() == {"heating": first.id}
with pytest.raises(FlowExists):
store.write_draft(FlowDef(name="heating"), 0)
store.create(FlowDef(name="heating"))
with pytest.raises(FlowExists):
store.create(FlowDef(name="heating"), "x_heating")
with pytest.raises(FlowExists):
store.write_flow(FlowDef(id="y_heating", name="heating"))
store.delete_flow("x_heating")
assert store.write_draft(FlowDef(name="heating"), 0).id == "heating"
assert store.key_index() == {"heating": "heating"}
store.delete_flow(first.id)
second = store.create(FlowDef(name="heating"))
assert second.id != first.id
assert store.key_index() == {"heating": second.id}
def test_a_key_belongs_to_one_dashboard_drafts_included(store: FlowStore):
dashboards = DashboardStore(store)
dashboards.write_draft(DashboardDef(id="x_home", name="home"), 0)
first = dashboards.create(default_dashboard("home"))
assert ISSUED_ID.fullmatch(first.id) and first.id.startswith("d_")
# Its own id space: a flow may share the key.
store.write_draft(FlowDef(name="home"), 0)
store.create(FlowDef(name="home"))
with pytest.raises(DashboardExists):
dashboards.write_draft(default_dashboard("home"), 0)
dashboards.create(default_dashboard("home"))
with pytest.raises(DashboardExists):
dashboards.write(DashboardDef(id="y_home", name="home"))
dashboards.delete("x_home")
assert dashboards.write(default_dashboard("home")).id == "home"
dashboards.delete(first.id)
assert dashboards.create(default_dashboard("home"), "home").id == "home"
def test_a_copy_takes_a_key_nobody_holds(store: FlowStore):
def test_a_pin_is_an_id_nothing_holds(store: FlowStore):
store.create(FlowDef(name="heating"), "x_heating")
with pytest.raises(IdTaken):
store.create(FlowDef(name="cooling"), "x_heating")
# Another flow's key would make naming either ambiguous.
with pytest.raises(IdTaken):
store.create(FlowDef(name="cooling"), "heating")
# And a directory left behind would hand the new flow what is in it.
(store.root / "leftover" / "nodes").mkdir(parents=True)
with pytest.raises(IdTaken):
store.create(FlowDef(name="leftover"), "leftover")
dashboards = DashboardStore(store)
dashboards.create(default_dashboard("home"), "x_home")
with pytest.raises(IdTaken):
dashboards.create(default_dashboard("house"), "x_home")
def test_an_issued_id_avoids_whatever_history_or_a_directory_holds(
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
):
candidates = iter(["f_aaaaaaaa", "f_bbbbbbbb", "f_cccccccc", "f_dddddddd"])
monkeypatch.setattr(
"fluksio.flow.store.new_id",
lambda kind, taken: next(c for c in candidates if not taken(c)),
)
graves.bury("f_aaaaaaaa", "gone")
graves.history.add("f_bbbbbbbb")
(store.root / "f_cccccccc").mkdir()
assert store.create(FlowDef(name="heating")).id == "f_dddddddd"
def test_a_deleted_flow_is_buried_before_its_files_go(
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
):
flow_id = store.create(FlowDef(name="heating")).id
def crash(path: Path) -> None:
raise OSError("disk gone")
monkeypatch.setattr("fluksio.flow.store.shutil.rmtree", crash)
with pytest.raises(OSError):
store.delete_flow(flow_id)
# The name is safe, and the live document still wins over it.
assert graves.stones == {flow_id: "heating"}
assert store.read_flow(flow_id, draft=True).name == "heating"
monkeypatch.undo()
assert store.delete_flow(flow_id) == "heating"
assert graves.stones == {flow_id: "heating"}
assert not store.exists(flow_id)
def test_a_delete_that_stops_after_the_files_has_kept_the_name(
store: FlowStore, graves: Graves, monkeypatch: pytest.MonkeyPatch
):
flow_id = store.create(FlowDef(name="heating")).id
def killed(message: str) -> None:
raise OSError("killed before the commit")
monkeypatch.setattr(store, "_commit", killed)
with pytest.raises(OSError):
store.delete_flow(flow_id)
monkeypatch.undo()
assert not store.exists(flow_id)
assert graves.stones == {flow_id: "heating"}
with pytest.raises(FlowNotFound):
store.delete_flow(flow_id)
def test_a_pinned_restore_takes_up_its_history_under_its_own_key(
store: FlowStore, graves: Graves
):
flow_id = store.create(FlowDef(name="heating")).id
store.delete_flow(flow_id)
with pytest.raises(RecordedKey):
store.create(FlowDef(name="warmth"), flow_id)
assert graves.stones == {flow_id: "heating"}
restored = store.create(FlowDef(name="heating"), flow_id)
assert restored.id == flow_id and graves.stones == {}
# History from before tombstones is filed under an id that was its key.
graves.history.add("legacy")
with pytest.raises(RecordedKey):
store.create(FlowDef(name="other"), "legacy")
assert store.create(FlowDef(name="legacy"), "legacy").id == "legacy"
def test_a_key_reused_after_a_delete_starts_a_new_history(
store: FlowStore, graves: Graves
):
old = store.create(FlowDef(name="heating")).id
store.delete_flow(old)
new = store.create(FlowDef(name="heating")).id
assert new != old
assert graves.stones == {old: "heating"}
def test_two_creates_of_one_key_make_one_flow(store: FlowStore):
def create(_: int) -> str:
try:
return store.create(FlowDef(name="heating")).id
except FlowExists:
return ""
with ThreadPoolExecutor(max_workers=8) as pool:
made = [flow_id for flow_id in pool.map(create, range(8)) if flow_id]
assert len(made) == 1
assert store.list_flows() == made
def test_two_results_dashboards_of_one_flow_make_one(store: FlowStore):
dashboards = DashboardStore(store)
flow = FlowDef(id="x_lab", name="lab", mode="batch")
def generate(_: int) -> str:
try:
return dashboards.create(results_dashboard(flow), "", True).id
except DashboardExists:
return ""
with ThreadPoolExecutor(max_workers=4) as pool:
made = [board for board in pool.map(generate, range(4)) if board]
assert len(made) == 1
assert dashboards.read(made[0]).name == "lab_results"
def test_a_discard_leaves_no_old_draft_version_to_match(store: FlowStore):
flow_id = store.create(FlowDef(name="heating")).id
published = store.publish_flow(flow_id, 1)
held = store.write_draft(published.model_copy(update={"title": "A"}), 1)
store.write_draft(held.model_copy(update={"title": "B"}), held.version)
restored = store.discard_draft(flow_id)
# Above every draft version handed out, so an editor holding one is stale.
assert restored.version == 4 and restored.name == "heating"
with pytest.raises(StaleVersion):
store.write_draft(held.model_copy(update={"title": "late"}), held.version)
assert store.write_draft(restored.model_copy(update={"title": "C"}), 4).version == 5
dashboards = DashboardStore(store)
board = dashboards.create(default_dashboard("home"))
dashboards.publish(board.id, 1)
dashboards.write_draft(dashboards.read(board.id).model_copy(), 1)
assert dashboards.discard_draft(board.id).version == 3
# A direct write goes above a draft too.
dashboards.write_draft(dashboards.read(board.id).model_copy(), 3)
assert dashboards.write(dashboards.read(board.id)).version == 5
def test_a_copy_gets_an_id_of_its_own_and_a_key_nobody_holds(store: FlowStore):
store.write_flow(FlowDef(name="heating"))
store.write_draft(FlowDef(id="x_other", name="heating_copy"), 0)
store.create(FlowDef(name="heating_copy"))
dashboards = DashboardStore(store)
dashboards.write(default_dashboard("home"))
dashboards.write_draft(DashboardDef(id="x_other", name="home_copy"), 0)
dashboards.create(default_dashboard("home_copy"))
copy = store.duplicate_flow("heating")
copy = store.duplicate_flow("heating", busy=lambda key: key == "heating_copy_2")
board = dashboards.duplicate("home")
assert (copy.id, copy.name) == ("heating_copy_2", "heating_copy_2")
assert (board.id, board.name) == ("home_copy_2", "home_copy_2")
# `_copy_2` still had work behind it, so the copy went past it.
assert copy.name == "heating_copy_3" and ISSUED_ID.fullmatch(copy.id)
assert board.name == "home_copy_2" and ISSUED_ID.fullmatch(board.id)
def test_a_new_key_shaped_like_an_issued_id_is_refused(store: FlowStore):
with pytest.raises(ReservedKey):
store.write_draft(FlowDef(name="f_k3q7m2xa"), 0)
store.create(FlowDef(name="f_k3q7m2xa"))
with pytest.raises(ReservedKey):
DashboardStore(store).write_draft(default_dashboard("d_k3q7m2xa"), 0)
assert store.write_draft(FlowDef(name="f_k3q7m2xa_lab"), 0).name == (
"f_k3q7m2xa_lab"
)
DashboardStore(store).create(default_dashboard("d_k3q7m2xa"))
assert store.create(FlowDef(name="f_k3q7m2xa_lab")).name == "f_k3q7m2xa_lab"
def test_an_existing_key_shaped_like_an_issued_id_stays_saveable(store: FlowStore):
@@ -515,3 +716,10 @@ def test_ls_lists_ids_keys_and_kinds(tmp_path: Path, capsys):
assert rows[0] == ["ID", "KEY", "KIND"]
assert ["home", "home", "dashboard"] in rows and ["home", "home", "flow"] in rows
assert ["warmth", "radiators", "flow"] in rows
def test_what_a_create_takes_is_a_document_but_for_its_version():
"""`FlowCreate` lists the fields again; one added to `FlowDef` belongs there."""
assert set(FlowCreate.model_fields) == set(FlowDef.model_fields) - {"version"}
with pytest.raises(ValueError):
FlowCreate(name="Not A Key")
+3 -2
View File
@@ -9,6 +9,7 @@ from fluksio.flow.controller import FlowController
from fluksio.flow.messages import MessageSpec
from fluksio.flow.schemas import FlowDef, NodeDef
from fluksio.flow.store import FlowStore, LibExists, LibNotFound
from tests.utils.stores import created
SOURCE = "def process():\n return {'temp': 1}\n"
@@ -79,7 +80,7 @@ def test_a_shared_source_change_rebuilds_only_what_is_published(store: FlowStore
# Shared in a draft that was never published.
cooling = a_flow("cooling")
cooling.nodes[0].source_ref = "read_temp"
store.write_draft(cooling, 0)
created(store, cooling)
controller = FlowController(store)
rebuilt: list[str] = []
@@ -131,7 +132,7 @@ def test_retiring_a_shared_source_leaves_each_user_a_copy(store: FlowStore):
# Shared in a draft that was never published.
cooling = a_flow("cooling")
cooling.nodes[0].source_ref = "read_temp"
store.write_draft(cooling, 0)
created(store, cooling)
store.detach_lib("read_temp")
+3 -2
View File
@@ -15,6 +15,7 @@ from fluksio.flow.pipeline import Pipeline
from fluksio.flow.schemas import FlowDef, NodeDef
from fluksio.flow.state import MemoryState
from fluksio.flow.store import FlowStore
from tests.utils.stores import created
SOURCE = "def process(reading, factor):\n return {'scaled': reading * factor}\n"
@@ -53,7 +54,7 @@ def test_a_setting_reaches_the_function_as_a_keyword_argument(store: FlowStore):
def test_a_setting_named_after_a_port_is_refused(store: FlowStore):
store.write_draft(a_flow(reading=3), 0)
created(store, a_flow(reading=3))
store.write_node_source("house", "scale", SOURCE, draft=True)
controller = FlowController(store)
@@ -81,7 +82,7 @@ READING = [MessageSpec(name="reading", dtype=DType.FLOAT)]
def _preview(store: FlowStore, flow: FlowDef, source: str | None):
store.write_draft(flow, 0)
created(store, flow)
if source is not None:
store.write_node_source("house", "scale", source, draft=True)
return FlowController(store).preview("house").nodes[0]
+25
View File
@@ -680,3 +680,28 @@ def test_done_markers_do_not_grow_without_bound():
# The newest are what redelivery would ask about.
assert queue.was_done(f"entry-{queue_module.DONE_MARKERS + 499}", "f.node")
assert not queue.was_done("entry-0", "f.node")
def test_clearing_a_flow_drops_its_work_and_nothing_else():
"""A new flow under the same key must never run what the old one queued."""
queue = MemoryWorkQueue()
queue.add(WorkItem(kind="cascade", node="gone.a", flow="gone"))
# Another flow's work carrying one of its messages, and a run, which is
# filed under a flow's id rather than its key: neither is the flow's.
queue.add(
WorkItem(kind="cascade", node="kept.a", flow="kept", outputs={"gone.x": 1})
)
queue.add(WorkItem(kind="run", node="", flow="gone", run_id="r1"))
queue.add_delayed(WorkItem(kind="flush", node="gone.b", flow="gone"), 9e9)
queue.add_delayed(WorkItem(kind="flush", node="kept.b", flow="kept"), 9e9)
queue.park("gone", WorkItem(kind="cascade", node="gone.c", flow="gone"))
assert len(queue.live_items()) == 6
queue.clear_flow("gone")
assert sorted((i.kind, i.node) for i in queue.live_items()) == [
("cascade", "kept.a"),
("flush", "kept.b"),
("run", ""),
]
assert queue.next_due() == 9e9
+27
View File
@@ -66,3 +66,30 @@ def test_a_refused_stream_write_keeps_the_timer_for_retry(queue):
assert queue.move_due(200.0) == 1
assert queue.move_due(200.0) == 0
assert [item.node for item in queue.claim(10, 10, due_only=True)] == ["f.stop"]
def test_clearing_a_flow_drops_its_queued_claimed_and_held_work(queue):
queue.add(WorkItem(kind="cascade", node="gone.p", flow="gone"))
(claimed,) = queue.claim(1, 10)
assert claimed.node == "gone.p"
queue.add(WorkItem(kind="cascade", node="gone.a", flow="gone"))
queue.add(
WorkItem(kind="cascade", node="kept.a", flow="kept", outputs={"gone.x": 1})
)
queue.add(WorkItem(kind="run", node="", flow="gone", run_id="r1"))
queue.add_delayed(WorkItem(kind="flush", node="gone.b", flow="gone"), 9e9)
queue.add_delayed(WorkItem(kind="flush", node="kept.b", flow="kept"), 9e9)
queue.park("gone", WorkItem(kind="cascade", node="gone.c", flow="gone"))
assert len(queue.live_items()) == 7
queue.clear_flow("gone")
assert sorted((i.kind, i.node) for i in queue.live_items()) == [
("cascade", "kept.a"),
("flush", "kept.b"),
("run", ""),
]
# Nothing comes back to be run again: not redelivered, not reclaimed.
assert queue.reclaim_stale(0) == []
queue.ack(claimed)
assert [i.node for i in queue.claim(10, 10)] == ["kept.a", ""]
+3 -3
View File
@@ -318,7 +318,7 @@ def test_stopping_one_flow_does_not_touch_the_others(
before = {name: engine.loaded[name].node for name in ("a.io", "b.io")}
lifecycle.clear()
await engine.set_enabled("a", False)
await engine.set_enabled("a", "a", False)
assert lifecycle == ["stop a.io"]
# Still built, still the same objects: only the lifecycle changed.
@@ -326,14 +326,14 @@ def test_stopping_one_flow_does_not_touch_the_others(
assert engine.disabled == {"a"}
lifecycle.clear()
await engine.set_enabled("a", True)
await engine.set_enabled("a", "a", True)
assert lifecycle == ["start a.io"]
assert {name: engine.loaded[name].node for name in ("a.io", "b.io")} == before
# Started while running is a restart: the cancelled loops come back.
lifecycle.clear()
await engine.set_enabled("a", True)
await engine.set_enabled("a", "a", True)
assert lifecycle == ["stop a.io", "start a.io"]
+2 -3
View File
@@ -15,6 +15,7 @@ from fluksio.flow.schemas import FlowDef, FlowInput, NodeDef
from fluksio.flow.state import MemoryState
from fluksio.flow.store import FlowStore
from fluksio.models import Run, RunArtifact
from tests.utils.stores import created
@pytest.fixture
@@ -126,9 +127,7 @@ def test_a_sweep_keeps_what_a_flow_input_starts_with(store, tmp_path):
flows.write_flow(
FlowDef(name="f", inputs=[FlowInput(spec=clip, initial=published)])
)
flows.write_draft(
FlowDef(name="g", inputs=[FlowInput(spec=clip, initial=drafted)]), 0
)
created(flows, FlowDef(name="g", inputs=[FlowInput(spec=clip, initial=drafted)]))
assert sweep_artifacts(store, MemoryState(), grace_s=0, flows=flows) == 1
assert store.path(orphan["digest"]) is None
+3 -1
View File
@@ -275,7 +275,9 @@ def test_a_panel_scoped_portal_token_reaches_only_its_panel(
assert saved.status_code == 200, saved.text
for name in ("kitchen", "bench"):
created = client.post(
f"{settings.API_V1_STR}/dashboards/{name}", headers=superuser_token_headers
f"{settings.API_V1_STR}/dashboards/",
headers=superuser_token_headers,
json={"id": name, "name": name},
)
assert created.status_code in (200, 201, 409), created.text
# A new dashboard is a draft, and a panel only reaches what is
+65
View File
@@ -0,0 +1,65 @@
"""A deleted flow's last key, kept for as long as history names its id."""
from datetime import UTC, datetime
from pathlib import Path
from sqlmodel import Session, delete, select
from fluksio.flow import tombstones
from fluksio.flow.schemas import FlowDef
from fluksio.flow.store import FlowStore
from fluksio.flow.tombstones import Tombstones, names_of
from fluksio.models import EngineEvent, FlowTombstone, MetricBucket, Run
def test_a_tombstone_stays_while_history_names_its_id(db: Session) -> None:
graves = Tombstones()
for flow_id in ("f_gonegone", "f_trainee", "f_measured", "f_forgotten"):
graves.bury(flow_id, flow_id.removeprefix("f_"))
now = datetime.now(UTC)
# A training run is filed under the training flow, and names the node it
# trains in another, by that flow's id.
db.add(
Run(
id="tomb-run",
flow="wren_train",
for_node="f_trainee.lights",
created_at=now,
)
)
db.add(Run(id="tomb-own", flow="f_gonegone", created_at=now))
db.add(MetricBucket(flow="other", node="f_measured.sensor", bucket=now))
db.add(EngineEvent(ts=now, type="audit", flow="f_forgotten_too"))
db.commit()
assert graves.recorded("f_trainee") == "trainee"
assert graves.recorded("wren_train") == "wren_train" # history, no tombstone
assert graves.recorded("f_nothing") is None
assert tombstones.prune(db) == 1
db.commit()
kept = {stone.id for stone in db.exec(select(FlowTombstone))}
assert kept == {"f_gonegone", "f_trainee", "f_measured"}
db.exec(delete(Run).where(Run.id.in_(["tomb-run", "tomb-own"]))) # type: ignore[attr-defined,call-overload]
db.exec(delete(MetricBucket)) # type: ignore[call-overload]
db.exec(delete(EngineEvent)) # type: ignore[call-overload]
db.commit()
assert tombstones.prune(db) == 3
db.commit()
def test_a_name_comes_from_the_live_flow_then_its_tombstone_then_the_id(
db: Session, tmp_path: Path
) -> None:
store = FlowStore(tmp_path / "flows")
store.write_flow(FlowDef(id="f_livelive", name="heating"))
Tombstones().bury("f_deadbeef", "cooling")
assert names_of(["f_livelive", "f_deadbeef", "old_key"], store) == {
"f_livelive": "heating",
"f_deadbeef": "cooling",
"old_key": "old_key",
}
Tombstones().lift("f_deadbeef")
assert db.get(FlowTombstone, "f_deadbeef") is None
+66
View File
@@ -0,0 +1,66 @@
"""Flows and dashboards over HTTP, the way most tests want them there."""
from typing import Any
import httpx
from fastapi.testclient import TestClient
from fluksio.core.config import settings
FLOWS = f"{settings.API_V1_STR}/flows"
DASHBOARDS = f"{settings.API_V1_STR}/dashboards"
def put_flow(
client: TestClient,
headers: dict[str, str],
definition: dict[str, Any],
flow_id: str = "",
) -> httpx.Response:
"""Create the flow under *flow_id*, its key unless given, or save over it.
Creating pins the id, so the test can address the flow by what it knows,
which needs a superuser's *headers*. Saving goes from whatever version is
stored. What a test meant by ``PUT`` before ``PUT`` stopped creating.
"""
flow_id = flow_id or definition["name"]
current = client.get(f"{FLOWS}/{flow_id}", headers=headers)
if current.status_code == 404:
return client.post(
f"{FLOWS}/", headers=headers, json={**definition, "id": flow_id}
)
version = current.json()["definition"]["version"]
return client.put(
f"{FLOWS}/{flow_id}",
headers=headers,
json={**definition, "id": flow_id, "version": version},
)
def put_dashboard(
client: TestClient,
headers: dict[str, str],
document: dict[str, Any],
dashboard_id: str = "",
) -> httpx.Response:
"""Create the dashboard under *dashboard_id*, its key unless given, and
save *document* over it, as `put_flow` does for a flow."""
dashboard_id = dashboard_id or document["name"]
current = client.get(f"{DASHBOARDS}/{dashboard_id}?draft=true", headers=headers)
if current.status_code == 404:
created = client.post(
f"{DASHBOARDS}/",
headers=headers,
json={"id": dashboard_id, "name": document["name"]},
)
assert created.status_code == 201, created.text
current = created
return client.put(
f"{DASHBOARDS}/{dashboard_id}",
headers=headers,
json={
**document,
"id": dashboard_id,
"version": current.json()["version"],
},
)
+12
View File
@@ -0,0 +1,12 @@
"""Store helpers for tests that work below the API."""
from typing import Any
def created(store: Any, document: Any) -> Any:
"""*document* created as a draft, under its key as its id.
How a store test made one before ids were issued: the id it addresses the
object by is the name it gave. Works for flows and dashboards alike.
"""
return store.create(document, document.name)