Merge remote-tracking branch 'origin/main' into HEAD

backend/tests/api/routes/test_flow_ids.py

@@ -20,6 +20,7 @@ from fluksio.flow.dashboards import DashboardDef, WidgetDef
 from fluksio.flow.store import ISSUED_ID
 from fluksio.flow.tombstones import names_of
 from fluksio.models import FlowTombstone, Run
+from tests.api.routes.test_rename import _until
 
 FLOWS = f"{settings.API_V1_STR}/flows"
 DASHBOARDS = f"{settings.API_V1_STR}/dashboards"
@@ -180,8 +181,14 @@ def test_a_flow_runs_under_its_key_and_is_reached_by_its_id(
     hooked = client.post(f"/hooks/{flow_id}/tick", json={"tick": 7.0})
     assert hooked.status_code == 200, hooked.text
     assert client.post(f"/hooks/{key}/tick", json={"tick": 8.0}).status_code == 404
-    state = client.get(f"{FLOWS}/{flow_id}/state", headers=superuser_token_headers)
-    assert state.json()["values"][f"{key}.tick"]["value"] == 7.0
+
+    def values() -> dict[str, Any]:
+        state = client.get(f"{FLOWS}/{flow_id}/state", headers=superuser_token_headers)
+        return dict(state.json()["values"])
+
+    # Once the cascade it queued has run.
+    _until(lambda: f"{key}.tick" in values())
+    assert values()[f"{key}.tick"]["value"] == 7.0
 
     paused = client.post(f"{FLOWS}/{flow_id}/pause", headers=superuser_token_headers)
     assert paused.status_code == 200

backend/tests/api/routes/test_runs.py

@@ -19,7 +19,7 @@ from fluksio.core.db import engine as db_engine
 from fluksio.flow.artifacts import ArtifactStore
 from fluksio.flow.controller import FlowController
 from fluksio.flow.messages import DType, MessageSpec
-from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, NodeOutcome
+from fluksio.flow.pipeline import CACHE_SHAPE_SINCE, NodeOutcome, Pipeline
 from fluksio.flow.runs import (
     LEASE_STALE_S,
     OUTPUT_CAP,
@@ -729,6 +729,105 @@ def test_a_draft_run_executes_the_draft(tmp_path):
             session.commit()
 
 
+def test_a_failed_stage_skips_what_waits_on_it_and_fails_the_run(tmp_path):
+    """Stages after a failure were called without their input and raised
+    TypeError, and a run whose only node could not load finished `ok`."""
+    model, score = (MessageSpec(name=n, dtype=DType.FLOAT) for n in ("model", "score"))
+    staged = FlowDef(
+        name="staged",
+        mode="batch",
+        nodes=[
+            NodeDef(id="fit", provides=[model]),
+            NodeDef(id="readout", requires=[model], provides=[score]),
+            NodeDef(id="evaluate", requires=[score]),
+        ],
+    )
+    broken = FlowDef(name="broken", mode="batch", nodes=[NodeDef(id="fit")])
+    store = FlowStore(tmp_path / "flows")
+    store.write_flow(staged)
+    store.write_flow(broken)
+    store.write_node_source("staged", "fit", "def process():\n    raise ValueError\n")
+    store.write_node_source(
+        "staged", "readout", "def process(model):\n    return {'score': model}\n"
+    )
+    store.write_node_source("staged", "evaluate", "def process(score):\n    pass\n")
+    store.write_node_source("broken", "fit", "def process(:\n")
+
+    service = RunService(controller=FlowController(store), queue=_Collect())
+    made = [service.submit(name, {}).id for name in ("staged", "broken")]
+    try:
+        for run_id in made:
+            service._drive(run_id)
+        with Session(db_engine) as session:
+            ran, failed = (session.get(Run, run_id) for run_id in made)
+            assert ran.status == failed.status == "error"
+            assert failed.status_reason.startswith("Node 'broken.fit' could not")
+            rows = session.exec(select(RunNode).where(col(RunNode.run_id).in_(made)))
+            assert {row.node: row.status for row in rows} == {
+                "staged.fit": "error",
+                "staged.readout": "skipped",
+                "staged.evaluate": "skipped",
+                "broken.fit": "error",
+            }
+    finally:
+        with Session(db_engine) as session:
+            for node in session.exec(
+                select(RunNode).where(col(RunNode.run_id).in_(made))
+            ).all():
+                session.delete(node)
+            for run in session.exec(select(Run).where(col(Run.id).in_(made))).all():
+                session.delete(run)
+            session.commit()
+
+
+def test_a_stage_with_nothing_on_an_input_fails_instead_of_being_called(tmp_path):
+    """The producer succeeded but published nothing on the port it waits on."""
+    model, score, report = (
+        MessageSpec(name=n, dtype=DType.FLOAT) for n in ("model", "score", "report")
+    )
+    flow = FlowDef(
+        name="split",
+        mode="batch",
+        nodes=[
+            NodeDef(id="fit", provides=[model, score]),
+            NodeDef(id="readout", requires=[score], provides=[report]),
+            NodeDef(id="evaluate", requires=[report]),
+        ],
+    )
+    store = FlowStore(tmp_path / "flows")
+    store.write_flow(flow)
+    store.write_node_source(
+        "split", "fit", "def process():\n    return {'model': 1.0}\n"
+    )
+    store.write_node_source(
+        "split", "readout", "def process(score):\n    return {'report': score}\n"
+    )
+    store.write_node_source("split", "evaluate", "def process(report):\n    pass\n")
+
+    service = RunService(controller=FlowController(store), queue=_Collect())
+    run_id = service.submit("split", {}).id
+    try:
+        service._drive(run_id)
+        with Session(db_engine) as session:
+            assert session.get(Run, run_id).status == "error"
+            rows = {
+                row.node: row
+                for row in session.exec(select(RunNode).where(RunNode.run_id == run_id))
+            }
+            assert {node: row.status for node, row in rows.items()} == {
+                "split.fit": "ok",
+                "split.readout": "error",
+                "split.evaluate": "skipped",
+            }
+            assert rows["split.readout"].error == "No value on input 'score' from 'fit'"
+    finally:
+        with Session(db_engine) as session:
+            for node in session.exec(select(RunNode).where(RunNode.run_id == run_id)):
+                session.delete(node)
+            session.delete(session.get(Run, run_id))
+            session.commit()
+
+
 def test_a_retry_is_a_new_run_that_names_the_one_it_repeats():
     """The way back from a run an engine restart interrupted.
 
@@ -905,6 +1004,39 @@ def test_a_run_waiting_for_a_worker_keeps_its_lease_and_can_be_cancelled(
             session.commit()
 
 
+class _Bus:
+    def __init__(self):
+        self.events = []
+
+    def publish(self, event):
+        self.events.append(event)
+
+
+def test_cancelling_a_run_does_not_pause_the_live_flow_it_shares_a_key_with():
+    """A run's pipeline publishes on the engine's bus under the flow's key, so
+    its pause marked the live flow paused in every open editor."""
+
+    class _Idle(_OneFlow):
+        workers = None
+        remote = None
+
+    service = RunService(
+        controller=_Idle(FlowDef(name="study", mode="batch")), queue=_Collect()
+    )
+    run = service.submit("study", {})
+    bus = _Bus()
+    pipeline = Pipeline(events=bus)
+    service._active[run.id] = pipeline
+    try:
+        assert service.cancel(run.id)
+        assert pipeline.is_disabled("study")
+        assert not [e for e in bus.events if e["type"] == "flow_paused"]
+    finally:
+        with Session(db_engine) as session:
+            session.delete(session.get(Run, run.id))
+            session.commit()
+
+
 def test_a_key_nobody_used_submits_normally(
     client, superuser_token_headers, monkeypatch
 ):

backend/tests/conftest.py

@@ -1,3 +1,4 @@
+import shutil
 import uuid
 from collections.abc import Generator
 from datetime import UTC, datetime
@@ -107,6 +108,19 @@ def _clean_engine_rows() -> Generator[None, None, None]:
         session.commit()
 
 
+@pytest.fixture(scope="module", autouse=True)
+def _clean_flow_store() -> Generator[None, None, None]:
+    """Start the next module on an empty flow store.
+
+    The store is session-wide like the database, so the flows, dashboards and
+    devices one module left were loaded by the next module's app at startup.
+    Removed whole, its git with it, after the module's app has stopped.
+    """
+    yield
+    shutil.rmtree(settings.FLOWS_DIR, ignore_errors=True)
+    settings.DEVICES_FILE.unlink(missing_ok=True)
+
+
 @pytest.fixture(scope="module")
 def client() -> Generator[TestClient, None, None]:
     with TestClient(app) as c:

backend/tests/flow/test_dashboards.py

@@ -5,6 +5,7 @@ from datetime import UTC, datetime
 import pytest
 
 from fluksio.flow.dashboards import (
+    CHART_RANGES,
     DashboardDef,
     DashboardNotFound,
     DashboardStore,
@@ -354,6 +355,26 @@ def test_a_querying_chart_asks_with_a_record_and_draws_a_series():
         WidgetDef(id="c", type="chart", config=query_chart(request_dtype="json"))
 
 
+def test_a_querying_chart_offers_its_own_ranges():
+    """A few labelled windows, where ``range_s: 0`` is the whole history."""
+    daily = [
+        {"label": "1M", "range_s": 30 * 86400, "interval_s": 86400},
+        {"label": "All", "range_s": 0, "interval_s": 86400},
+    ]
+    WidgetDef(id="c", type="chart", config=query_chart(ranges=daily))
+
+    for bad in (
+        [{"label": "", "range_s": 3600, "interval_s": 60}],
+        [{"label": "1h", "range_s": -1, "interval_s": 60}],
+        [{"label": "1h", "range_s": 3600, "interval_s": 0}],
+        [{"label": "1h", "range_s": 3600.5, "interval_s": 60}],
+        daily * CHART_RANGES,
+        "1M",
+    ):
+        with pytest.raises(ValueError, match="range"):
+            WidgetDef(id="c", type="chart", config=query_chart(ranges=bad))
+
+
 def test_a_colour_widget_publishes_the_shape_its_format_names():
     """The format picks the payload, so the binding is held to that one."""
     wheel = WidgetDef(

backend/tests/flow/test_engine_hold.py

@@ -5,13 +5,14 @@ service stops claiming and its timers once claimed work has finished. Both
 wait without a limit and can be aborted.
 """
 
+import asyncio
 import threading
 import time
 
 import pytest
 
 from fluksio.flow import executor
-from fluksio.flow.controller import FlowController, LoadedNode
+from fluksio.flow.controller import FlowController, LoadedNode, RebuildBusy
 from fluksio.flow.executor import ExecutionService
 from fluksio.flow.messages import DType, MessageSpec
 from fluksio.flow.namespace import Rename
@@ -125,8 +126,54 @@ class SlowClaim(MemoryWorkQueue):
         return items
 
 
+def test_pause_intake_waits_out_a_claim_already_under_way(monkeypatch):
+    seen: list = []
+    queue = SlowClaim()
+    _, service, source = _engine(seen.append, queue)
+    wake = service._capacity_changed
+
+    def closing() -> None:
+        wake()
+        # The claim lands only once the intake has closed.
+        queue.go.set()
+
+    monkeypatch.setattr(service, "_capacity_changed", closing)
+    service.start()
+    try:
+        source.inject({"reading": 1.0})
+        assert queue.claiming.wait(5)
+        service.pause_intake()
+        assert seen == [1.0]
+        assert service.inflight == 0
+    finally:
+        queue.go.set()
+        service.stop()
+
+
+def test_a_reload_that_cannot_drain_leaves_the_pipeline_as_it_was(
+    tmp_path, monkeypatch
+):
+    monkeypatch.setattr(executor, "DRAIN_TIMEOUT_S", 0.05)
+    seen: list = []
+    running, finish, consume = _blocking(seen)
+    pipeline, service, source = _engine(consume)
+    controller = FlowController(FlowStore(tmp_path / "flows"), execution=service)
+    controller.pipeline = pipeline
+    service.start()
+    try:
+        source.inject({"reading": 1.0})
+        assert running.wait(5)
+        with pytest.raises(RebuildBusy):
+            asyncio.run(controller.reload())
+        assert controller.pipeline is pipeline
+        assert service._pipeline is pipeline
+        assert service._intake.is_set()
+    finally:
+        finish.set()
+        service.stop()
+
+
 def test_a_hold_waits_out_a_claim_already_under_way():
-    """`pause_intake` returns here and the item runs after it."""
     seen: list = []
     queue = SlowClaim()
     _, service, source = _engine(seen.append, queue)

backend/tests/flow/test_redis_state.py

@@ -0,0 +1,54 @@
+"""RedisState against a real server, enabled by FLUKSIO_TEST_REDIS_HOST (and
+optional PORT)."""
+
+import os
+import uuid
+
+import pytest
+
+from fluksio.flow.state import RedisState
+
+
+@pytest.fixture
+def state():
+    host = os.environ.get("FLUKSIO_TEST_REDIS_HOST")
+    if not host:
+        pytest.skip("Set FLUKSIO_TEST_REDIS_HOST to run Redis integration checks")
+    port = int(os.environ.get("FLUKSIO_TEST_REDIS_PORT", "6379"))
+    state = RedisState(host, port, namespace=f"test-state-{uuid.uuid4().hex}", ttl=600)
+    try:
+        yield state
+    finally:
+        state.clear()
+        state.close()
+
+
+def test_a_ttl_expires_counters_and_the_names_set(state):
+    state.set("heat.temp", 21)
+    state.increment("__version__:heat.a")
+    state.increment_multi(["__version__:heat.b"])
+    state.record({"heat.v": 1}, {}, ["__version__:heat.c"], 0.0)
+
+    keys = state.keys()
+    assert "__names__" in keys and "__version__:heat.c" in keys
+    assert all(state._client.ttl(state._key(k)) > 0 for k in keys)
+
+
+def test_a_compare_and_swap_names_what_it_wrote(state):
+    state.set("heat.a", 1)
+    assert state.compare_and_swap_multi({"heat.a": 1}, {"heat.b": 2})
+    assert "heat.b" in state.message_names()
+
+
+def test_a_compare_and_swap_loses_to_a_write_after_its_read(state, monkeypatch):
+    state.set("heat.a", 1)
+    note = RedisState._note
+
+    def racing_note(self, pipe, keys):
+        # Another writer lands between the compare and EXEC.
+        self._client.set(self._key("heat.a"), b"2")
+        note(self, pipe, keys)
+
+    monkeypatch.setattr(RedisState, "_note", racing_note)
+    assert not state.compare_and_swap_multi({"heat.a": 1}, {"heat.a": 3})
+    assert state.get("heat.a") == 2

backend/tests/flow/test_runs.py

@@ -346,6 +346,16 @@ def test_a_rate_limited_port_cannot_be_run_as_a_batch():
     assert not batch_issues(double_flow())
 
 
+def test_a_delay_nodes_interval_cannot_be_run_as_a_batch():
+    flow = double_flow()
+    flow.nodes[0].type = "delay"
+    flow.nodes[0].params = {"interval": 5}
+    # The node puts it on every output, where it holds values back the same way.
+    assert batch_issues(flow)
+    flow.nodes[0].params = {"delay": 5}
+    assert not batch_issues(flow)
+
+
 def test_a_node_with_no_stored_code_cannot_be_run():
     """The store answers a template for one, and a template publishes nothing."""
 

backend/tests/sdk/test_build.py

@@ -240,6 +240,28 @@ def test_a_port_named_like_its_function_still_calls_the_function():
     assert namespace["process"](scale=2.0) == {"scaled": 4.0}
 
 
+@node(
+    requires=["value", Port("factor", "float", trigger=False)],
+    provides=Port("scaled", "float"),
+)
+def scale_by(value, factor=3.0):
+    return value * factor
+
+
+def test_a_port_that_does_not_trigger_may_be_missing_from_the_call():
+    """The engine leaves out an input with no value yet, so the default applies."""
+    flow = Flow(
+        "optional",
+        nodes=[scale_by],
+        inputs=[Port("value", "float", initial=1.0), Port("factor", "float")],
+    )
+    namespace: dict = {}
+    exec(compile(flow.shims()["scale_by"], "<shim>", "exec"), namespace)
+
+    assert namespace["process"](value=2.0) == {"scaled": 6.0}
+    assert namespace["process"](value=2.0, factor=0.5) == {"scaled": 1.0}
+
+
 def reads_settings(settings):
     return settings
 

backend/tests/sdk/test_client.py

@@ -33,6 +33,19 @@ def a_client(handler, monkeypatch, **kwargs):
     return Client(url="http://engine", token="t", http=http, **kwargs)
 
 
+@pytest.mark.parametrize(
+    "url",
+    [
+        "http://e:8000",
+        "http://e:8000/",
+        "http://e:8000/api/v1",
+        "http://e:8000/api/v1/",
+    ],
+)
+def test_an_engine_url_may_carry_the_api_path(url):
+    assert Client(url=url, token="t").url == "http://e:8000"
+
+
 def test_idempotent_get_is_tried_again(monkeypatch):
     calls = []
 

backend/tests/test_cloud.py

@@ -788,7 +788,9 @@ async def test_a_revoked_link_stands_down(enrolled: User) -> None:  # noqa: ARG0
         watcher = asyncio.create_task(connector.watch_enrolment(app))
         await asyncio.sleep(0.05)
         watcher.cancel()
-    assert not started
+    # The module's app runs a watcher of its own on the same patched names and
+    # may dial the enrolment above inside the window, so only this one counts.
+    assert app not in started
 
 
 def test_a_member_removed_at_the_portal_loses_their_user(db: Session) -> None: