Merge branch 'rename-s2'

backend/tests/api/routes/test_flows.py

@@ -9,7 +9,9 @@ from sqlmodel import Session, select
 
 from fluksio.api.routes import flows as flows_route
 from fluksio.core.config import settings
+from fluksio.flow.devices import Binding
 from fluksio.flow.events import event_bus
+from fluksio.flow.pipeline import EngineHeld
 from fluksio.models import Run, RunArtifact, RunMetric, RunNode
 from tests.utils.flows import put_flow
 
@@ -600,3 +602,35 @@ def test_a_shared_source_write_is_audited(
         headers=superuser_token_headers,
     )
     client.delete(f"{PREFIX}/audited", headers=superuser_token_headers)
+
+
+def test_a_publish_while_a_rename_holds_the_engine_is_refused(
+    client: TestClient,
+    superuser_token_headers: dict[str, str],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Over REST as a 503 to retry, from a device's socket as ``busy``."""
+    controller = client.app.state.flow_controller  # type: ignore[attr-defined]
+
+    def held(*_: object) -> None:
+        raise EngineHeld("A rename holds the engine; try again shortly")
+
+    monkeypatch.setattr(controller, "publish_message", held)
+    response = client.post(
+        f"{settings.API_V1_STR}/messages/porch.doorbell",
+        headers=superuser_token_headers,
+        json={"value": True},
+    )
+    assert response.status_code == 503
+    assert response.headers["Retry-After"] == "5"
+
+    bound = Binding("panel", "porch", "panel", "gpio.4", "porch.doorbell", "in", "bool")
+    app = SimpleNamespace(
+        state=SimpleNamespace(
+            device_store=SimpleNamespace(bound=lambda _: [bound]),
+            flow_controller=controller,
+        )
+    )
+    device = SimpleNamespace(id="panel", title="Panel")
+    value = {"type": "publish", "name": "porch.doorbell", "value": True}
+    assert flows_route.ingest_publish(app, device, value) == "busy"  # type: ignore[arg-type]

backend/tests/flow/test_engine_hold.py

@@ -0,0 +1,272 @@
+"""Engine holds for a rename (section 2.2 of docs/private/id_refac_rename.md).
+
+The pipeline refuses what it would run outside the queue, and the execution
+service stops claiming and its timers once claimed work has finished. Both
+wait without a limit and can be aborted.
+"""
+
+import threading
+import time
+
+import pytest
+
+from fluksio.flow import executor
+from fluksio.flow.executor import ExecutionService
+from fluksio.flow.messages import DType, MessageSpec
+from fluksio.flow.namespace import Rename
+from fluksio.flow.nodes import Node
+from fluksio.flow.pipeline import EngineHeld, Pipeline
+from fluksio.flow.queue import MemoryWorkQueue, WorkItem
+from fluksio.flow.state import MemoryState
+
+
+def _engine(consume, queue=None) -> tuple[Pipeline, ExecutionService, Node]:
+    """A source whose readings ``consume`` gets, and a service draining the
+    queue one cascade at a time, so order shows."""
+
+    def consumer_body(reading, params):
+        consume(reading)
+
+    source = Node(
+        f=lambda params: None,
+        provides=[MessageSpec(name="reading", dtype=DType.FLOAT)],
+        name="source",
+    )
+    consumer = Node(
+        f=consumer_body,
+        requires=[MessageSpec(name="reading", dtype=DType.FLOAT)],
+        name="consumer",
+    )
+    source.assign_flow("f", "source")
+    consumer.assign_flow("f", "consumer")
+    queue = queue or MemoryWorkQueue()
+    service = ExecutionService(queue, max_cascades=1)
+    pipeline = Pipeline(
+        nodes=[source, consumer],
+        state=MemoryState(),
+        work_queue=queue,
+        node_pool=service.node_pool,
+    )
+    service.bind(pipeline)
+    return pipeline, service, source
+
+
+def _in_thread(fn, *args) -> tuple[threading.Thread, list]:
+    result: list = []
+    thread = threading.Thread(target=lambda: result.append(fn(*args)), daemon=True)
+    thread.start()
+    return thread, result
+
+
+def _until(check, timeout: float = 5.0) -> None:
+    deadline = time.monotonic() + timeout
+    while not check():
+        assert time.monotonic() < deadline, "timed out"
+        time.sleep(0.02)
+
+
+def _blocking(seen: list) -> tuple[threading.Event, threading.Event, object]:
+    """A consumer that records and then waits until told to finish."""
+    running, finish = threading.Event(), threading.Event()
+
+    def consume(reading):
+        seen.append(reading)
+        running.set()
+        finish.wait(5)
+
+    return running, finish, consume
+
+
+def test_a_hold_waits_out_a_running_cascade_without_a_limit(monkeypatch):
+    # What a rebuild's drain gives up after, which a hold does not.
+    monkeypatch.setattr(executor, "DRAIN_TIMEOUT_S", 0.05)
+    seen: list = []
+    running, finish, consume = _blocking(seen)
+    _, service, source = _engine(consume)
+    service.start()
+    try:
+        source.inject({"reading": 1.0})
+        assert running.wait(5)
+        busy: list = []
+        thread, held = _in_thread(service.hold, threading.Event(), busy.append)
+        thread.join(0.3)
+        assert thread.is_alive()
+        assert busy == [["f.source"]]
+
+        source.inject({"reading": 2.0})
+        finish.set()
+        thread.join(5)
+        assert held == [True]
+        time.sleep(0.2)
+        assert seen == [1.0]
+
+        service.release()
+        _until(lambda: seen == [1.0, 2.0])
+    finally:
+        finish.set()
+        service.stop()
+
+
+class SlowClaim(MemoryWorkQueue):
+    """A queue whose claim is still under way when a hold begins."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.claiming = threading.Event()
+        self.go = threading.Event()
+
+    def claim(self, count, block_ms, due_only=False):
+        items = super().claim(count, block_ms, due_only)
+        if items:
+            self.claiming.set()
+            self.go.wait(5)
+        return items
+
+
+def test_a_hold_waits_out_a_claim_already_under_way():
+    """`pause_intake` returns here and the item runs after it."""
+    seen: list = []
+    queue = SlowClaim()
+    _, service, source = _engine(seen.append, queue)
+    service.start()
+    try:
+        source.inject({"reading": 1.0})
+        assert queue.claiming.wait(5)
+        thread, held = _in_thread(service.hold, threading.Event())
+        thread.join(0.2)
+        assert thread.is_alive()
+
+        queue.go.set()
+        thread.join(5)
+        assert held == [True]
+        assert seen == [1.0]
+        assert service.inflight == 0
+        service.release()
+    finally:
+        queue.go.set()
+        service.stop()
+
+
+def test_an_aborted_hold_leaves_the_engine_as_it_was():
+    seen: list = []
+    running, finish, consume = _blocking(seen)
+    pipeline, service, source = _engine(consume)
+    service.start()
+    try:
+        source.inject({"reading": 1.0})
+        assert running.wait(5)
+        abort = threading.Event()
+        thread, held = _in_thread(service.hold, abort)
+        thread.join(0.2)
+        assert thread.is_alive()
+
+        abort.set()
+        thread.join(5)
+        assert held == [False]
+        finish.set()
+
+        # Claiming and promotion carry on.
+        assert pipeline.defer(source, {"f.reading": 2.0}, 0.05)
+        source.inject({"reading": 3.0})
+        _until(lambda: sorted(seen) == [1.0, 2.0, 3.0])
+        assert not service.queue.held
+    finally:
+        finish.set()
+        service.stop()
+
+
+def test_an_aborted_pipeline_hold_lets_publishes_through_again():
+    seen: list = []
+    running, finish, consume = _blocking(seen)
+    pipeline, _, _ = _engine(consume)
+    publishing, _ = _in_thread(pipeline.publish, {"f.reading": 1.0})
+    try:
+        assert running.wait(5)
+        busy: list = []
+        abort = threading.Event()
+        thread, held = _in_thread(pipeline.hold, abort, busy.append)
+        thread.join(0.2)
+        assert thread.is_alive()
+        assert busy == [["f.reading"]]
+
+        abort.set()
+        thread.join(5)
+        assert held == [False]
+        finish.set()
+        publishing.join(5)
+        pipeline.publish({"f.reading": 2.0})
+        assert seen == [1.0, 2.0]
+    finally:
+        finish.set()
+
+
+def test_publish_is_refused_while_held():
+    seen: list = []
+    pipeline, _, _ = _engine(seen.append)
+    assert pipeline.hold(threading.Event())
+
+    with pytest.raises(EngineHeld):
+        pipeline.publish({"f.reading": 1.0})
+    assert "f.reading" not in pipeline.state
+
+    pipeline.release()
+    pipeline.publish({"f.reading": 2.0})
+    assert seen == [2.0]
+
+
+def test_work_arriving_during_a_hold_runs_after_it_in_order():
+    seen: list = []
+    pipeline, service, source = _engine(seen.append)
+    service.start()
+    try:
+        abort = threading.Event()
+        assert pipeline.hold(abort)
+        assert service.hold(abort)
+        source.inject({"reading": 1.0})
+        service.queue.hold()
+        source.inject({"reading": 2.0})
+        source.inject({"reading": 3.0})
+        time.sleep(0.2)
+        assert seen == []
+
+        service.queue.release()
+        service.release()
+        pipeline.release()
+        _until(lambda: seen == [1.0, 2.0, 3.0])
+    finally:
+        service.stop()
+
+
+def test_a_queue_failure_while_held_drops_the_value_rather_than_run_it(
+    monkeypatch,
+):
+    seen: list = []
+    pipeline, service, source = _engine(seen.append)
+
+    def down(item):
+        raise ConnectionError("queue down")
+
+    monkeypatch.setattr(service.queue, "add", down)
+    assert pipeline.hold(threading.Event())
+    source.inject({"reading": 1.0})
+    assert seen == []
+
+    pipeline.release()
+    source.inject({"reading": 2.0})
+    assert seen == [2.0]
+
+
+def test_reclaimed_work_waiting_to_run_follows_a_rename():
+    service = ExecutionService(MemoryWorkQueue())
+    service._reclaimed.extend(
+        [
+            WorkItem(kind="cascade", node="heat.valve", flow="heat", entry_id="1-0"),
+            WorkItem(kind="cascade", node="heating.pump", flow="heating"),
+        ]
+    )
+    service.rename_reclaimed(Rename("heat", "warm"))
+    assert [(i.node, i.flow) for i in service._reclaimed] == [
+        ("warm.valve", "warm"),
+        ("heating.pump", "heating"),
+    ]
+    assert service._reclaimed[0].entry_id == "1-0"

backend/tests/flow/test_rename_runtime.py

@@ -3,6 +3,7 @@ and 3 of docs/private/id_refac_rename.md). Memory always, Redis with
 FLUKSIO_TEST_REDIS_HOST (and optional PORT)."""
 
 import os
+import threading
 import uuid
 
 import pytest
@@ -218,6 +219,36 @@ def test_held_work_is_released_in_order_behind_older_work(queue):
     ]
 
 
+def test_a_hold_waits_for_an_add_already_writing_to_the_stream(
+    redis_queue, monkeypatch
+):
+    """Adds do not wait for each other, but a hold waits for them, so no
+    entry lands in a stream after the rebuild has read it."""
+    writing, finish = threading.Event(), threading.Event()
+    xadd = redis_queue._redis.xadd
+
+    def slow(*args, **kwargs):
+        writing.set()
+        finish.wait(5)
+        return xadd(*args, **kwargs)
+
+    monkeypatch.setattr(redis_queue._redis, "xadd", slow)
+    adding = threading.Thread(target=redis_queue.add, args=(item("heat.a"),))
+    adding.start()
+    assert writing.wait(5)
+    holding = threading.Thread(target=redis_queue.hold)
+    holding.start()
+    holding.join(0.2)
+    assert holding.is_alive()
+
+    finish.set()
+    holding.join(5)
+    adding.join(5)
+    assert redis_queue.count(HEAT)["queue_items"] == 1
+    assert redis_queue.held
+    redis_queue.release()
+
+
 def test_a_guarded_timer_still_finds_its_guard(queue):
     state = MemoryState()
     state["__node__:heat.limit:generation"] = 3