Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
This commit is contained in:
@@ -2,8 +2,11 @@
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from contextlib import AsyncExitStack
|
||||
|
||||
from fluksio.flow.events import QUEUE_SIZE, EventBus
|
||||
import pytest
|
||||
|
||||
from fluksio.flow.events import DISPATCH_BUDGET, QUEUE_SIZE, EventBus
|
||||
|
||||
|
||||
def _event(i: int) -> dict[str, object]:
|
||||
@@ -100,3 +103,91 @@ def test_nothing_is_buffered_with_nobody_listening() -> None:
|
||||
return len(bus._pending)
|
||||
|
||||
assert asyncio.run(run()) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flush", [False, True])
|
||||
def test_overflow_keeps_errors_and_control_events_in_order(flush: bool) -> None:
|
||||
"""The staging bound and a slow subscriber use the same eviction policy."""
|
||||
|
||||
async def run() -> None:
|
||||
bus = EventBus()
|
||||
bus.bind(asyncio.get_running_loop())
|
||||
critical = [
|
||||
{"type": "node_log", "level": "error"},
|
||||
{"type": "node_error"},
|
||||
{"type": "run_started"},
|
||||
{"type": "audit"},
|
||||
{"type": "dashboard_changed"},
|
||||
{"type": "future_control_event"},
|
||||
]
|
||||
async with bus.subscribe() as first, bus.subscribe() as second:
|
||||
sent = [*critical, *(_event(i) for i in range(QUEUE_SIZE + 50))]
|
||||
sent.append({"type": "run_finished"})
|
||||
for sequence, event in enumerate(sent):
|
||||
event["sequence"] = sequence
|
||||
bus.publish(event)
|
||||
assert len(bus._pending) <= QUEUE_SIZE
|
||||
if flush:
|
||||
# Queue overflow, with no staging overflow.
|
||||
await asyncio.sleep(0)
|
||||
await asyncio.sleep(0)
|
||||
for queue in (first, second):
|
||||
received = [queue.get_nowait() for _ in range(queue.qsize())]
|
||||
assert len(received) == QUEUE_SIZE
|
||||
assert received[: len(critical)] == critical
|
||||
assert received[-1]["type"] == "run_finished"
|
||||
assert received == sorted(received, key=lambda e: e["sequence"])
|
||||
assert bus.dropped(queue)
|
||||
assert not bus.dropped(queue)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_a_full_critical_backlog_rejects_telemetry_and_reports_loss() -> None:
|
||||
async def run() -> None:
|
||||
bus = EventBus()
|
||||
bus.bind(asyncio.get_running_loop())
|
||||
async with bus.subscribe() as queue:
|
||||
for i in range(QUEUE_SIZE + 1):
|
||||
bus.publish({"type": "node_error", "sequence": i})
|
||||
bus.publish(_event(0))
|
||||
assert len(bus._pending) == QUEUE_SIZE
|
||||
await asyncio.sleep(0)
|
||||
assert bus.dropped(queue)
|
||||
assert (await queue.get())["sequence"] == 1
|
||||
bus.publish({"type": "audit"})
|
||||
await asyncio.sleep(0)
|
||||
bus.publish(_event(1))
|
||||
await asyncio.sleep(0)
|
||||
assert bus.dropped(queue)
|
||||
assert queue.qsize() == QUEUE_SIZE
|
||||
assert all(
|
||||
queue.get_nowait()["type"] != "node_executed" for _ in range(QUEUE_SIZE)
|
||||
)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_fanout_yields_before_all_subscribers_are_served() -> None:
|
||||
async def run() -> None:
|
||||
bus = EventBus()
|
||||
loop = asyncio.get_running_loop()
|
||||
bus.bind(loop)
|
||||
async with AsyncExitStack() as stack:
|
||||
queues = [
|
||||
await stack.enter_async_context(bus.subscribe())
|
||||
for _ in range(DISPATCH_BUDGET // QUEUE_SIZE + 1)
|
||||
]
|
||||
for i in range(QUEUE_SIZE):
|
||||
bus.publish(_event(i))
|
||||
heartbeat: asyncio.Future[int] = loop.create_future()
|
||||
loop.call_soon(
|
||||
lambda: heartbeat.set_result(sum(queue.qsize() for queue in queues))
|
||||
)
|
||||
assert await heartbeat == DISPATCH_BUDGET
|
||||
await asyncio.sleep(0)
|
||||
assert all(queue.qsize() == QUEUE_SIZE for queue in queues)
|
||||
assert not any(bus.dropped(queue) for queue in queues)
|
||||
assert not bus._flush_scheduled
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
@@ -242,6 +242,48 @@ def test_a_trigger_is_journaled_rather_than_run_on_the_spot():
|
||||
assert state["f.doubled"] == 6.0
|
||||
|
||||
|
||||
def test_a_state_failure_leaves_the_delivery_unacknowledged(monkeypatch):
|
||||
pipeline, source, state, seen = _pipeline_with_a_consumer()
|
||||
service = ExecutionService(pipeline._queue)
|
||||
service.bind(pipeline)
|
||||
|
||||
def unavailable(*args, **kwargs):
|
||||
raise ConnectionError("State unavailable")
|
||||
|
||||
monkeypatch.setattr(state, "record", unavailable)
|
||||
source.inject({"reading": 3.0})
|
||||
item = pipeline._queue.claim(1, 10)[0]
|
||||
try:
|
||||
service._dispatch(item)
|
||||
service.pause_intake()
|
||||
assert seen == []
|
||||
assert pipeline._queue.stats()["pending"] == 1
|
||||
assert service.inflight == 0
|
||||
finally:
|
||||
service.stop()
|
||||
|
||||
|
||||
def test_a_node_failure_still_acknowledges_its_delivery(monkeypatch):
|
||||
pipeline, source, _state, seen = _pipeline_with_a_consumer()
|
||||
service = ExecutionService(pipeline._queue)
|
||||
service.bind(pipeline)
|
||||
|
||||
def broken(*args, **kwargs):
|
||||
raise ValueError("Node failed")
|
||||
|
||||
monkeypatch.setattr(pipeline.get_node_by_id("f.consumer"), "f", broken)
|
||||
source.inject({"reading": 3.0})
|
||||
item = pipeline._queue.claim(1, 10)[0]
|
||||
try:
|
||||
service._dispatch(item)
|
||||
service.pause_intake()
|
||||
assert seen == []
|
||||
assert pipeline._queue.stats()["pending"] == 0
|
||||
assert service.inflight == 0
|
||||
finally:
|
||||
service.stop()
|
||||
|
||||
|
||||
def test_work_for_a_paused_flow_is_held_and_released_on_resume():
|
||||
pipeline, source, state, seen = _pipeline_with_a_consumer()
|
||||
service = ExecutionService(pipeline._queue)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Redis delivery checks, enabled by FLUKSIO_TEST_REDIS_HOST (and optional PORT)."""
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
import redis
|
||||
|
||||
from fluksio.flow.queue import RedisWorkQueue, WorkItem
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def queue():
|
||||
host = os.environ.get("FLUKSIO_TEST_REDIS_HOST")
|
||||
if not host:
|
||||
pytest.skip("Set FLUKSIO_TEST_REDIS_HOST to run Redis integration checks")
|
||||
queue = RedisWorkQueue(
|
||||
host=host,
|
||||
port=int(os.environ.get("FLUKSIO_TEST_REDIS_PORT", "6379")),
|
||||
namespace=f"test-queue-{uuid.uuid4().hex}",
|
||||
)
|
||||
try:
|
||||
yield queue
|
||||
finally:
|
||||
keys = list(queue._redis.scan_iter(match=f"{queue._ns}:*"))
|
||||
if keys:
|
||||
queue._redis.delete(*keys)
|
||||
queue.close()
|
||||
|
||||
|
||||
def test_concurrent_promotions_deliver_each_due_item_once(queue):
|
||||
for i in range(150):
|
||||
queue.add_delayed(
|
||||
WorkItem(kind="cascade", node=f"f.n{i}", flow="f", outputs={"f.x": i}),
|
||||
100.0,
|
||||
)
|
||||
queue.add_delayed(WorkItem(kind="cascade", node="f.later", flow="f"), 300.0)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
moved = list(pool.map(queue.move_due, [200.0, 200.0]))
|
||||
|
||||
assert sorted(moved) == [50, 100]
|
||||
assert queue.move_due(200.0) == 0
|
||||
assert queue.next_due() == 300.0
|
||||
claimed = queue.claim(200, 10, due_only=True)
|
||||
assert len(claimed) == 150
|
||||
assert {item.outputs["f.x"] for item in claimed} == set(range(150))
|
||||
assert all(item.not_before == 100.0 for item in claimed)
|
||||
assert all(item.entry_id.startswith("due:") for item in claimed)
|
||||
|
||||
|
||||
def test_a_refused_stream_write_keeps_the_timer_for_retry(queue):
|
||||
queue.add_delayed(WorkItem(kind="cascade", node="f.stop", flow="f"), 100.0)
|
||||
# A Redis command error does not roll a Lua script back. The stream write
|
||||
# must succeed before the delayed entry is removed.
|
||||
queue._redis.delete(queue._due_stream)
|
||||
queue._redis.set(queue._due_stream, "not a stream")
|
||||
|
||||
with pytest.raises(redis.ResponseError, match="WRONGTYPE"):
|
||||
queue.move_due(200.0)
|
||||
|
||||
assert queue.next_due() == 100.0
|
||||
queue._redis.delete(queue._due_stream)
|
||||
queue._ensure_group()
|
||||
assert queue.move_due(200.0) == 1
|
||||
assert queue.move_due(200.0) == 0
|
||||
assert [item.node for item in queue.claim(10, 10, due_only=True)] == ["f.stop"]
|
||||
@@ -122,3 +122,24 @@ def test_a_hop_costs_a_bounded_number_of_state_operations() -> None:
|
||||
|
||||
total = sum(state.calls[name] for name in COUNTED)
|
||||
assert total <= 8, f"state operations per hop grew: {dict(state.calls)}"
|
||||
|
||||
|
||||
def test_reading_one_value_does_not_list_or_read_other_messages(monkeypatch) -> None:
|
||||
state = MemoryState()
|
||||
state.update({f"f.m{i}": i for i in range(1000)})
|
||||
state.update({"f.m0": None, "__ts__:f.m0": 123.0})
|
||||
pipeline = Pipeline([], state=state)
|
||||
reads = []
|
||||
get_multi = state.get_multi
|
||||
|
||||
def read(keys):
|
||||
reads.append(keys)
|
||||
return get_multi(keys)
|
||||
|
||||
def no_listing():
|
||||
raise AssertionError("A single-message read must not enumerate state")
|
||||
|
||||
monkeypatch.setattr(MemoryState, "get_multi", lambda self, keys: read(keys))
|
||||
monkeypatch.setattr(MemoryState, "message_names", lambda self: no_listing())
|
||||
assert pipeline.value("f.m0") == {"value": None, "ts": 123.0}
|
||||
assert reads == [["f.m0", "__ts__:f.m0"]]
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Scheduling work scales with the graph and observes runtime gates."""
|
||||
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import nullcontext
|
||||
|
||||
import pytest
|
||||
|
||||
from fluksio.flow.messages import DType, MessageSpec
|
||||
from fluksio.flow.nodes import Node
|
||||
from fluksio.flow.pipeline import Pipeline
|
||||
|
||||
|
||||
def _node(name, function, requires=(), provides=(), flow="f"):
|
||||
node = Node(
|
||||
f=function,
|
||||
requires=[MessageSpec(name=key, dtype=DType.FLOAT) for key in requires],
|
||||
provides=[MessageSpec(name=key, dtype=DType.FLOAT) for key in provides],
|
||||
name=name,
|
||||
)
|
||||
node.assign_flow(flow, name)
|
||||
return node
|
||||
|
||||
|
||||
def test_a_long_chain_checks_each_candidate_once():
|
||||
class CountingPipeline(Pipeline):
|
||||
checks = 0
|
||||
|
||||
def _gate_blocks(self, node):
|
||||
self.checks += 1
|
||||
return super()._gate_blocks(node)
|
||||
|
||||
nodes = []
|
||||
for i in range(100):
|
||||
|
||||
def relay(params, _index=i, **inputs):
|
||||
return {f"v{_index + 1}": inputs[f"v{_index}"] + 1.0}
|
||||
|
||||
nodes.append(_node(f"n{i}", relay, requires=[f"v{i}"], provides=[f"v{i + 1}"]))
|
||||
pipeline = CountingPipeline(nodes)
|
||||
|
||||
pipeline.run({"f.v0": 0.0})
|
||||
|
||||
assert pipeline.values()["f.v100"]["value"] == 100.0
|
||||
assert pipeline.checks == len(nodes)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disabled", [False, True])
|
||||
def test_a_gate_opened_mid_wave_releases_its_waiting_candidate(disabled):
|
||||
running, release, held, acted = (threading.Event() for _ in range(4))
|
||||
|
||||
class WatchingPipeline(Pipeline):
|
||||
def _gate_blocks(self, node):
|
||||
blocked = super()._gate_blocks(node)
|
||||
if blocked:
|
||||
held.set()
|
||||
return blocked
|
||||
|
||||
def slow(params):
|
||||
running.set()
|
||||
assert release.wait(3)
|
||||
|
||||
pipeline = WatchingPipeline(
|
||||
[
|
||||
_node("slow", slow, flow="running"),
|
||||
_node("action", lambda params: acted.set(), flow="paused"),
|
||||
]
|
||||
)
|
||||
if disabled:
|
||||
pipeline.set_disabled({"paused"})
|
||||
else:
|
||||
pipeline.pause("paused")
|
||||
with ThreadPoolExecutor(max_workers=1) as driver:
|
||||
future = driver.submit(pipeline.run)
|
||||
try:
|
||||
assert running.wait(1) and held.wait(1)
|
||||
if disabled:
|
||||
pipeline.set_disabled(set())
|
||||
with nullcontext() if disabled else pipeline.stepping("paused"):
|
||||
release.set()
|
||||
future.result(2)
|
||||
assert acted.is_set()
|
||||
finally:
|
||||
release.set()
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Direct and portal browsers recover state without widening camera access."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from contextlib import suppress
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from fluksio.api import deps
|
||||
from fluksio.api.routes import flows
|
||||
from fluksio.cloud.connector import CloudConnector
|
||||
from fluksio.flow import events
|
||||
from fluksio.flow.events import QUEUE_SIZE, EventBus
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tunnel", [False, True])
|
||||
@pytest.mark.parametrize("burst", [0, QUEUE_SIZE + 1])
|
||||
def test_a_browser_catches_changes_during_its_snapshot(monkeypatch, tunnel, burst):
|
||||
async def run():
|
||||
bus = EventBus()
|
||||
bus.bind(asyncio.get_running_loop())
|
||||
monkeypatch.setattr(events, "event_bus", bus)
|
||||
monkeypatch.setattr(flows, "event_bus", bus)
|
||||
user = SimpleNamespace(id="viewer", is_superuser=False)
|
||||
for module in (flows, deps):
|
||||
monkeypatch.setattr(module, "user_from_token", lambda *a, **kw: user)
|
||||
monkeypatch.setattr(
|
||||
module, "hidden_from", lambda *a, **kw: frozenset({"f.camera"})
|
||||
)
|
||||
monkeypatch.setattr(flows, "panel_scope", lambda *a: None)
|
||||
monkeypatch.setattr(flows, "socket_token", lambda *a: "test-token")
|
||||
|
||||
async def immediate(function, *args):
|
||||
return function(*args)
|
||||
|
||||
monkeypatch.setattr(flows, "run_in_threadpool", immediate)
|
||||
current = {"f.flag": {"value": 0}, "f.camera": {"value": "private"}}
|
||||
controller = SimpleNamespace(
|
||||
values=lambda: dict(current),
|
||||
node_statuses=lambda: [],
|
||||
issues=[],
|
||||
paused_flows=lambda: [],
|
||||
placer=None,
|
||||
)
|
||||
app = SimpleNamespace(state=SimpleNamespace(flow_controller=controller))
|
||||
received = []
|
||||
client_values = {}
|
||||
caught_up = asyncio.Event()
|
||||
drained = asyncio.Event()
|
||||
|
||||
class Socket:
|
||||
headers = {}
|
||||
|
||||
async def accept(self):
|
||||
pass
|
||||
|
||||
async def receive(self):
|
||||
await asyncio.Future()
|
||||
|
||||
async def send(self, text):
|
||||
frame = json.loads(text)
|
||||
if frame["op"] == "ws_msg":
|
||||
await self.send_text(frame["text"])
|
||||
|
||||
async def send_text(self, text):
|
||||
payload = json.loads(text)
|
||||
received.append(payload)
|
||||
if payload["type"] == "snapshot":
|
||||
assert "f.camera" not in payload["values"]
|
||||
client_values.update(payload["values"])
|
||||
if payload["values"]["f.flag"]["value"] == 1:
|
||||
caught_up.set()
|
||||
elif len(received) == 1:
|
||||
# The state was read, but its frame is still sending.
|
||||
# This value changes only once, then a burst loses its
|
||||
# event and leaves only a snapshot able to recover it.
|
||||
current["f.flag"] = {"value": 1}
|
||||
bus.publish(
|
||||
{"type": "message_value", "name": "f.flag", "value": 1}
|
||||
)
|
||||
for i in range(burst):
|
||||
current["f.noise"] = {"value": i}
|
||||
bus.publish(
|
||||
{"type": "message_value", "name": "f.noise", "value": i}
|
||||
)
|
||||
# Recovery must forward retained critical events too.
|
||||
bus.publish({"type": "node_error", "node": "f.example"})
|
||||
await asyncio.sleep(0)
|
||||
for event in payload.get("events", [payload]):
|
||||
if event["type"] == "message_value":
|
||||
client_values[event["name"]] = {"value": event["value"]}
|
||||
if event.get("name") == "f.flag" and event.get("value") == 1:
|
||||
caught_up.set()
|
||||
if event["type"] == "node_error":
|
||||
drained.set()
|
||||
|
||||
socket = Socket()
|
||||
socket.app = app
|
||||
if tunnel:
|
||||
action = CloudConnector(app)._stream(
|
||||
socket,
|
||||
{"id": "review", "path": "/api/v1/flows/ws", "query": "token=test"},
|
||||
)
|
||||
else:
|
||||
action = flows.flow_events(socket)
|
||||
task = asyncio.create_task(action)
|
||||
try:
|
||||
await asyncio.wait_for(caught_up.wait(), timeout=2)
|
||||
await asyncio.wait_for(drained.wait(), timeout=2)
|
||||
finally:
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task
|
||||
assert sum(item["type"] == "snapshot" for item in received) == (
|
||||
2 if burst else 1
|
||||
)
|
||||
flattened = [
|
||||
event for payload in received for event in payload.get("events", [payload])
|
||||
]
|
||||
assert sum(event["type"] == "node_error" for event in flattened) == 1
|
||||
if burst:
|
||||
# The client replaces values in arrival order. A snapshot ahead
|
||||
# of the retained backlog temporarily rolls values backwards.
|
||||
assert flattened[-1]["type"] == "snapshot"
|
||||
assert client_values == {k: v for k, v in current.items() if k != "f.camera"}
|
||||
assert not bus._subscribers
|
||||
|
||||
asyncio.run(run())
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Deadlines retain worker capacity when ordinary work is blocked."""
|
||||
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from fluksio.flow import executor
|
||||
from fluksio.flow.executor import ExecutionService
|
||||
from fluksio.flow.messages import DType, MessageSpec
|
||||
from fluksio.flow.nodes import Node
|
||||
from fluksio.flow.pipeline import Pipeline
|
||||
from fluksio.flow.queue import MemoryWorkQueue, WorkItem
|
||||
|
||||
|
||||
def _node(name, function, requires=(), provides=()):
|
||||
node = Node(
|
||||
f=function,
|
||||
requires=[MessageSpec(name=key, dtype=DType.FLOAT) for key in requires],
|
||||
provides=[MessageSpec(name=key, dtype=DType.FLOAT) for key in provides],
|
||||
name=name,
|
||||
)
|
||||
node.assign_flow("f", name)
|
||||
return node
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reclaimed", [False, True])
|
||||
def test_a_due_node_runs_while_ordinary_nodes_and_cascades_are_full(
|
||||
monkeypatch, reclaimed
|
||||
):
|
||||
started, release, fired, recovered, touched = (threading.Event() for _ in range(5))
|
||||
calls = []
|
||||
|
||||
class Queue(MemoryWorkQueue):
|
||||
def reclaim_stale(self, min_idle_ms):
|
||||
if not reclaimed or not started.is_set() or recovered.is_set():
|
||||
return []
|
||||
recovered.set()
|
||||
return [
|
||||
WorkItem(
|
||||
kind="cascade",
|
||||
node="f.source",
|
||||
flow="f",
|
||||
outputs={"f.raw": float(i)},
|
||||
entry_id=f"recovered-{i}",
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
|
||||
def touch(self, entry_ids):
|
||||
if "recovered-0" in entry_ids:
|
||||
touched.set()
|
||||
|
||||
def slow(raw, params):
|
||||
calls.append(raw)
|
||||
started.set()
|
||||
assert release.wait(5)
|
||||
|
||||
source = _node("source", lambda params: None, provides=["raw"])
|
||||
slow_node = _node("slow", slow, requires=["raw"])
|
||||
timer = _node("timer", lambda params: None, provides=["stop"])
|
||||
stop = _node("stop", lambda stop, params: fired.set(), requires=["stop"])
|
||||
queue = Queue()
|
||||
service = ExecutionService(queue, max_workers=1, max_cascades=1)
|
||||
pipeline = Pipeline(
|
||||
[source, slow_node, timer, stop], work_queue=queue, node_pool=service.node_pool
|
||||
)
|
||||
service.bind(pipeline)
|
||||
monkeypatch.setattr(executor, "DELAYED_INTERVAL_S", 0.02)
|
||||
monkeypatch.setattr(executor, "RECLAIM_INTERVAL_S", 0.02)
|
||||
monkeypatch.setattr(executor, "TOUCH_INTERVAL_S", 0.02)
|
||||
service.start()
|
||||
try:
|
||||
source.inject({"raw": 1.0})
|
||||
assert started.wait(1)
|
||||
if reclaimed:
|
||||
assert touched.wait(1), "reclaimed work blocked the timer's heartbeat"
|
||||
else:
|
||||
for i in range(3):
|
||||
source.inject({"raw": float(i)})
|
||||
queue.add_delayed(
|
||||
WorkItem(kind="cascade", node=timer.id, flow="f", outputs={"f.stop": 1.0}),
|
||||
time.time(),
|
||||
)
|
||||
assert fired.wait(2), "the due node waited for an ordinary worker"
|
||||
assert len(calls) == 1, "ordinary work used the timer reserve"
|
||||
finally:
|
||||
release.set()
|
||||
service.stop()
|
||||
|
||||
|
||||
def test_timers_can_use_all_idle_normal_workers():
|
||||
"""A reserve must not cap an otherwise idle engine at two workers."""
|
||||
service = ExecutionService(MemoryWorkQueue(), max_workers=3, max_cascades=3)
|
||||
started = [threading.Event() for _ in range(3)]
|
||||
release = threading.Event()
|
||||
|
||||
def block(index):
|
||||
started[index].set()
|
||||
assert release.wait(3)
|
||||
|
||||
futures = []
|
||||
try:
|
||||
for i in range(3):
|
||||
futures.append(service._node_pools.due.submit(block, i))
|
||||
assert all(event.wait(1) for event in started)
|
||||
assert service._node_pools.available() == (0, executor.DUE_RESERVE)
|
||||
finally:
|
||||
release.set()
|
||||
for future in futures:
|
||||
future.result(1)
|
||||
service.stop()
|
||||
Reference in New Issue
Block a user