Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s

This commit is contained in:
2026-10-29 14:24:17 +01:00
parent 20a0853f69
commit 75b3a93094
25 changed files with 1316 additions and 301 deletions
+92 -1
View File
@@ -2,8 +2,11 @@
import asyncio
import threading
from contextlib import AsyncExitStack
from fluksio.flow.events import QUEUE_SIZE, EventBus
import pytest
from fluksio.flow.events import DISPATCH_BUDGET, QUEUE_SIZE, EventBus
def _event(i: int) -> dict[str, object]:
@@ -100,3 +103,91 @@ def test_nothing_is_buffered_with_nobody_listening() -> None:
return len(bus._pending)
assert asyncio.run(run()) == 0
@pytest.mark.parametrize("flush", [False, True])
def test_overflow_keeps_errors_and_control_events_in_order(flush: bool) -> None:
"""The staging bound and a slow subscriber use the same eviction policy."""
async def run() -> None:
bus = EventBus()
bus.bind(asyncio.get_running_loop())
critical = [
{"type": "node_log", "level": "error"},
{"type": "node_error"},
{"type": "run_started"},
{"type": "audit"},
{"type": "dashboard_changed"},
{"type": "future_control_event"},
]
async with bus.subscribe() as first, bus.subscribe() as second:
sent = [*critical, *(_event(i) for i in range(QUEUE_SIZE + 50))]
sent.append({"type": "run_finished"})
for sequence, event in enumerate(sent):
event["sequence"] = sequence
bus.publish(event)
assert len(bus._pending) <= QUEUE_SIZE
if flush:
# Queue overflow, with no staging overflow.
await asyncio.sleep(0)
await asyncio.sleep(0)
for queue in (first, second):
received = [queue.get_nowait() for _ in range(queue.qsize())]
assert len(received) == QUEUE_SIZE
assert received[: len(critical)] == critical
assert received[-1]["type"] == "run_finished"
assert received == sorted(received, key=lambda e: e["sequence"])
assert bus.dropped(queue)
assert not bus.dropped(queue)
asyncio.run(run())
def test_a_full_critical_backlog_rejects_telemetry_and_reports_loss() -> None:
async def run() -> None:
bus = EventBus()
bus.bind(asyncio.get_running_loop())
async with bus.subscribe() as queue:
for i in range(QUEUE_SIZE + 1):
bus.publish({"type": "node_error", "sequence": i})
bus.publish(_event(0))
assert len(bus._pending) == QUEUE_SIZE
await asyncio.sleep(0)
assert bus.dropped(queue)
assert (await queue.get())["sequence"] == 1
bus.publish({"type": "audit"})
await asyncio.sleep(0)
bus.publish(_event(1))
await asyncio.sleep(0)
assert bus.dropped(queue)
assert queue.qsize() == QUEUE_SIZE
assert all(
queue.get_nowait()["type"] != "node_executed" for _ in range(QUEUE_SIZE)
)
asyncio.run(run())
def test_fanout_yields_before_all_subscribers_are_served() -> None:
async def run() -> None:
bus = EventBus()
loop = asyncio.get_running_loop()
bus.bind(loop)
async with AsyncExitStack() as stack:
queues = [
await stack.enter_async_context(bus.subscribe())
for _ in range(DISPATCH_BUDGET // QUEUE_SIZE + 1)
]
for i in range(QUEUE_SIZE):
bus.publish(_event(i))
heartbeat: asyncio.Future[int] = loop.create_future()
loop.call_soon(
lambda: heartbeat.set_result(sum(queue.qsize() for queue in queues))
)
assert await heartbeat == DISPATCH_BUDGET
await asyncio.sleep(0)
assert all(queue.qsize() == QUEUE_SIZE for queue in queues)
assert not any(bus.dropped(queue) for queue in queues)
assert not bus._flush_scheduled
asyncio.run(run())
+42
View File
@@ -242,6 +242,48 @@ def test_a_trigger_is_journaled_rather_than_run_on_the_spot():
assert state["f.doubled"] == 6.0
def test_a_state_failure_leaves_the_delivery_unacknowledged(monkeypatch):
pipeline, source, state, seen = _pipeline_with_a_consumer()
service = ExecutionService(pipeline._queue)
service.bind(pipeline)
def unavailable(*args, **kwargs):
raise ConnectionError("State unavailable")
monkeypatch.setattr(state, "record", unavailable)
source.inject({"reading": 3.0})
item = pipeline._queue.claim(1, 10)[0]
try:
service._dispatch(item)
service.pause_intake()
assert seen == []
assert pipeline._queue.stats()["pending"] == 1
assert service.inflight == 0
finally:
service.stop()
def test_a_node_failure_still_acknowledges_its_delivery(monkeypatch):
pipeline, source, _state, seen = _pipeline_with_a_consumer()
service = ExecutionService(pipeline._queue)
service.bind(pipeline)
def broken(*args, **kwargs):
raise ValueError("Node failed")
monkeypatch.setattr(pipeline.get_node_by_id("f.consumer"), "f", broken)
source.inject({"reading": 3.0})
item = pipeline._queue.claim(1, 10)[0]
try:
service._dispatch(item)
service.pause_intake()
assert seen == []
assert pipeline._queue.stats()["pending"] == 0
assert service.inflight == 0
finally:
service.stop()
def test_work_for_a_paused_flow_is_held_and_released_on_resume():
pipeline, source, state, seen = _pipeline_with_a_consumer()
service = ExecutionService(pipeline._queue)
+68
View File
@@ -0,0 +1,68 @@
"""Redis delivery checks, enabled by FLUKSIO_TEST_REDIS_HOST (and optional PORT)."""
import os
import uuid
from concurrent.futures import ThreadPoolExecutor
import pytest
import redis
from fluksio.flow.queue import RedisWorkQueue, WorkItem
@pytest.fixture
def queue():
host = os.environ.get("FLUKSIO_TEST_REDIS_HOST")
if not host:
pytest.skip("Set FLUKSIO_TEST_REDIS_HOST to run Redis integration checks")
queue = RedisWorkQueue(
host=host,
port=int(os.environ.get("FLUKSIO_TEST_REDIS_PORT", "6379")),
namespace=f"test-queue-{uuid.uuid4().hex}",
)
try:
yield queue
finally:
keys = list(queue._redis.scan_iter(match=f"{queue._ns}:*"))
if keys:
queue._redis.delete(*keys)
queue.close()
def test_concurrent_promotions_deliver_each_due_item_once(queue):
for i in range(150):
queue.add_delayed(
WorkItem(kind="cascade", node=f"f.n{i}", flow="f", outputs={"f.x": i}),
100.0,
)
queue.add_delayed(WorkItem(kind="cascade", node="f.later", flow="f"), 300.0)
with ThreadPoolExecutor(max_workers=2) as pool:
moved = list(pool.map(queue.move_due, [200.0, 200.0]))
assert sorted(moved) == [50, 100]
assert queue.move_due(200.0) == 0
assert queue.next_due() == 300.0
claimed = queue.claim(200, 10, due_only=True)
assert len(claimed) == 150
assert {item.outputs["f.x"] for item in claimed} == set(range(150))
assert all(item.not_before == 100.0 for item in claimed)
assert all(item.entry_id.startswith("due:") for item in claimed)
def test_a_refused_stream_write_keeps_the_timer_for_retry(queue):
queue.add_delayed(WorkItem(kind="cascade", node="f.stop", flow="f"), 100.0)
# A Redis command error does not roll a Lua script back. The stream write
# must succeed before the delayed entry is removed.
queue._redis.delete(queue._due_stream)
queue._redis.set(queue._due_stream, "not a stream")
with pytest.raises(redis.ResponseError, match="WRONGTYPE"):
queue.move_due(200.0)
assert queue.next_due() == 100.0
queue._redis.delete(queue._due_stream)
queue._ensure_group()
assert queue.move_due(200.0) == 1
assert queue.move_due(200.0) == 0
assert [item.node for item in queue.claim(10, 10, due_only=True)] == ["f.stop"]
+21
View File
@@ -122,3 +122,24 @@ def test_a_hop_costs_a_bounded_number_of_state_operations() -> None:
total = sum(state.calls[name] for name in COUNTED)
assert total <= 8, f"state operations per hop grew: {dict(state.calls)}"
def test_reading_one_value_does_not_list_or_read_other_messages(monkeypatch) -> None:
state = MemoryState()
state.update({f"f.m{i}": i for i in range(1000)})
state.update({"f.m0": None, "__ts__:f.m0": 123.0})
pipeline = Pipeline([], state=state)
reads = []
get_multi = state.get_multi
def read(keys):
reads.append(keys)
return get_multi(keys)
def no_listing():
raise AssertionError("A single-message read must not enumerate state")
monkeypatch.setattr(MemoryState, "get_multi", lambda self, keys: read(keys))
monkeypatch.setattr(MemoryState, "message_names", lambda self: no_listing())
assert pipeline.value("f.m0") == {"value": None, "ts": 123.0}
assert reads == [["f.m0", "__ts__:f.m0"]]
+84
View File
@@ -0,0 +1,84 @@
"""Scheduling work scales with the graph and observes runtime gates."""
import threading
from concurrent.futures import ThreadPoolExecutor
from contextlib import nullcontext
import pytest
from fluksio.flow.messages import DType, MessageSpec
from fluksio.flow.nodes import Node
from fluksio.flow.pipeline import Pipeline
def _node(name, function, requires=(), provides=(), flow="f"):
node = Node(
f=function,
requires=[MessageSpec(name=key, dtype=DType.FLOAT) for key in requires],
provides=[MessageSpec(name=key, dtype=DType.FLOAT) for key in provides],
name=name,
)
node.assign_flow(flow, name)
return node
def test_a_long_chain_checks_each_candidate_once():
class CountingPipeline(Pipeline):
checks = 0
def _gate_blocks(self, node):
self.checks += 1
return super()._gate_blocks(node)
nodes = []
for i in range(100):
def relay(params, _index=i, **inputs):
return {f"v{_index + 1}": inputs[f"v{_index}"] + 1.0}
nodes.append(_node(f"n{i}", relay, requires=[f"v{i}"], provides=[f"v{i + 1}"]))
pipeline = CountingPipeline(nodes)
pipeline.run({"f.v0": 0.0})
assert pipeline.values()["f.v100"]["value"] == 100.0
assert pipeline.checks == len(nodes)
@pytest.mark.parametrize("disabled", [False, True])
def test_a_gate_opened_mid_wave_releases_its_waiting_candidate(disabled):
running, release, held, acted = (threading.Event() for _ in range(4))
class WatchingPipeline(Pipeline):
def _gate_blocks(self, node):
blocked = super()._gate_blocks(node)
if blocked:
held.set()
return blocked
def slow(params):
running.set()
assert release.wait(3)
pipeline = WatchingPipeline(
[
_node("slow", slow, flow="running"),
_node("action", lambda params: acted.set(), flow="paused"),
]
)
if disabled:
pipeline.set_disabled({"paused"})
else:
pipeline.pause("paused")
with ThreadPoolExecutor(max_workers=1) as driver:
future = driver.submit(pipeline.run)
try:
assert running.wait(1) and held.wait(1)
if disabled:
pipeline.set_disabled(set())
with nullcontext() if disabled else pipeline.stepping("paused"):
release.set()
future.result(2)
assert acted.is_set()
finally:
release.set()
+129
View File
@@ -0,0 +1,129 @@
"""Direct and portal browsers recover state without widening camera access."""
import asyncio
import json
from contextlib import suppress
from types import SimpleNamespace
import pytest
from fluksio.api import deps
from fluksio.api.routes import flows
from fluksio.cloud.connector import CloudConnector
from fluksio.flow import events
from fluksio.flow.events import QUEUE_SIZE, EventBus
@pytest.mark.parametrize("tunnel", [False, True])
@pytest.mark.parametrize("burst", [0, QUEUE_SIZE + 1])
def test_a_browser_catches_changes_during_its_snapshot(monkeypatch, tunnel, burst):
async def run():
bus = EventBus()
bus.bind(asyncio.get_running_loop())
monkeypatch.setattr(events, "event_bus", bus)
monkeypatch.setattr(flows, "event_bus", bus)
user = SimpleNamespace(id="viewer", is_superuser=False)
for module in (flows, deps):
monkeypatch.setattr(module, "user_from_token", lambda *a, **kw: user)
monkeypatch.setattr(
module, "hidden_from", lambda *a, **kw: frozenset({"f.camera"})
)
monkeypatch.setattr(flows, "panel_scope", lambda *a: None)
monkeypatch.setattr(flows, "socket_token", lambda *a: "test-token")
async def immediate(function, *args):
return function(*args)
monkeypatch.setattr(flows, "run_in_threadpool", immediate)
current = {"f.flag": {"value": 0}, "f.camera": {"value": "private"}}
controller = SimpleNamespace(
values=lambda: dict(current),
node_statuses=lambda: [],
issues=[],
paused_flows=lambda: [],
placer=None,
)
app = SimpleNamespace(state=SimpleNamespace(flow_controller=controller))
received = []
client_values = {}
caught_up = asyncio.Event()
drained = asyncio.Event()
class Socket:
headers = {}
async def accept(self):
pass
async def receive(self):
await asyncio.Future()
async def send(self, text):
frame = json.loads(text)
if frame["op"] == "ws_msg":
await self.send_text(frame["text"])
async def send_text(self, text):
payload = json.loads(text)
received.append(payload)
if payload["type"] == "snapshot":
assert "f.camera" not in payload["values"]
client_values.update(payload["values"])
if payload["values"]["f.flag"]["value"] == 1:
caught_up.set()
elif len(received) == 1:
# The state was read, but its frame is still sending.
# This value changes only once, then a burst loses its
# event and leaves only a snapshot able to recover it.
current["f.flag"] = {"value": 1}
bus.publish(
{"type": "message_value", "name": "f.flag", "value": 1}
)
for i in range(burst):
current["f.noise"] = {"value": i}
bus.publish(
{"type": "message_value", "name": "f.noise", "value": i}
)
# Recovery must forward retained critical events too.
bus.publish({"type": "node_error", "node": "f.example"})
await asyncio.sleep(0)
for event in payload.get("events", [payload]):
if event["type"] == "message_value":
client_values[event["name"]] = {"value": event["value"]}
if event.get("name") == "f.flag" and event.get("value") == 1:
caught_up.set()
if event["type"] == "node_error":
drained.set()
socket = Socket()
socket.app = app
if tunnel:
action = CloudConnector(app)._stream(
socket,
{"id": "review", "path": "/api/v1/flows/ws", "query": "token=test"},
)
else:
action = flows.flow_events(socket)
task = asyncio.create_task(action)
try:
await asyncio.wait_for(caught_up.wait(), timeout=2)
await asyncio.wait_for(drained.wait(), timeout=2)
finally:
task.cancel()
with suppress(asyncio.CancelledError):
await task
assert sum(item["type"] == "snapshot" for item in received) == (
2 if burst else 1
)
flattened = [
event for payload in received for event in payload.get("events", [payload])
]
assert sum(event["type"] == "node_error" for event in flattened) == 1
if burst:
# The client replaces values in arrival order. A snapshot ahead
# of the retained backlog temporarily rolls values backwards.
assert flattened[-1]["type"] == "snapshot"
assert client_values == {k: v for k, v in current.items() if k != "f.camera"}
assert not bus._subscribers
asyncio.run(run())
+112
View File
@@ -0,0 +1,112 @@
"""Deadlines retain worker capacity when ordinary work is blocked."""
import threading
import time
import pytest
from fluksio.flow import executor
from fluksio.flow.executor import ExecutionService
from fluksio.flow.messages import DType, MessageSpec
from fluksio.flow.nodes import Node
from fluksio.flow.pipeline import Pipeline
from fluksio.flow.queue import MemoryWorkQueue, WorkItem
def _node(name, function, requires=(), provides=()):
node = Node(
f=function,
requires=[MessageSpec(name=key, dtype=DType.FLOAT) for key in requires],
provides=[MessageSpec(name=key, dtype=DType.FLOAT) for key in provides],
name=name,
)
node.assign_flow("f", name)
return node
@pytest.mark.parametrize("reclaimed", [False, True])
def test_a_due_node_runs_while_ordinary_nodes_and_cascades_are_full(
monkeypatch, reclaimed
):
started, release, fired, recovered, touched = (threading.Event() for _ in range(5))
calls = []
class Queue(MemoryWorkQueue):
def reclaim_stale(self, min_idle_ms):
if not reclaimed or not started.is_set() or recovered.is_set():
return []
recovered.set()
return [
WorkItem(
kind="cascade",
node="f.source",
flow="f",
outputs={"f.raw": float(i)},
entry_id=f"recovered-{i}",
)
for i in range(3)
]
def touch(self, entry_ids):
if "recovered-0" in entry_ids:
touched.set()
def slow(raw, params):
calls.append(raw)
started.set()
assert release.wait(5)
source = _node("source", lambda params: None, provides=["raw"])
slow_node = _node("slow", slow, requires=["raw"])
timer = _node("timer", lambda params: None, provides=["stop"])
stop = _node("stop", lambda stop, params: fired.set(), requires=["stop"])
queue = Queue()
service = ExecutionService(queue, max_workers=1, max_cascades=1)
pipeline = Pipeline(
[source, slow_node, timer, stop], work_queue=queue, node_pool=service.node_pool
)
service.bind(pipeline)
monkeypatch.setattr(executor, "DELAYED_INTERVAL_S", 0.02)
monkeypatch.setattr(executor, "RECLAIM_INTERVAL_S", 0.02)
monkeypatch.setattr(executor, "TOUCH_INTERVAL_S", 0.02)
service.start()
try:
source.inject({"raw": 1.0})
assert started.wait(1)
if reclaimed:
assert touched.wait(1), "reclaimed work blocked the timer's heartbeat"
else:
for i in range(3):
source.inject({"raw": float(i)})
queue.add_delayed(
WorkItem(kind="cascade", node=timer.id, flow="f", outputs={"f.stop": 1.0}),
time.time(),
)
assert fired.wait(2), "the due node waited for an ordinary worker"
assert len(calls) == 1, "ordinary work used the timer reserve"
finally:
release.set()
service.stop()
def test_timers_can_use_all_idle_normal_workers():
"""A reserve must not cap an otherwise idle engine at two workers."""
service = ExecutionService(MemoryWorkQueue(), max_workers=3, max_cascades=3)
started = [threading.Event() for _ in range(3)]
release = threading.Event()
def block(index):
started[index].set()
assert release.wait(3)
futures = []
try:
for i in range(3):
futures.append(service._node_pools.due.submit(block, i))
assert all(event.wait(1) for event in started)
assert service._node_pools.available() == (0, executor.DUE_RESERVE)
finally:
release.set()
for future in futures:
future.result(1)
service.stop()
+55
View File
@@ -160,6 +160,61 @@ def test_a_flush_the_database_refused_is_written_by_the_next_one(
assert audit.detail == "held back"
def test_batched_rollups_add_and_redelivery_clears_a_finished_run(db: Session) -> None:
collector = MetricsCollector(EventBus())
ts = time.time()
for i in range(2):
collector.handle(
{"type": "cascade_started", "run": f"bulk-{i}", "flow": "bulk", "ts": ts}
)
collector.handle(
{
"type": "node_executed",
"flow": "bulk",
"node": f"bulk.n{i}",
"run": f"bulk-{i}",
"outputs": 1,
"duration_ms": 10.0,
"ts": ts,
}
)
collector.handle({"type": "cascade_finished", "run": "bulk-0", "ts": ts + 1})
# Write the same open/finished batch twice, as a retained run is upserted.
collector._write(collector._buckets, [], list(collector._runs.values()), False)
collector.handle(
{
"type": "cascade_started",
"run": "bulk-0",
"flow": "bulk",
"deliveries": 2,
"ts": ts + 2,
}
)
collector.handle({"type": "cascade_finished", "run": "bulk-1", "ts": ts + 3})
for bucket in collector._buckets.values():
bucket["duration_sum_ms"] = 4.0
bucket["duration_max_ms"] = 4.0
collector._write(collector._buckets, [], list(collector._runs.values()), False)
buckets = db.exec(select(MetricBucket).where(MetricBucket.flow == "bulk")).all()
assert len(buckets) == 2
assert all(
(b.executions, b.messages, b.duration_sum_ms, b.duration_max_ms)
== (2, 2, 14.0, 10.0)
for b in buckets
)
reopened = db.get(FlowRun, "bulk-0")
assert reopened is not None
assert (reopened.status, reopened.finished_at, reopened.deliveries) == (
"running",
None,
2,
)
finished = db.get(FlowRun, "bulk-1")
assert finished is not None and finished.finished_at is not None
assert finished.status == "ok"
def test_a_node_id_wider_than_the_column_still_records(db: Session) -> None:
collector = MetricsCollector(EventBus())
ts = datetime.now(UTC).replace(second=0, microsecond=0).timestamp()
+52
View File
@@ -1,7 +1,9 @@
import base64
import gzip
import hashlib
from pathlib import Path
import pytest
from starlette.applications import Starlette
from starlette.responses import PlainTextResponse
from starlette.routing import Route
@@ -37,3 +39,53 @@ def test_the_dashboard_and_the_api_share_one_origin(tmp_path: Path) -> None:
assert client.get("/api/v1/ping").text == "api"
# Nothing outside the build directory is served, only the shell.
assert THEME in client.get("/assets/../../etc/passwd").text
@pytest.mark.parametrize(
("accepted", "compressed"),
[
("gzip", True),
("br, gzip;q=0.8", True),
("gzip ; q=1, identity;q=0", True),
("*", True),
("gzip;q=0, *;q=1", False),
("gzip;q=0.5, identity;q=1", False),
("br", False),
("identity", False),
],
)
def test_precompressed_assets_keep_their_type_and_negotiate_encoding(
tmp_path: Path, accepted: str, compressed: bool
) -> None:
client = _client(tmp_path)
content = b"console.log('built once');\n" * 200
asset = tmp_path / "assets" / "app-1a2b.js"
asset.write_bytes(content)
packed = gzip.compress(content)
asset.with_name(asset.name + ".gz").write_bytes(packed)
response = client.get("/assets/app-1a2b.js", headers={"Accept-Encoding": accepted})
assert response.content == content
assert (response.headers.get("content-encoding") == "gzip") is compressed
assert "javascript" in response.headers["content-type"]
assert response.headers["vary"] == "Accept-Encoding"
assert "immutable" in response.headers["cache-control"]
expected = len(packed) if compressed else len(content)
assert int(response.headers["content-length"]) == expected
head = client.head("/assets/app-1a2b.js", headers={"Accept-Encoding": accepted})
assert head.content == b""
assert head.headers["content-length"] == str(expected)
def test_a_build_without_compressed_assets_still_serves_plain_files(
tmp_path: Path,
) -> None:
client = _client(tmp_path)
content = b"console.log('plain build');\n" * 200
(tmp_path / "assets" / "app-1a2b.js").write_bytes(content)
response = client.get("/assets/app-1a2b.js", headers={"Accept-Encoding": "gzip"})
assert response.content == content
assert "content-encoding" not in response.headers