Prove the rename's stream rebuild on a real Redis: same ids, group, lag and pending entries after RENAME, ENTRIESREAD set after XSETID because Redis caps it; backend CI gets a Redis 8.2 service
This commit is contained in:
@@ -25,10 +25,18 @@ jobs:
|
|||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
python-version: ["3.12", "3.14"]
|
python-version: ["3.12", "3.14"]
|
||||||
|
# The Redis queue and stream tests run against a real server. Same major
|
||||||
|
# version as the shipped image in docker/compose.yml.
|
||||||
|
services:
|
||||||
|
redis:
|
||||||
|
image: redis:8.2-alpine
|
||||||
env:
|
env:
|
||||||
# uv would otherwise take a managed interpreter it already has over the
|
# uv would otherwise take a managed interpreter it already has over the
|
||||||
# one set up below.
|
# one set up below.
|
||||||
UV_PYTHON: ${{ matrix.python-version }}
|
UV_PYTHON: ${{ matrix.python-version }}
|
||||||
|
# The job runs in a container next to the service, which answers to its
|
||||||
|
# name on the default port.
|
||||||
|
FLUKSIO_TEST_REDIS_HOST: redis
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
@@ -46,9 +54,10 @@ jobs:
|
|||||||
# The extras too, as `make test-backend` does: the mail, MQTT and Influx
|
# The extras too, as `make test-backend` does: the mail, MQTT and Influx
|
||||||
# tests import what only the `server` extra installs.
|
# tests import what only the `server` extra installs.
|
||||||
- run: uv sync --all-extras
|
- run: uv sync --all-extras
|
||||||
# Nothing to start: the suite names its own SQLite file in a temp
|
# Nothing else to start: the suite names its own SQLite file in a temp
|
||||||
# directory (backend/tests/__init__.py), the mail paths are patched, and
|
# directory (backend/tests/__init__.py), the mail paths are patched, and
|
||||||
# the flow engine falls back to in-memory state without a Redis host.
|
# the flow engine falls back to in-memory state without a Redis host.
|
||||||
|
# Only the tests gated on FLUKSIO_TEST_REDIS_HOST use the service.
|
||||||
- name: Run tests with coverage
|
- name: Run tests with coverage
|
||||||
run: uv run bash scripts/test.sh "Coverage for ${{ github.sha }}"
|
run: uv run bash scripts/test.sh "Coverage for ${{ github.sha }}"
|
||||||
working-directory: backend
|
working-directory: backend
|
||||||
|
|||||||
@@ -0,0 +1,239 @@
|
|||||||
|
"""Stream rebuild checks for the rename's queue migration, enabled by
|
||||||
|
FLUKSIO_TEST_REDIS_HOST (and optional PORT).
|
||||||
|
|
||||||
|
They prove on a real server that a stream copied into a temp key with its ids,
|
||||||
|
group and pending entries and swapped in with RENAME looks like the original to
|
||||||
|
the engine (docs/private/id_refac_rename.md, 3.1).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import redis
|
||||||
|
|
||||||
|
GROUP = "engine"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def ns():
|
||||||
|
return f"test-streams-{uuid.uuid4().hex}"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def r(ns):
|
||||||
|
host = os.environ.get("FLUKSIO_TEST_REDIS_HOST")
|
||||||
|
if not host:
|
||||||
|
pytest.skip("Set FLUKSIO_TEST_REDIS_HOST to run Redis integration checks")
|
||||||
|
client = redis.Redis(
|
||||||
|
host=host,
|
||||||
|
port=int(os.environ.get("FLUKSIO_TEST_REDIS_PORT", "6379")),
|
||||||
|
decode_responses=True,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
yield client
|
||||||
|
finally:
|
||||||
|
keys = list(client.scan_iter(match=f"{ns}:*"))
|
||||||
|
if keys:
|
||||||
|
client.delete(*keys)
|
||||||
|
client.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _id(entry_id: str) -> tuple[int, int]:
|
||||||
|
ms, seq = entry_id.split("-")
|
||||||
|
return int(ms), int(seq)
|
||||||
|
|
||||||
|
|
||||||
|
def _rename_flow(fields: dict[str, str]) -> dict[str, str]:
|
||||||
|
return {**fields, "flow": "new"} if fields.get("flow") == "old" else fields
|
||||||
|
|
||||||
|
|
||||||
|
# ponytail: unpaged. S3 moves it into RedisWorkQueue and pages XRANGE and XPENDING.
|
||||||
|
def rebuild(r, stream: str, rewrite, *, swap: bool = True) -> None:
|
||||||
|
"""Copy *stream* into a temp key with the same ids, group and pending
|
||||||
|
entries, rewriting the entries still to run, and swap it in."""
|
||||||
|
tmp = f"{stream}:rename"
|
||||||
|
r.delete(tmp)
|
||||||
|
info = r.xinfo_stream(stream)
|
||||||
|
(group,) = [g for g in r.xinfo_groups(stream) if g["name"] == GROUP]
|
||||||
|
last = group["last-delivered-id"]
|
||||||
|
pel = (
|
||||||
|
r.xpending_range(stream, GROUP, "-", "+", count=group["pending"])
|
||||||
|
if group["pending"]
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
pending = {p["message_id"] for p in pel}
|
||||||
|
|
||||||
|
pipe = r.pipeline(transaction=False)
|
||||||
|
# MKSTREAM so XSETID finds the key even when nothing is copied.
|
||||||
|
pipe.xgroup_create(tmp, GROUP, id=last, mkstream=True)
|
||||||
|
for consumer in r.xinfo_consumers(stream, GROUP):
|
||||||
|
pipe.xgroup_createconsumer(tmp, GROUP, consumer["name"])
|
||||||
|
copied = set()
|
||||||
|
for entry_id, fields in r.xrange(stream):
|
||||||
|
live = entry_id in pending or _id(entry_id) > _id(last)
|
||||||
|
pipe.xadd(tmp, rewrite(fields) if live else fields, id=entry_id)
|
||||||
|
copied.add(entry_id)
|
||||||
|
pipe.execute_command(
|
||||||
|
"XSETID",
|
||||||
|
tmp,
|
||||||
|
info["last-generated-id"],
|
||||||
|
"ENTRIESADDED",
|
||||||
|
info["entries-added"],
|
||||||
|
"MAXDELETEDID",
|
||||||
|
info["max-deleted-entry-id"],
|
||||||
|
)
|
||||||
|
# Redis caps ENTRIESREAD at the stream's entries-added counter, which is
|
||||||
|
# still the copied count before XSETID, so it is set only now.
|
||||||
|
if group["entries-read"] is not None:
|
||||||
|
pipe.execute_command(
|
||||||
|
"XGROUP", "SETID", tmp, GROUP, last, "ENTRIESREAD", group["entries-read"]
|
||||||
|
)
|
||||||
|
for p in pel:
|
||||||
|
if p["message_id"] in copied:
|
||||||
|
pipe.xclaim(
|
||||||
|
tmp,
|
||||||
|
GROUP,
|
||||||
|
p["consumer"],
|
||||||
|
0,
|
||||||
|
[p["message_id"]],
|
||||||
|
idle=p["time_since_delivered"],
|
||||||
|
retrycount=p["times_delivered"],
|
||||||
|
force=True,
|
||||||
|
justid=True,
|
||||||
|
)
|
||||||
|
if swap:
|
||||||
|
pipe.rename(tmp, stream)
|
||||||
|
pipe.execute()
|
||||||
|
|
||||||
|
|
||||||
|
def _pel(r, stream: str) -> list[dict]:
|
||||||
|
return r.xpending_range(stream, GROUP, "-", "+", count=100)
|
||||||
|
|
||||||
|
|
||||||
|
def _snapshot(r, stream: str) -> dict:
|
||||||
|
info = r.xinfo_stream(stream)
|
||||||
|
(group,) = r.xinfo_groups(stream)
|
||||||
|
return {
|
||||||
|
"entries": r.xrange(stream),
|
||||||
|
"stream": {
|
||||||
|
k: info[k]
|
||||||
|
for k in (
|
||||||
|
"length",
|
||||||
|
"last-generated-id",
|
||||||
|
"entries-added",
|
||||||
|
"max-deleted-entry-id",
|
||||||
|
"recorded-first-entry-id",
|
||||||
|
)
|
||||||
|
},
|
||||||
|
"group": {
|
||||||
|
k: group[k] for k in ("last-delivered-id", "entries-read", "lag", "pending")
|
||||||
|
},
|
||||||
|
"pel": [
|
||||||
|
(p["message_id"], p["consumer"], p["times_delivered"])
|
||||||
|
for p in _pel(r, stream)
|
||||||
|
],
|
||||||
|
"consumers": sorted(c["name"] for c in r.xinfo_consumers(stream, GROUP)),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _busy_stream(r, stream: str) -> list[str]:
|
||||||
|
"""Nine entries: 0 acked, 1 and 2 pending with a (1 delivered three times),
|
||||||
|
3 to 5 pending with b (3 deleted from the stream, 4 delivered twice), 6 to 8
|
||||||
|
undelivered. a's entries are idle 300 ms longer than b's."""
|
||||||
|
r.xgroup_create(stream, GROUP, id="0", mkstream=True)
|
||||||
|
ids = [r.xadd(stream, {"flow": "old", "n": str(i)}) for i in range(9)]
|
||||||
|
r.xreadgroup(GROUP, "a", {stream: ">"}, count=3)
|
||||||
|
r.xack(stream, GROUP, ids[0])
|
||||||
|
for _ in range(2):
|
||||||
|
r.xclaim(stream, GROUP, "a", 0, [ids[1]])
|
||||||
|
time.sleep(0.3)
|
||||||
|
r.xreadgroup(GROUP, "b", {stream: ">"}, count=3)
|
||||||
|
r.xclaim(stream, GROUP, "b", 0, [ids[4]])
|
||||||
|
r.xdel(stream, ids[3])
|
||||||
|
return ids
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebuild_keeps_ids_group_and_pending_entries(r, ns):
|
||||||
|
stream = f"{ns}:queue"
|
||||||
|
ids = _busy_stream(r, stream)
|
||||||
|
before = _snapshot(r, stream)
|
||||||
|
idle_before = {p["message_id"]: p["time_since_delivered"] for p in _pel(r, stream)}
|
||||||
|
started = time.monotonic()
|
||||||
|
|
||||||
|
rebuild(r, stream, _rename_flow)
|
||||||
|
|
||||||
|
idle_after = {p["message_id"]: p["time_since_delivered"] for p in _pel(r, stream)}
|
||||||
|
elapsed_ms = (time.monotonic() - started) * 1000
|
||||||
|
after = _snapshot(r, stream)
|
||||||
|
assert [i for i, _ in after["entries"]] == [i for i, _ in before["entries"]]
|
||||||
|
assert [f["flow"] for _, f in after["entries"]] == ["old"] + ["new"] * 7
|
||||||
|
assert after["stream"] == before["stream"]
|
||||||
|
assert before["group"]["lag"] == 3
|
||||||
|
# The pending entry whose stream entry is gone is not recreated.
|
||||||
|
assert after["group"] == {**before["group"], "pending": 4}
|
||||||
|
assert after["pel"] == [
|
||||||
|
(ids[1], "a", 3),
|
||||||
|
(ids[2], "a", 1),
|
||||||
|
(ids[4], "b", 2),
|
||||||
|
(ids[5], "b", 1),
|
||||||
|
]
|
||||||
|
assert after["consumers"] == ["a", "b"]
|
||||||
|
assert min(idle_before[ids[1]], idle_before[ids[2]]) >= 300
|
||||||
|
for entry_id, idle in idle_after.items():
|
||||||
|
assert idle_before[entry_id] <= idle <= idle_before[entry_id] + elapsed_ms + 5
|
||||||
|
|
||||||
|
new_id = r.xadd(stream, {"flow": "new", "n": "9"})
|
||||||
|
assert _id(new_id) > _id(before["stream"]["last-generated-id"])
|
||||||
|
((_, delivered),) = r.xreadgroup(GROUP, "c", {stream: ">"})
|
||||||
|
assert [i for i, _ in delivered] == ids[6:] + [new_id]
|
||||||
|
assert not r.exists(f"{stream}:rename")
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebuild_of_an_emptied_stream_keeps_its_id_counter(r, ns):
|
||||||
|
stream = f"{ns}:due"
|
||||||
|
# Ids ahead of the clock, so only XSETID keeps the next `*` id above them.
|
||||||
|
ahead = int(time.time() * 1000) + 86_400_000
|
||||||
|
r.xgroup_create(stream, GROUP, id="0", mkstream=True)
|
||||||
|
ids = [r.xadd(stream, {"flow": "old"}, id=f"{ahead}-{i}") for i in range(3)]
|
||||||
|
r.xreadgroup(GROUP, "a", {stream: ">"}, count=2)
|
||||||
|
r.xack(stream, GROUP, *ids[:2])
|
||||||
|
r.xdel(stream, *ids)
|
||||||
|
before = _snapshot(r, stream)
|
||||||
|
|
||||||
|
rebuild(r, stream, _rename_flow)
|
||||||
|
|
||||||
|
assert _snapshot(r, stream) == before
|
||||||
|
assert _id(r.xadd(stream, {"flow": "new"})) > _id(ids[-1])
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_rerun_after_a_crash_on_either_side_of_rename_converges(r, ns):
|
||||||
|
stream = f"{ns}:queue"
|
||||||
|
_busy_stream(r, stream)
|
||||||
|
original = r.xrange(stream)
|
||||||
|
|
||||||
|
# Crash before RENAME: the stream is untouched and the temp key is left.
|
||||||
|
rebuild(r, stream, _rename_flow, swap=False)
|
||||||
|
assert r.xrange(stream) == original
|
||||||
|
assert r.exists(f"{stream}:rename")
|
||||||
|
|
||||||
|
rebuild(r, stream, _rename_flow)
|
||||||
|
once = _snapshot(r, stream)
|
||||||
|
# Crash after RENAME, before the journal checkpoint: the rerun changes nothing.
|
||||||
|
rebuild(r, stream, _rename_flow)
|
||||||
|
assert _snapshot(r, stream) == once
|
||||||
|
assert not r.exists(f"{stream}:rename")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("command", ["rename", "renamenx"])
|
||||||
|
def test_rename_keeps_the_ttl_of_strings_and_lists(r, ns, command):
|
||||||
|
r.set(f"{ns}:string", "v", ex=600)
|
||||||
|
r.rpush(f"{ns}:list", "a", "b")
|
||||||
|
r.expire(f"{ns}:list", 600)
|
||||||
|
|
||||||
|
for kind in ("string", "list"):
|
||||||
|
assert getattr(r, command)(f"{ns}:{kind}", f"{ns}:{kind}.new")
|
||||||
|
assert not r.exists(f"{ns}:{kind}")
|
||||||
|
assert 590 < r.ttl(f"{ns}:{kind}.new") <= 600
|
||||||
|
assert r.lrange(f"{ns}:list.new", 0, -1) == ["a", "b"]
|
||||||
Reference in New Issue
Block a user