Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s

This commit is contained in:
2026-10-29 14:24:17 +01:00
parent 20a0853f69
commit 75b3a93094
25 changed files with 1316 additions and 301 deletions
+25 -29
View File
@@ -53,6 +53,23 @@ STALE_CONSUMER_IDLE_MS = 3_600_000
# count, since redelivery happens seconds after the claim and never later.
DONE_MARKERS = 10_000
# A promotion is one Redis operation, including when the caller loses its
# connection before hearing the result. Write first: Lua does not roll back a
# command error, so a refused XADD must leave its timer in the delayed set.
_MOVE_DUE = """
local due = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[1], 'LIMIT', 0, 100)
for _, raw in ipairs(due) do
local fields = {}
for key, value in pairs(cjson.decode(raw)) do
fields[#fields + 1] = key
fields[#fields + 1] = value
end
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[2], '*', unpack(fields))
redis.call('ZREM', KEYS[1], raw)
end
return #due
"""
@dataclass
class WorkItem:
@@ -446,6 +463,7 @@ class RedisWorkQueue(WorkQueue):
# finding them with a keyspace scan walked every state and idempotency
# key in the database, twice per health poll.
self._parked_flows_key = f"{namespace}:__parked_flows__"
self._move_due = self._redis.register_script(_MOVE_DUE)
self._ensure_group()
self._drop_stale_consumers()
@@ -619,36 +637,14 @@ class RedisWorkQueue(WorkQueue):
]
def move_due(self, now: float) -> int:
due = cast(
list[str],
self._redis.zrangebyscore(self._delayed_key, "-inf", now, start=0, num=100),
"""Move up to 100 timers into the due lane atomically."""
return cast(
int,
self._move_due(
keys=[self._delayed_key, self._due_stream],
args=[now, STREAM_MAXLEN],
),
)
if not due:
return 0
# Whoever removes it owns it: a second engine gets 0 for that member and
# leaves it alone. One round trip for the batch rather than one each,
# which matters because this runs every second.
pipe = self._redis.pipeline()
for raw in due:
pipe.zrem(self._delayed_key, raw)
claimed = cast(list[int], pipe.execute())
pipe = self._redis.pipeline()
moved = 0
for raw, owned in zip(due, claimed, strict=True):
if owned:
# Into the due lane, which `claim` reads first: what has waited
# out a deadline should not then queue behind a backlog.
pipe.xadd(
self._due_stream,
cast(Any, orjson.loads(raw)),
maxlen=STREAM_MAXLEN,
approximate=True,
)
moved += 1
if moved:
pipe.execute()
return moved
def dead_letter(self, item: WorkItem, reason: str) -> None:
fields = item.to_fields()