Fix work recovery and reduce engine and chart overhead
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
Docs / docs (push) Successful in 33s
Playwright Tests / test-playwright (1, 2) (push) Failing after 1m37s
Playwright Tests / test-playwright (2, 2) (push) Failing after 27s
pre-commit / pre-commit (push) Failing after 2m37s
Test Backend / test-backend (3.12) (push) Successful in 5m7s
Test Backend / test-backend (3.14) (push) Successful in 5m2s
Compose Smoke Test / test-compose (push) Failing after 28s
Playwright Tests / merge-reports (push) Failing after 2m29s
This commit is contained in:
@@ -53,6 +53,23 @@ STALE_CONSUMER_IDLE_MS = 3_600_000
|
||||
# count, since redelivery happens seconds after the claim and never later.
|
||||
DONE_MARKERS = 10_000
|
||||
|
||||
# A promotion is one Redis operation, including when the caller loses its
|
||||
# connection before hearing the result. Write first: Lua does not roll back a
|
||||
# command error, so a refused XADD must leave its timer in the delayed set.
|
||||
_MOVE_DUE = """
|
||||
local due = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[1], 'LIMIT', 0, 100)
|
||||
for _, raw in ipairs(due) do
|
||||
local fields = {}
|
||||
for key, value in pairs(cjson.decode(raw)) do
|
||||
fields[#fields + 1] = key
|
||||
fields[#fields + 1] = value
|
||||
end
|
||||
redis.call('XADD', KEYS[2], 'MAXLEN', '~', ARGV[2], '*', unpack(fields))
|
||||
redis.call('ZREM', KEYS[1], raw)
|
||||
end
|
||||
return #due
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkItem:
|
||||
@@ -446,6 +463,7 @@ class RedisWorkQueue(WorkQueue):
|
||||
# finding them with a keyspace scan walked every state and idempotency
|
||||
# key in the database, twice per health poll.
|
||||
self._parked_flows_key = f"{namespace}:__parked_flows__"
|
||||
self._move_due = self._redis.register_script(_MOVE_DUE)
|
||||
self._ensure_group()
|
||||
self._drop_stale_consumers()
|
||||
|
||||
@@ -619,36 +637,14 @@ class RedisWorkQueue(WorkQueue):
|
||||
]
|
||||
|
||||
def move_due(self, now: float) -> int:
|
||||
due = cast(
|
||||
list[str],
|
||||
self._redis.zrangebyscore(self._delayed_key, "-inf", now, start=0, num=100),
|
||||
"""Move up to 100 timers into the due lane atomically."""
|
||||
return cast(
|
||||
int,
|
||||
self._move_due(
|
||||
keys=[self._delayed_key, self._due_stream],
|
||||
args=[now, STREAM_MAXLEN],
|
||||
),
|
||||
)
|
||||
if not due:
|
||||
return 0
|
||||
# Whoever removes it owns it: a second engine gets 0 for that member and
|
||||
# leaves it alone. One round trip for the batch rather than one each,
|
||||
# which matters because this runs every second.
|
||||
pipe = self._redis.pipeline()
|
||||
for raw in due:
|
||||
pipe.zrem(self._delayed_key, raw)
|
||||
claimed = cast(list[int], pipe.execute())
|
||||
|
||||
pipe = self._redis.pipeline()
|
||||
moved = 0
|
||||
for raw, owned in zip(due, claimed, strict=True):
|
||||
if owned:
|
||||
# Into the due lane, which `claim` reads first: what has waited
|
||||
# out a deadline should not then queue behind a backlog.
|
||||
pipe.xadd(
|
||||
self._due_stream,
|
||||
cast(Any, orjson.loads(raw)),
|
||||
maxlen=STREAM_MAXLEN,
|
||||
approximate=True,
|
||||
)
|
||||
moved += 1
|
||||
if moved:
|
||||
pipe.execute()
|
||||
return moved
|
||||
|
||||
def dead_letter(self, item: WorkItem, reason: str) -> None:
|
||||
fields = item.to_fields()
|
||||
|
||||
Reference in New Issue
Block a user