Say how much work is waiting, not just how much is running

`RedisWorkQueue.stats` read XPENDING, which counts entries delivered to a
consumer and not yet acknowledged — work in progress. Entries sitting in the
stream undelivered were counted nowhere, so an engine hours behind reported
itself idle: on the house, `pending: 4` while the group's lag was 1554.

The group's own `lag` is the missing number. `backlog` now carries it on both
queues (`len(_items)` in memory), leads the health tile, and a sustained one
publishes `engine_degraded` from the timer thread — named with the flow most
of the waiting work belongs to, sampled from the undelivered tail, since that
is the actionable half. It is a summary problem rather than a /utils/health
503: a backlog should not restart the container.

Also drops the keyspace `scan_iter` `stats()` did per poll to count parked
items — it walked every state and idempotency key twice per ten seconds — for
a set the park/unpark path maintains.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BpfSinyCBfjuieikyfMPbf

backend/fluksio/flow/executor.py

@@ -41,6 +41,12 @@ DELAYED_INTERVAL_S = 1.0
 MAX_CASCADES = 4
 # How long a reload waits for claimed work to finish before rebuilding anyway.
 DRAIN_TIMEOUT_S = 10.0
+# Work waiting in the stream, undelivered. A burst is normal — the pool claims
+# only what it can start — so what marks an engine as falling behind is the
+# backlog staying up across several checks rather than any one reading.
+BACKLOG_INTERVAL_S = 5.0
+BACKLOG_DEGRADED = 50
+BACKLOG_STRIKES = 3
 
 
 class ExecutionService:
@@ -70,6 +76,10 @@ class ExecutionService:
         )
         self._consumer: threading.Thread | None = None
         self._timers: threading.Thread | None = None
+        # Consecutive backlog readings over the threshold, and whether the last
+        # of them said so out loud.
+        self._backlog_strikes = 0
+        self.behind = False
 
     # -------------------------------------------------------------------------
     # Lifecycle
@@ -157,6 +167,7 @@ class ExecutionService:
         """Promote delayed items, and take back what a dead engine dropped."""
         last_reclaim = 0.0
         last_touch = 0.0
+        last_backlog = 0.0
         while not self._stop.is_set():
             self._stop.wait(DELAYED_INTERVAL_S)
             if self._stop.is_set():
@@ -167,6 +178,13 @@ class ExecutionService:
                 logger.error("Could not promote delayed work: %s", exc)
 
             now = time.monotonic()
+            if now - last_backlog >= BACKLOG_INTERVAL_S:
+                last_backlog = now
+                try:
+                    self._check_backlog()
+                except Exception as exc:
+                    logger.error("Could not read the queue backlog: %s", exc)
+
             if now - last_touch >= TOUCH_INTERVAL_S:
                 last_touch = now
                 with self._inflight_lock:
@@ -190,6 +208,38 @@ class ExecutionService:
             except Exception as exc:
                 logger.error("Could not reclaim stale work: %s", exc)
 
+    def _check_backlog(self) -> None:
+        """Say so when work has been waiting in the stream for a while.
+
+        A flow enqueuing faster than the pool drains produces no event of its
+        own: the backlog simply grows, every timer and connector poll drifts
+        behind it, and nothing on the health screen moves. This is that event.
+        The flow named is the one most of the waiting work belongs to, which is
+        the half somebody can act on.
+        """
+        backlog = self.queue.backlog()
+        if backlog < BACKLOG_DEGRADED:
+            self._backlog_strikes = 0
+            self.behind = False
+            return
+
+        self._backlog_strikes += 1
+        if self._backlog_strikes < BACKLOG_STRIKES or self.behind:
+            return
+
+        self.behind = True
+        flows = self.queue.backlog_flows()
+        worst = max(flows, key=lambda f: flows[f], default="")
+        logger.warning("engine behind: %d items waiting (%s)", backlog, worst or "?")
+        self._publish(
+            {
+                "type": "engine_degraded",
+                "reason": f"{backlog} items waiting in the queue",
+                "flow": worst,
+                "ts": time.time(),
+            }
+        )
+
     def _await_capacity(self) -> int:
         """How many cascades may be claimed now. Zero means the service stops.
 
@@ -366,6 +416,7 @@ class ExecutionService:
             return {"error": str(exc), "consumer_alive": self.alive()}
         stats["consumer_alive"] = self.alive()
         stats["cascades_busy"] = self._inflight
+        stats["behind"] = self.behind
         return stats
 
     def _publish_unavailable(self, exc: Exception) -> None: