Coalesce the event bus, and fix the socket that ended on a client frame

A three-node cascade publishes 13-16 events and each one crossed to the
event loop on its own. They are one `call_soon_threadsafe` now — whatever
was published between two turns of the loop goes over together — and every
subscriber still receives every event, oldest still dropped first when one
falls behind.

The socket end of the same path:

- **any frame from the client ended its stream.** `receive_text` was
  awaited once, outside the loop, so a keepalive — or anything else a
  client decided to say — satisfied it and was read as the client going
  away. It is recreated per iteration; only a disconnect ends the stream.
- events go out in one frame per wave (`{"type": "batch", "events": [...]}`,
  capped at 64), serialised once with orjson rather than per client with
  the stdlib's `json.dumps` through `send_json`. The client unpacks a batch
  and still understands single frames, so an older engine behind a newer
  bundle keeps working.
- authenticating and building the snapshot happen on a thread. Both were on
  the event loop: one is a database round trip, the other reads the whole
  of state, per connect and again per `dashboard_changed` per panel.

`Pipeline.values()` — what that snapshot is — no longer SCANs the whole
Redis namespace. It scanned five bookkeeping keys for every message to find
the messages; `RedisState` keeps a set of the names beside them and answers
from it. Maintained wherever a message is written, so a seeded value or a
deleted flow keeps it exact.

On the client, while in the same file:

- a `node_health` event invalidates the flow's detail. The canvas draws
  health from the server-derived `issues`, so a node going down or
  recovering only showed on mount, navigation or a rebuild. The store had
  a health map of its own that nothing ever read; it and `useNodeHealth`
  are gone rather than wired up, since the server's view is the one the
  canvas already uses.
- a reconnect invalidates the five key families this socket feeds instead
  of the entire cache, and the backoff is jittered. The usual reason a
  socket dropped is the engine restarting, so every tab and every wall
  panel refetched everything, together, at the moment it was least able to
  answer.
- a frame that will not parse costs the frame, not the connection. It was
  the one unguarded `JSON.parse` in the app; an exception there escaped to
  `window.onerror` and left whatever it had already applied behind.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01M6hPWS6YEbT1P8LxhhFb2T

frontend/src/components/Flow/liveStore.ts

@@ -29,11 +29,6 @@ export type LiveStatus = {
   /** Why it is queued — what it is waiting for, and on which machine. */
   detail?: string | null
 }
-/** How a node's connection is doing, which is not how its last run went. */
-export type NodeHealth = {
-  health: "ok" | "down" | "unknown"
-  detail?: string | null
-}
 /**
  * A node's last failure, kept after it has run again.
  *
@@ -72,7 +67,6 @@ const ENGINE_EVENT_LIMIT = 100
 const values = new Map<string, LiveValue>()
 const statuses = new Map<string, LiveStatus>()
 const failures = new Map<string, NodeFailure>()
-const health = new Map<string, NodeHealth>()
 let engineEvents: EngineEvent[] = []
 // How many times this page has seen a node emit. The number itself means
 // nothing; a change is what restarts the pulse.
@@ -194,13 +188,6 @@ export const liveStore = {
       path: { name: flow, node_id: node },
     }).catch(() => {})
   },
-  setHealth(nodeId: string, entry: NodeHealth) {
-    health.set(nodeId, entry)
-    notify(`health:${nodeId}`)
-  },
-  getHealth(nodeId: string) {
-    return health.get(nodeId)
-  },
   recordEngineEvent(event: EngineEvent) {
     // A new array each time, so the hook's snapshot comparison sees the change.
     engineEvents = [...engineEvents, event].slice(-ENGINE_EVENT_LIMIT)
@@ -271,8 +258,6 @@ export const liveStore = {
       notify(`emit:${key}`)
     emits.clear()
     priorEmits.clear()
-    for (const key of health.keys()) notify(`health:${key}`)
-    health.clear()
     engineEvents = []
     notify("engine")
     logLines = []
@@ -304,14 +289,6 @@ export function useNodeFailure(nodeId: string): NodeFailure | undefined {
   )
 }
 
-/** How the node's connection is doing, once it has said anything about it. */
-export function useNodeHealth(nodeId: string): NodeHealth | undefined {
-  return useSyncExternalStore(
-    (listener) => subscribeKey(`health:${nodeId}`, listener),
-    () => health.get(nodeId),
-  )
-}
-
 /** The last hundred things the engine said about itself, oldest first. */
 export function useEngineEvents(): EngineEvent[] {
   return useSyncExternalStore(

frontend/src/components/Flow/useFlowSocket.ts

@@ -4,6 +4,7 @@ import { useEffect } from "react"
 
 import { OpenAPI } from "@/client"
 import { dashboardKeys, panelKeys } from "@/components/Dashboard/queries"
+import { healthKeys } from "@/components/Health/queries"
 import { runKeys } from "@/components/Runs/queries"
 import { connectionStore } from "@/lib/connectionStore"
 import { apiToken } from "@/lib/portal"
@@ -146,10 +147,16 @@ const authHandlers = new Set<() => void>()
 
 function schedule() {
   if (timer) return
-  timer = setTimeout(() => {
-    timer = null
-    connect()
-  }, retry)
+  // Jittered, because every client of an engine that restarted is counting
+  // the same backoff from the same moment: without it they all come back
+  // together, and keep coming back together.
+  timer = setTimeout(
+    () => {
+      timer = null
+      connect()
+    },
+    retry * (0.5 + Math.random()),
+  )
   retry = Math.min(retry * 2, RECONNECT_MAX)
 }
 
@@ -164,12 +171,23 @@ function connect() {
     liveStore.setConnected(true)
     connectionStore.setSocketOpen(true)
     // Whatever happened while the socket was down was missed, so nothing
-    // held in cache can be trusted to still be current.
-    client?.invalidateQueries()
+    // held in cache can be trusted to still be current. Scoped to what this
+    // socket actually feeds: an unqualified invalidation refetches every
+    // query the page holds, and the usual reason the socket dropped is the
+    // engine restarting — so every open tab and every wall panel did that at
+    // once, at the moment it was least able to answer.
+    for (const queryKey of [
+      flowKeys.all,
+      dashboardKeys.all,
+      panelKeys.all,
+      runKeys.all,
+      healthKeys.all,
+    ]) {
+      client?.invalidateQueries({ queryKey })
+    }
   }
 
-  ws.onmessage = (event) => {
-    const message: FlowEvent = JSON.parse(event.data)
+  const handle = (message: FlowEvent) => {
     switch (message.type) {
       case "snapshot":
         liveStore.setValues(message.values)
@@ -216,9 +234,12 @@ function connect() {
         })
         break
       case "node_health":
-        liveStore.setHealth(message.node, {
-          health: message.health,
-          detail: message.detail,
+        // The canvas draws node health from the flow detail's `issues`, which
+        // the server derives — so the screen only moved on mount, navigation
+        // or a rebuild, never when health actually flipped. The store had a
+        // health map of its own and nothing ever read it.
+        client?.invalidateQueries({
+          queryKey: message.flow ? flowKeys.detail(message.flow) : flowKeys.all,
         })
         if (message.health === "down") {
           liveStore.recordEngineEvent({
@@ -292,6 +313,25 @@ function connect() {
     }
   }
 
+  ws.onmessage = (event) => {
+    // A frame that is not JSON, or one this bundle cannot read, costs the
+    // frame rather than the connection: an exception thrown here escapes into
+    // `window.onerror` and leaves whatever it had already applied behind.
+    try {
+      const payload = JSON.parse(event.data)
+      if (payload?.type === "batch") {
+        // A cascade publishes a dozen events at once and the engine coalesces
+        // them into one frame. An installation older than this bundle sends
+        // them one at a time, which is the branch below.
+        for (const message of payload.events ?? []) handle(message)
+      } else {
+        handle(payload)
+      }
+    } catch (error) {
+      console.warn("Dropped an unreadable socket frame", error)
+    }
+  }
+
   ws.onclose = (event) => {
     // A socket we already dropped: its close says nothing about the connection
     // we want now.