Surface a node's health when the engine's floor expires
`node_health` fires once at the transition, while `HEALTH_FLOOR` still hides the node, and nothing is published when the floor expires 60 s later. Schedule a per-node re-read for that moment, cancelled when the node recovers and on socket teardown. Health now also invalidates the brain graph, whose key is a sibling of the flow's rather than a child, so Home's fault rings never moved on a health flip at all. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VfgHWr7AQDiLsTQVu7hpKB
This commit is contained in:
@@ -25,6 +25,15 @@ const RECONNECT_MAX = 30000
|
||||
* handler instead of retrying forever.
|
||||
*/
|
||||
const AUTH_ATTEMPTS = 3
|
||||
/**
|
||||
* How long the engine holds a node down before it surfaces as an issue.
|
||||
*
|
||||
* ponytail: a hand copy of `HEALTH_FLOOR` in
|
||||
* `backend/fluksio/flow/controller.py:122`, which drifts silently if that one
|
||||
* moves — nothing breaks, the border just turns early or late. Carrying the
|
||||
* floor in the `node_health` payload is the upgrade path.
|
||||
*/
|
||||
const HEALTH_FLOOR = 60_000
|
||||
|
||||
type FlowEvent =
|
||||
| {
|
||||
@@ -152,6 +161,20 @@ let watchers = 0
|
||||
let rejected = 0
|
||||
let client: QueryClient | null = null
|
||||
const authHandlers = new Set<() => void>()
|
||||
/** Floor re-reads still owed, per node, so a recovery can cancel its own. */
|
||||
const healthTimers = new Map<string, ReturnType<typeof setTimeout>>()
|
||||
|
||||
/** Everything a node's health shows up in, which is two unrelated keys. */
|
||||
function invalidateHealth(flow: string | undefined) {
|
||||
// `graph` is a sibling of the flow's own key rather than a child, so the
|
||||
// detail invalidation alone never reaches Home's fault rings.
|
||||
for (const queryKey of [
|
||||
flow ? flowKeys.detail(flow) : flowKeys.all,
|
||||
flowKeys.graph,
|
||||
]) {
|
||||
client?.invalidateQueries({ queryKey })
|
||||
}
|
||||
}
|
||||
|
||||
function schedule() {
|
||||
if (timer) return
|
||||
@@ -258,15 +281,28 @@ function connect() {
|
||||
ts: message.ts ?? Date.now() / 1000,
|
||||
})
|
||||
break
|
||||
case "node_health":
|
||||
case "node_health": {
|
||||
// The canvas draws node health from the flow detail's `issues`, which
|
||||
// the server derives — so the screen only moved on mount, navigation
|
||||
// or a rebuild, never when health actually flipped. The store had a
|
||||
// health map of its own and nothing ever read it.
|
||||
client?.invalidateQueries({
|
||||
queryKey: message.flow ? flowKeys.detail(message.flow) : flowKeys.all,
|
||||
})
|
||||
invalidateHealth(message.flow)
|
||||
if (message.health === "down") {
|
||||
// This fires at the transition, while the floor is still hiding the
|
||||
// node, and the engine publishes nothing when the floor expires — so
|
||||
// read again ourselves at that moment, or the issue waits for an
|
||||
// unrelated invalidation. Only the first `down` schedules it: the
|
||||
// engine restarts its own clock on a transition, not on a connector
|
||||
// rotating its error string while staying down.
|
||||
if (!healthTimers.has(message.node)) {
|
||||
healthTimers.set(
|
||||
message.node,
|
||||
setTimeout(() => {
|
||||
healthTimers.delete(message.node)
|
||||
invalidateHealth(message.flow)
|
||||
}, HEALTH_FLOOR),
|
||||
)
|
||||
}
|
||||
liveStore.recordEngineEvent({
|
||||
type: message.type,
|
||||
flow: message.flow,
|
||||
@@ -274,8 +310,14 @@ function connect() {
|
||||
detail: message.detail ?? "Reported itself down.",
|
||||
ts: message.ts ?? Date.now() / 1000,
|
||||
})
|
||||
} else {
|
||||
// Recovery clears at once server-side, so nothing is owed.
|
||||
const owed = healthTimers.get(message.node)
|
||||
if (owed) clearTimeout(owed)
|
||||
healthTimers.delete(message.node)
|
||||
}
|
||||
break
|
||||
}
|
||||
case "flow_quarantined":
|
||||
case "engine_degraded":
|
||||
case "engine_fatal":
|
||||
@@ -404,6 +446,8 @@ function release(onAuthFailure?: () => void) {
|
||||
clearTimeout(timer)
|
||||
timer = null
|
||||
}
|
||||
for (const owed of healthTimers.values()) clearTimeout(owed)
|
||||
healthTimers.clear()
|
||||
const ws = socket
|
||||
socket = null
|
||||
ws?.close()
|
||||
|
||||
Reference in New Issue
Block a user