Surface a node's health when the engine's floor expires
`node_health` fires once at the transition, while `HEALTH_FLOOR` still hides the node, and nothing is published when the floor expires 60 s later. Schedule a per-node re-read for that moment, cancelled when the node recovers and on socket teardown. Health now also invalidates the brain graph, whose key is a sibling of the flow's rather than a child, so Home's fault rings never moved on a health flip at all. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VfgHWr7AQDiLsTQVu7hpKB
This commit is contained in:
@@ -25,6 +25,15 @@ const RECONNECT_MAX = 30000
|
|||||||
* handler instead of retrying forever.
|
* handler instead of retrying forever.
|
||||||
*/
|
*/
|
||||||
const AUTH_ATTEMPTS = 3
|
const AUTH_ATTEMPTS = 3
|
||||||
|
/**
|
||||||
|
* How long the engine holds a node down before it surfaces as an issue.
|
||||||
|
*
|
||||||
|
* ponytail: a hand copy of `HEALTH_FLOOR` in
|
||||||
|
* `backend/fluksio/flow/controller.py:122`, which drifts silently if that one
|
||||||
|
* moves — nothing breaks, the border just turns early or late. Carrying the
|
||||||
|
* floor in the `node_health` payload is the upgrade path.
|
||||||
|
*/
|
||||||
|
const HEALTH_FLOOR = 60_000
|
||||||
|
|
||||||
type FlowEvent =
|
type FlowEvent =
|
||||||
| {
|
| {
|
||||||
@@ -152,6 +161,20 @@ let watchers = 0
|
|||||||
let rejected = 0
|
let rejected = 0
|
||||||
let client: QueryClient | null = null
|
let client: QueryClient | null = null
|
||||||
const authHandlers = new Set<() => void>()
|
const authHandlers = new Set<() => void>()
|
||||||
|
/** Floor re-reads still owed, per node, so a recovery can cancel its own. */
|
||||||
|
const healthTimers = new Map<string, ReturnType<typeof setTimeout>>()
|
||||||
|
|
||||||
|
/** Everything a node's health shows up in, which is two unrelated keys. */
|
||||||
|
function invalidateHealth(flow: string | undefined) {
|
||||||
|
// `graph` is a sibling of the flow's own key rather than a child, so the
|
||||||
|
// detail invalidation alone never reaches Home's fault rings.
|
||||||
|
for (const queryKey of [
|
||||||
|
flow ? flowKeys.detail(flow) : flowKeys.all,
|
||||||
|
flowKeys.graph,
|
||||||
|
]) {
|
||||||
|
client?.invalidateQueries({ queryKey })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function schedule() {
|
function schedule() {
|
||||||
if (timer) return
|
if (timer) return
|
||||||
@@ -258,15 +281,28 @@ function connect() {
|
|||||||
ts: message.ts ?? Date.now() / 1000,
|
ts: message.ts ?? Date.now() / 1000,
|
||||||
})
|
})
|
||||||
break
|
break
|
||||||
case "node_health":
|
case "node_health": {
|
||||||
// The canvas draws node health from the flow detail's `issues`, which
|
// The canvas draws node health from the flow detail's `issues`, which
|
||||||
// the server derives — so the screen only moved on mount, navigation
|
// the server derives — so the screen only moved on mount, navigation
|
||||||
// or a rebuild, never when health actually flipped. The store had a
|
// or a rebuild, never when health actually flipped. The store had a
|
||||||
// health map of its own and nothing ever read it.
|
// health map of its own and nothing ever read it.
|
||||||
client?.invalidateQueries({
|
invalidateHealth(message.flow)
|
||||||
queryKey: message.flow ? flowKeys.detail(message.flow) : flowKeys.all,
|
|
||||||
})
|
|
||||||
if (message.health === "down") {
|
if (message.health === "down") {
|
||||||
|
// This fires at the transition, while the floor is still hiding the
|
||||||
|
// node, and the engine publishes nothing when the floor expires — so
|
||||||
|
// read again ourselves at that moment, or the issue waits for an
|
||||||
|
// unrelated invalidation. Only the first `down` schedules it: the
|
||||||
|
// engine restarts its own clock on a transition, not on a connector
|
||||||
|
// rotating its error string while staying down.
|
||||||
|
if (!healthTimers.has(message.node)) {
|
||||||
|
healthTimers.set(
|
||||||
|
message.node,
|
||||||
|
setTimeout(() => {
|
||||||
|
healthTimers.delete(message.node)
|
||||||
|
invalidateHealth(message.flow)
|
||||||
|
}, HEALTH_FLOOR),
|
||||||
|
)
|
||||||
|
}
|
||||||
liveStore.recordEngineEvent({
|
liveStore.recordEngineEvent({
|
||||||
type: message.type,
|
type: message.type,
|
||||||
flow: message.flow,
|
flow: message.flow,
|
||||||
@@ -274,8 +310,14 @@ function connect() {
|
|||||||
detail: message.detail ?? "Reported itself down.",
|
detail: message.detail ?? "Reported itself down.",
|
||||||
ts: message.ts ?? Date.now() / 1000,
|
ts: message.ts ?? Date.now() / 1000,
|
||||||
})
|
})
|
||||||
|
} else {
|
||||||
|
// Recovery clears at once server-side, so nothing is owed.
|
||||||
|
const owed = healthTimers.get(message.node)
|
||||||
|
if (owed) clearTimeout(owed)
|
||||||
|
healthTimers.delete(message.node)
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
|
}
|
||||||
case "flow_quarantined":
|
case "flow_quarantined":
|
||||||
case "engine_degraded":
|
case "engine_degraded":
|
||||||
case "engine_fatal":
|
case "engine_fatal":
|
||||||
@@ -404,6 +446,8 @@ function release(onAuthFailure?: () => void) {
|
|||||||
clearTimeout(timer)
|
clearTimeout(timer)
|
||||||
timer = null
|
timer = null
|
||||||
}
|
}
|
||||||
|
for (const owed of healthTimers.values()) clearTimeout(owed)
|
||||||
|
healthTimers.clear()
|
||||||
const ws = socket
|
const ws = socket
|
||||||
socket = null
|
socket = null
|
||||||
ws?.close()
|
ws?.close()
|
||||||
|
|||||||
Reference in New Issue
Block a user