Live view over WebRTC (openpanel L1, server and browser)

A device reporting a `live` port streams its camera to the browser's own
RTCPeerConnection. The flows socket carries the signalling as text, so it
crosses the portal: `live` messages on a person's socket, `signal` on a
device's (start, sdp, candidate, bye, busy). The server routes by session,
holds a device to two viewers, ends a session when either socket closes or
no answer comes within 10 s, and gives each socket an outbox of its own
that ends a session rather than drop a message. TURN credentials are
minted per session in coturn's REST scheme (STUN_URLS, TURN_URLS,
TURN_SECRET), the viewer's with the device's offer; GET /devices/{id}/ice
hands them out too. The tunnel keeps a stream's early frames instead of
dropping them past eight and forgets them when the stream or link goes.

A media tile on a device's camera, and the Devices tab's preview, play
live when tapped, pause frames meanwhile, and fall back to frames with a
short note on busy, failure or no picture within 10 s. coturn is an
optional compose service behind the `turn` profile.
This commit is contained in:
Melvin Strobl
2026-10-17 08:37:39 +02:00
parent efd883d092
commit 6a98dcd23f
20 changed files with 1535 additions and 24 deletions
@@ -0,0 +1,153 @@
import { useQuery } from "@tanstack/react-query"
import { useCallback, useEffect, useRef, useState } from "react"
import { LIVE_PORT, reportedPorts } from "@/components/Devices/ports"
import { devicesQueryOptions } from "@/components/Devices/queries"
import {
type LiveSignal,
listenLive,
sendLive,
} from "@/components/Flow/useFlowSocket"
import { appRoute } from "@/lib/portal"
/** How long a live view gets from the tap to its first frame. */
const FIRST_FRAME_MS = 10_000
/** How long the note after a fallback stays. */
const NOTE_MS = 4000
const BUSY = "Live view busy, showing frames"
const FAILED = "Live view unavailable, showing frames"
export type LivePhase = "off" | "connecting" | "playing"
/**
* The device whose camera publishes `message` and can stream it live, or "".
*
* A device offers it by reporting a `live` port, and only while it is online.
* Read off the device list, which a wall panel's credential cannot read, so a
* panel never offers it.
*/
export function useLiveDevice(message: string): string {
const enabled = Boolean(message) && !appRoute().startsWith("/panel")
const { data } = useQuery({ ...devicesQueryOptions(), enabled })
if (!enabled) return ""
const device = data?.data?.find(
(one) =>
one.online &&
(one.bindings ?? []).some(
(binding) =>
binding.port === "camera" &&
binding.direction === "in" &&
binding.message === message,
) &&
(reportedPorts(one) ?? []).some((port) => port.port === LIVE_PORT),
)
return device?.id ?? ""
}
/**
* A live view of one device's camera: the browser's own `RTCPeerConnection`
* answering the device's offer, signalled over the page's socket.
*
* Busy, a failure or no first frame within ten seconds ends it with a short
* note, and the tile goes on showing frames. Leaving says `bye`.
*/
export function useLiveView(device: string) {
const [phase, setPhase] = useState<LivePhase>("off")
const [stream, setStream] = useState<MediaStream | null>(null)
const [note, setNote] = useState("")
const running = useRef<{ stop: () => void; played: () => void } | null>(null)
const stop = useCallback(() => running.current?.stop(), [])
const played = useCallback(() => running.current?.played(), [])
const start = useCallback(() => {
if (!device || running.current) return
// Not `randomUUID`, which a page on plain HTTP on a LAN does not have.
const session = Array.from(
crypto.getRandomValues(new Uint8Array(12)),
(byte) => byte.toString(16).padStart(2, "0"),
).join("")
let pc: RTCPeerConnection | null = null
let over = false
// The device's candidates that came before its offer was applied.
const early: string[] = []
const end = (why: string, tell: boolean) => {
if (over) return
over = true
clearTimeout(timer)
unlisten()
if (tell) sendLive({ device, session, action: "bye" })
pc?.close()
running.current = null
setStream(null)
setPhase("off")
setNote(why)
}
const add = (candidate: string) =>
pc
?.addIceCandidate({ candidate, sdpMLineIndex: 0 })
.catch(() => undefined)
const answer = async (offer: LiveSignal) => {
const peer = new RTCPeerConnection({
iceServers: offer.ice_servers ?? [],
})
pc = peer
peer.ontrack = (event) =>
setStream(event.streams[0] ?? new MediaStream([event.track]))
peer.onicecandidate = (event) => {
// The bare `candidate:` line, which is what the device reads.
const candidate = event.candidate?.candidate
if (candidate)
sendLive({ device, session, action: "candidate", candidate })
}
peer.onconnectionstatechange = () => {
if (peer.connectionState === "failed") end(FAILED, true)
}
await peer.setRemoteDescription({ type: "offer", sdp: offer.sdp })
const reply = await peer.createAnswer()
await peer.setLocalDescription(reply)
sendLive({ device, session, action: "sdp", sdp: reply.sdp })
for (const candidate of early.splice(0)) await add(candidate)
}
const unlisten = listenLive(session, (message) => {
if (over) return
if (message.action === "busy") end(BUSY, false)
else if (message.action === "bye") end(FAILED, false)
else if (message.action === "sdp" && message.sdp && !pc)
answer(message).catch(() => end(FAILED, true))
else if (message.action === "candidate" && message.candidate) {
if (pc?.remoteDescription) add(message.candidate)
else early.push(message.candidate)
}
})
const timer = setTimeout(() => end(FAILED, true), FIRST_FRAME_MS)
running.current = {
stop: () => end("", true),
played: () => {
clearTimeout(timer)
setPhase("playing")
},
}
setNote("")
setPhase("connecting")
if (!sendLive({ device, session, action: "start" })) end(FAILED, false)
}, [device])
// A tile that goes away, or is pointed at another device, ends its view.
useEffect(() => {
if (!device) return
return () => running.current?.stop()
}, [device])
useEffect(() => {
if (!note) return
const timer = setTimeout(() => setNote(""), NOTE_MS)
return () => clearTimeout(timer)
}, [note])
return { phase, stream, note, start, stop, played }
}