Live view over WebRTC (openpanel L1, server and browser)
A device reporting a `live` port streams its camera to the browser's own
RTCPeerConnection. The flows socket carries the signalling as text, so it
crosses the portal: `live` messages on a person's socket, `signal` on a
device's (start, sdp, candidate, bye, busy). The server routes by session,
holds a device to two viewers, ends a session when either socket closes or
no answer comes within 10 s, and gives each socket an outbox of its own
that ends a session rather than drop a message. TURN credentials are
minted per session in coturn's REST scheme (STUN_URLS, TURN_URLS,
TURN_SECRET), the viewer's with the device's offer; GET /devices/{id}/ice
hands them out too. The tunnel keeps a stream's early frames instead of
dropping them past eight and forgets them when the stream or link goes.
A media tile on a device's camera, and the Devices tab's preview, play
live when tapped, pause frames meanwhile, and fall back to frames with a
short note on busy, failure or no picture within 10 s. coturn is an
optional compose service behind the `turn` profile.
This commit is contained in:
@@ -0,0 +1,153 @@
|
||||
import { useQuery } from "@tanstack/react-query"
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
|
||||
import { LIVE_PORT, reportedPorts } from "@/components/Devices/ports"
|
||||
import { devicesQueryOptions } from "@/components/Devices/queries"
|
||||
import {
|
||||
type LiveSignal,
|
||||
listenLive,
|
||||
sendLive,
|
||||
} from "@/components/Flow/useFlowSocket"
|
||||
import { appRoute } from "@/lib/portal"
|
||||
|
||||
/** How long a live view gets from the tap to its first frame. */
|
||||
const FIRST_FRAME_MS = 10_000
|
||||
/** How long the note after a fallback stays. */
|
||||
const NOTE_MS = 4000
|
||||
|
||||
const BUSY = "Live view busy, showing frames"
|
||||
const FAILED = "Live view unavailable, showing frames"
|
||||
|
||||
export type LivePhase = "off" | "connecting" | "playing"
|
||||
|
||||
/**
|
||||
* The device whose camera publishes `message` and can stream it live, or "".
|
||||
*
|
||||
* A device offers it by reporting a `live` port, and only while it is online.
|
||||
* Read off the device list, which a wall panel's credential cannot read, so a
|
||||
* panel never offers it.
|
||||
*/
|
||||
export function useLiveDevice(message: string): string {
|
||||
const enabled = Boolean(message) && !appRoute().startsWith("/panel")
|
||||
const { data } = useQuery({ ...devicesQueryOptions(), enabled })
|
||||
if (!enabled) return ""
|
||||
const device = data?.data?.find(
|
||||
(one) =>
|
||||
one.online &&
|
||||
(one.bindings ?? []).some(
|
||||
(binding) =>
|
||||
binding.port === "camera" &&
|
||||
binding.direction === "in" &&
|
||||
binding.message === message,
|
||||
) &&
|
||||
(reportedPorts(one) ?? []).some((port) => port.port === LIVE_PORT),
|
||||
)
|
||||
return device?.id ?? ""
|
||||
}
|
||||
|
||||
/**
|
||||
* A live view of one device's camera: the browser's own `RTCPeerConnection`
|
||||
* answering the device's offer, signalled over the page's socket.
|
||||
*
|
||||
* Busy, a failure or no first frame within ten seconds ends it with a short
|
||||
* note, and the tile goes on showing frames. Leaving says `bye`.
|
||||
*/
|
||||
export function useLiveView(device: string) {
|
||||
const [phase, setPhase] = useState<LivePhase>("off")
|
||||
const [stream, setStream] = useState<MediaStream | null>(null)
|
||||
const [note, setNote] = useState("")
|
||||
const running = useRef<{ stop: () => void; played: () => void } | null>(null)
|
||||
|
||||
const stop = useCallback(() => running.current?.stop(), [])
|
||||
const played = useCallback(() => running.current?.played(), [])
|
||||
|
||||
const start = useCallback(() => {
|
||||
if (!device || running.current) return
|
||||
// Not `randomUUID`, which a page on plain HTTP on a LAN does not have.
|
||||
const session = Array.from(
|
||||
crypto.getRandomValues(new Uint8Array(12)),
|
||||
(byte) => byte.toString(16).padStart(2, "0"),
|
||||
).join("")
|
||||
let pc: RTCPeerConnection | null = null
|
||||
let over = false
|
||||
// The device's candidates that came before its offer was applied.
|
||||
const early: string[] = []
|
||||
|
||||
const end = (why: string, tell: boolean) => {
|
||||
if (over) return
|
||||
over = true
|
||||
clearTimeout(timer)
|
||||
unlisten()
|
||||
if (tell) sendLive({ device, session, action: "bye" })
|
||||
pc?.close()
|
||||
running.current = null
|
||||
setStream(null)
|
||||
setPhase("off")
|
||||
setNote(why)
|
||||
}
|
||||
const add = (candidate: string) =>
|
||||
pc
|
||||
?.addIceCandidate({ candidate, sdpMLineIndex: 0 })
|
||||
.catch(() => undefined)
|
||||
|
||||
const answer = async (offer: LiveSignal) => {
|
||||
const peer = new RTCPeerConnection({
|
||||
iceServers: offer.ice_servers ?? [],
|
||||
})
|
||||
pc = peer
|
||||
peer.ontrack = (event) =>
|
||||
setStream(event.streams[0] ?? new MediaStream([event.track]))
|
||||
peer.onicecandidate = (event) => {
|
||||
// The bare `candidate:` line, which is what the device reads.
|
||||
const candidate = event.candidate?.candidate
|
||||
if (candidate)
|
||||
sendLive({ device, session, action: "candidate", candidate })
|
||||
}
|
||||
peer.onconnectionstatechange = () => {
|
||||
if (peer.connectionState === "failed") end(FAILED, true)
|
||||
}
|
||||
await peer.setRemoteDescription({ type: "offer", sdp: offer.sdp })
|
||||
const reply = await peer.createAnswer()
|
||||
await peer.setLocalDescription(reply)
|
||||
sendLive({ device, session, action: "sdp", sdp: reply.sdp })
|
||||
for (const candidate of early.splice(0)) await add(candidate)
|
||||
}
|
||||
|
||||
const unlisten = listenLive(session, (message) => {
|
||||
if (over) return
|
||||
if (message.action === "busy") end(BUSY, false)
|
||||
else if (message.action === "bye") end(FAILED, false)
|
||||
else if (message.action === "sdp" && message.sdp && !pc)
|
||||
answer(message).catch(() => end(FAILED, true))
|
||||
else if (message.action === "candidate" && message.candidate) {
|
||||
if (pc?.remoteDescription) add(message.candidate)
|
||||
else early.push(message.candidate)
|
||||
}
|
||||
})
|
||||
const timer = setTimeout(() => end(FAILED, true), FIRST_FRAME_MS)
|
||||
running.current = {
|
||||
stop: () => end("", true),
|
||||
played: () => {
|
||||
clearTimeout(timer)
|
||||
setPhase("playing")
|
||||
},
|
||||
}
|
||||
setNote("")
|
||||
setPhase("connecting")
|
||||
if (!sendLive({ device, session, action: "start" })) end(FAILED, false)
|
||||
}, [device])
|
||||
|
||||
// A tile that goes away, or is pointed at another device, ends its view.
|
||||
useEffect(() => {
|
||||
if (!device) return
|
||||
return () => running.current?.stop()
|
||||
}, [device])
|
||||
|
||||
useEffect(() => {
|
||||
if (!note) return
|
||||
const timer = setTimeout(() => setNote(""), NOTE_MS)
|
||||
return () => clearTimeout(timer)
|
||||
}, [note])
|
||||
|
||||
return { phase, stream, note, start, stop, played }
|
||||
}
|
||||
Reference in New Issue
Block a user