Live view over WebRTC (openpanel L1, server and browser)

A device reporting a `live` port streams its camera to the browser's own
RTCPeerConnection. The flows socket carries the signalling as text, so it
crosses the portal: `live` messages on a person's socket, `signal` on a
device's (start, sdp, candidate, bye, busy). The server routes by session,
holds a device to two viewers, ends a session when either socket closes or
no answer comes within 10 s, and gives each socket an outbox of its own
that ends a session rather than drop a message. TURN credentials are
minted per session in coturn's REST scheme (STUN_URLS, TURN_URLS,
TURN_SECRET), the viewer's with the device's offer; GET /devices/{id}/ice
hands them out too. The tunnel keeps a stream's early frames instead of
dropping them past eight and forgets them when the stream or link goes.

A media tile on a device's camera, and the Devices tab's preview, play
live when tapped, pause frames meanwhile, and fall back to frames with a
short note on busy, failure or no picture within 10 s. coturn is an
optional compose service behind the `turn` profile.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0138N7k5PT4dcR1iEqXaaQ9k

frontend/src/components/Dashboard/MediaWidget.tsx

@@ -1,10 +1,12 @@
-import { useRef } from "react"
+import { LoaderCircle, Radio, X } from "lucide-react"
+import { type ReactNode, useEffect, useRef } from "react"
 
 import { useOnScreen } from "@/hooks/useOnScreen"
 import { isRef, useArtifactUrl } from "@/lib/media"
 import { cn } from "@/lib/utils"
 import { useBoundValue } from "./dataContext"
 import { config, text } from "./ui/core/config"
+import { useLiveDevice, useLiveView } from "./useLiveView"
 import type { WidgetProps } from "./widgets"
 
 /**
@@ -37,6 +39,10 @@ export function MediaWidget({ widget }: WidgetProps) {
  * looked at: a camera is only sent while somebody watches it, and a tile
  * scrolled away or behind another tab is nobody. A clip keeps playing either
  * way.
+ *
+ * A camera whose device streams live is tapped for it. Frames pause while
+ * the live view plays, so the device encodes one picture rather than two,
+ * and come back when it ends.
  */
 export function MediaView({
   message,
@@ -55,12 +61,15 @@ export function MediaView({
 }) {
   const box = useRef<HTMLDivElement>(null)
   const shown = useOnScreen(box)
-  const live = useBoundValue(message || undefined)
-  const value = live?.value
+  const bound = useBoundValue(message || undefined)
+  const value = bound?.value
   const ref = isRef(value) ? value : null
   const kind = (ref?.media_type ?? dtype).split("/")[0]
   const playing = kind === "audio" || kind === "video"
-  const active = shown || playing
+  const camera = useLiveDevice(stream ? "" : message)
+  const live = useLiveView(camera)
+  const onAir = live.phase === "playing"
+  const active = (shown || playing) && !onAir
   const url = useArtifactUrl(
     active ? ref : null,
     active ? message || undefined : undefined,
@@ -69,12 +78,24 @@ export function MediaView({
   const objectFit = fit === "contain" ? "object-contain" : "object-cover"
   const alt = label || ref?.name || message
 
+  // Scrolled away, or the tab hidden: nobody is watching.
+  const { stop } = live
+  useEffect(() => {
+    if (!shown) stop()
+  }, [shown, stop])
+
   return (
     <div
       ref={box}
       className="flex min-h-0 w-full flex-1 flex-col justify-center"
     >
-      {body()}
+      {camera ? (
+        <LiveFrame live={live} label={alt} fit={objectFit}>
+          {body()}
+        </LiveFrame>
+      ) : (
+        body()
+      )}
     </div>
   )
 
@@ -141,3 +162,95 @@ export function MediaView({
     )
   }
 }
+
+/**
+ * A camera tile that streams live when tapped, and stops when tapped again.
+ *
+ * The whole tile is the control, so it is as easy to hit on a wall panel as
+ * on a desk. The video waits under the frames until its first picture, then
+ * takes their place.
+ */
+function LiveFrame({
+  live,
+  label,
+  fit,
+  children,
+}: {
+  live: ReturnType<typeof useLiveView>
+  label: string
+  fit: string
+  children: ReactNode
+}) {
+  const { phase, stream, note } = live
+  const onAir = phase === "playing"
+  const chip =
+    "pointer-events-none absolute z-20 flex items-center gap-1 rounded-full border border-border px-2.5 py-1 text-xs font-medium shadow-e2"
+  const frosted = "bg-card/80 text-foreground backdrop-blur-md"
+
+  return (
+    <button
+      type="button"
+      onClick={phase === "off" ? live.start : live.stop}
+      aria-label={
+        phase === "off" ? `Watch ${label} live` : `Stop watching ${label} live`
+      }
+      aria-pressed={phase !== "off"}
+      data-testid="media-live"
+      data-live={phase}
+      className="relative isolate flex min-h-0 w-full flex-1 cursor-pointer flex-col justify-center rounded-md text-center focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
+    >
+      <span
+        className={cn(
+          "relative z-10 flex min-h-0 w-full flex-1 flex-col justify-center",
+          onAir && "invisible",
+        )}
+      >
+        {children}
+      </span>
+      {stream ? (
+        <video
+          ref={(element) => {
+            if (element && element.srcObject !== stream)
+              element.srcObject = stream
+          }}
+          autoPlay
+          muted
+          playsInline
+          onPlaying={live.played}
+          className={cn("absolute inset-0 h-full w-full rounded-md", fit)}
+        />
+      ) : null}
+      {onAir ? (
+        <span
+          className={cn(
+            chip,
+            "right-2 top-2 border-transparent bg-primary text-primary-foreground",
+          )}
+        >
+          Live
+          <X className="size-3.5" aria-hidden />
+        </span>
+      ) : (
+        <span className={cn(chip, frosted, "right-2 top-2")}>
+          {phase === "connecting" ? (
+            <LoaderCircle
+              className="size-3.5 motion-safe:animate-spin"
+              aria-hidden
+            />
+          ) : (
+            <Radio className="size-3.5" aria-hidden />
+          )}
+          {phase === "connecting" ? "Connecting" : "Live"}
+        </span>
+      )}
+      {note ? (
+        <output
+          className={cn(chip, frosted, "bottom-2 left-2 text-muted-foreground")}
+          data-testid="media-live-note"
+        >
+          {note}
+        </output>
+      ) : null}
+    </button>
+  )
+}

frontend/src/components/Dashboard/useLiveView.ts

@@ -0,0 +1,153 @@
+import { useQuery } from "@tanstack/react-query"
+import { useCallback, useEffect, useRef, useState } from "react"
+
+import { LIVE_PORT, reportedPorts } from "@/components/Devices/ports"
+import { devicesQueryOptions } from "@/components/Devices/queries"
+import {
+  type LiveSignal,
+  listenLive,
+  sendLive,
+} from "@/components/Flow/useFlowSocket"
+import { appRoute } from "@/lib/portal"
+
+/** How long a live view gets from the tap to its first frame. */
+const FIRST_FRAME_MS = 10_000
+/** How long the note after a fallback stays. */
+const NOTE_MS = 4000
+
+const BUSY = "Live view busy, showing frames"
+const FAILED = "Live view unavailable, showing frames"
+
+export type LivePhase = "off" | "connecting" | "playing"
+
+/**
+ * The device whose camera publishes `message` and can stream it live, or "".
+ *
+ * A device offers it by reporting a `live` port, and only while it is online.
+ * Read off the device list, which a wall panel's credential cannot read, so a
+ * panel never offers it.
+ */
+export function useLiveDevice(message: string): string {
+  const enabled = Boolean(message) && !appRoute().startsWith("/panel")
+  const { data } = useQuery({ ...devicesQueryOptions(), enabled })
+  if (!enabled) return ""
+  const device = data?.data?.find(
+    (one) =>
+      one.online &&
+      (one.bindings ?? []).some(
+        (binding) =>
+          binding.port === "camera" &&
+          binding.direction === "in" &&
+          binding.message === message,
+      ) &&
+      (reportedPorts(one) ?? []).some((port) => port.port === LIVE_PORT),
+  )
+  return device?.id ?? ""
+}
+
+/**
+ * A live view of one device's camera: the browser's own `RTCPeerConnection`
+ * answering the device's offer, signalled over the page's socket.
+ *
+ * Busy, a failure or no first frame within ten seconds ends it with a short
+ * note, and the tile goes on showing frames. Leaving says `bye`.
+ */
+export function useLiveView(device: string) {
+  const [phase, setPhase] = useState<LivePhase>("off")
+  const [stream, setStream] = useState<MediaStream | null>(null)
+  const [note, setNote] = useState("")
+  const running = useRef<{ stop: () => void; played: () => void } | null>(null)
+
+  const stop = useCallback(() => running.current?.stop(), [])
+  const played = useCallback(() => running.current?.played(), [])
+
+  const start = useCallback(() => {
+    if (!device || running.current) return
+    // Not `randomUUID`, which a page on plain HTTP on a LAN does not have.
+    const session = Array.from(
+      crypto.getRandomValues(new Uint8Array(12)),
+      (byte) => byte.toString(16).padStart(2, "0"),
+    ).join("")
+    let pc: RTCPeerConnection | null = null
+    let over = false
+    // The device's candidates that came before its offer was applied.
+    const early: string[] = []
+
+    const end = (why: string, tell: boolean) => {
+      if (over) return
+      over = true
+      clearTimeout(timer)
+      unlisten()
+      if (tell) sendLive({ device, session, action: "bye" })
+      pc?.close()
+      running.current = null
+      setStream(null)
+      setPhase("off")
+      setNote(why)
+    }
+    const add = (candidate: string) =>
+      pc
+        ?.addIceCandidate({ candidate, sdpMLineIndex: 0 })
+        .catch(() => undefined)
+
+    const answer = async (offer: LiveSignal) => {
+      const peer = new RTCPeerConnection({
+        iceServers: offer.ice_servers ?? [],
+      })
+      pc = peer
+      peer.ontrack = (event) =>
+        setStream(event.streams[0] ?? new MediaStream([event.track]))
+      peer.onicecandidate = (event) => {
+        // The bare `candidate:` line, which is what the device reads.
+        const candidate = event.candidate?.candidate
+        if (candidate)
+          sendLive({ device, session, action: "candidate", candidate })
+      }
+      peer.onconnectionstatechange = () => {
+        if (peer.connectionState === "failed") end(FAILED, true)
+      }
+      await peer.setRemoteDescription({ type: "offer", sdp: offer.sdp })
+      const reply = await peer.createAnswer()
+      await peer.setLocalDescription(reply)
+      sendLive({ device, session, action: "sdp", sdp: reply.sdp })
+      for (const candidate of early.splice(0)) await add(candidate)
+    }
+
+    const unlisten = listenLive(session, (message) => {
+      if (over) return
+      if (message.action === "busy") end(BUSY, false)
+      else if (message.action === "bye") end(FAILED, false)
+      else if (message.action === "sdp" && message.sdp && !pc)
+        answer(message).catch(() => end(FAILED, true))
+      else if (message.action === "candidate" && message.candidate) {
+        if (pc?.remoteDescription) add(message.candidate)
+        else early.push(message.candidate)
+      }
+    })
+    const timer = setTimeout(() => end(FAILED, true), FIRST_FRAME_MS)
+    running.current = {
+      stop: () => end("", true),
+      played: () => {
+        clearTimeout(timer)
+        setPhase("playing")
+      },
+    }
+    setNote("")
+    setPhase("connecting")
+    if (!sendLive({ device, session, action: "start" })) end(FAILED, false)
+  }, [device])
+
+  // A tile that goes away, or is pointed at another device, ends its view.
+  useEffect(() => {
+    if (!device) return
+    return () => running.current?.stop()
+  }, [device])
+
+  useEffect(() => {
+    if (!note) return
+    const timer = setTimeout(() => setNote(""), NOTE_MS)
+    return () => clearTimeout(timer)
+  }, [note])
+
+  return { phase, stream, note, start, stop, played }
+}