Live view over WebRTC (openpanel L1, server and browser)

A device reporting a `live` port streams its camera to the browser's own
RTCPeerConnection. The flows socket carries the signalling as text, so it
crosses the portal: `live` messages on a person's socket, `signal` on a
device's (start, sdp, candidate, bye, busy). The server routes by session,
holds a device to two viewers, ends a session when either socket closes or
no answer comes within 10 s, and gives each socket an outbox of its own
that ends a session rather than drop a message. TURN credentials are
minted per session in coturn's REST scheme (STUN_URLS, TURN_URLS,
TURN_SECRET), the viewer's with the device's offer; GET /devices/{id}/ice
hands them out too. The tunnel keeps a stream's early frames instead of
dropping them past eight and forgets them when the stream or link goes.

A media tile on a device's camera, and the Devices tab's preview, play
live when tapped, pause frames meanwhile, and fall back to frames with a
short note on busy, failure or no picture within 10 s. coturn is an
optional compose service behind the `turn` profile.

frontend/src/client/schemas.gen.ts

@@ -2006,6 +2006,63 @@ export const HistoryPointSchema = {
     description: 'One numeric value a message carried, and when.'
 } as const;
 
+export const IceServerSchema = {
+    properties: {
+        urls: {
+            items: {
+                type: 'string'
+            },
+            type: 'array',
+            title: 'Urls'
+        },
+        username: {
+            anyOf: [
+                {
+                    type: 'string'
+                },
+                {
+                    type: 'null'
+                }
+            ],
+            title: 'Username'
+        },
+        credential: {
+            anyOf: [
+                {
+                    type: 'string'
+                },
+                {
+                    type: 'null'
+                }
+            ],
+            title: 'Credential'
+        }
+    },
+    type: 'object',
+    required: ['urls'],
+    title: 'IceServer',
+    description: 'One STUN or TURN server, as `RTCPeerConnection` takes it.'
+} as const;
+
+export const IceServersSchema = {
+    properties: {
+        ice_servers: {
+            items: {
+                '$ref': '#/components/schemas/IceServer'
+            },
+            type: 'array',
+            title: 'Ice Servers'
+        },
+        ttl: {
+            type: 'integer',
+            title: 'Ttl',
+            default: 0
+        }
+    },
+    type: 'object',
+    title: 'IceServers'
+} as const;
+
 export const ImportArtifactSchema = {
     properties: {
         name: {

frontend/src/client/types.gen.ts

@@ -686,6 +686,20 @@ export type HTTPValidationError = {
     detail?: Array<ValidationError>;
 };
 
+/**
+ * One STUN or TURN server, as `RTCPeerConnection` takes it.
+ */
+export type IceServer = {
+    urls: Array<(string)>;
+    username?: (string | null);
+    credential?: (string | null);
+};
+
+export type IceServers = {
+    ice_servers?: Array<IceServer>;
+    ttl?: number;
+};
+
 export type ImportArtifact = {
     name: string;
     filename?: (string | null);
@@ -1879,6 +1893,12 @@ export type DevicesDeleteDeviceData = {
 
 export type DevicesDeleteDeviceResponse = (Message);
 
+export type DevicesReadIceData = {
+    deviceId: string;
+};
+
+export type DevicesReadIceResponse = (IceServers);
+
 export type DevicesPutReportData = {
     deviceId: string;
 };

frontend/src/components/Dashboard/MediaWidget.tsx

@@ -1,10 +1,12 @@
-import { useRef } from "react"
+import { LoaderCircle, Radio, X } from "lucide-react"
+import { type ReactNode, useEffect, useRef } from "react"
 
 import { useOnScreen } from "@/hooks/useOnScreen"
 import { isRef, useArtifactUrl } from "@/lib/media"
 import { cn } from "@/lib/utils"
 import { useBoundValue } from "./dataContext"
 import { config, text } from "./ui/core/config"
+import { useLiveDevice, useLiveView } from "./useLiveView"
 import type { WidgetProps } from "./widgets"
 
 /**
@@ -37,6 +39,10 @@ export function MediaWidget({ widget }: WidgetProps) {
  * looked at: a camera is only sent while somebody watches it, and a tile
  * scrolled away or behind another tab is nobody. A clip keeps playing either
  * way.
+ *
+ * A camera whose device streams live is tapped for it. Frames pause while
+ * the live view plays, so the device encodes one picture rather than two,
+ * and come back when it ends.
  */
 export function MediaView({
   message,
@@ -55,12 +61,15 @@ export function MediaView({
 }) {
   const box = useRef<HTMLDivElement>(null)
   const shown = useOnScreen(box)
-  const live = useBoundValue(message || undefined)
-  const value = live?.value
+  const bound = useBoundValue(message || undefined)
+  const value = bound?.value
   const ref = isRef(value) ? value : null
   const kind = (ref?.media_type ?? dtype).split("/")[0]
   const playing = kind === "audio" || kind === "video"
-  const active = shown || playing
+  const camera = useLiveDevice(stream ? "" : message)
+  const live = useLiveView(camera)
+  const onAir = live.phase === "playing"
+  const active = (shown || playing) && !onAir
   const url = useArtifactUrl(
     active ? ref : null,
     active ? message || undefined : undefined,
@@ -69,12 +78,24 @@ export function MediaView({
   const objectFit = fit === "contain" ? "object-contain" : "object-cover"
   const alt = label || ref?.name || message
 
+  // Scrolled away, or the tab hidden: nobody is watching.
+  const { stop } = live
+  useEffect(() => {
+    if (!shown) stop()
+  }, [shown, stop])
+
   return (
     <div
       ref={box}
       className="flex min-h-0 w-full flex-1 flex-col justify-center"
     >
-      {body()}
+      {camera ? (
+        <LiveFrame live={live} label={alt} fit={objectFit}>
+          {body()}
+        </LiveFrame>
+      ) : (
+        body()
+      )}
     </div>
   )
 
@@ -141,3 +162,95 @@ export function MediaView({
     )
   }
 }
+
+/**
+ * A camera tile that streams live when tapped, and stops when tapped again.
+ *
+ * The whole tile is the control, so it is as easy to hit on a wall panel as
+ * on a desk. The video waits under the frames until its first picture, then
+ * takes their place.
+ */
+function LiveFrame({
+  live,
+  label,
+  fit,
+  children,
+}: {
+  live: ReturnType<typeof useLiveView>
+  label: string
+  fit: string
+  children: ReactNode
+}) {
+  const { phase, stream, note } = live
+  const onAir = phase === "playing"
+  const chip =
+    "pointer-events-none absolute z-20 flex items-center gap-1 rounded-full border border-border px-2.5 py-1 text-xs font-medium shadow-e2"
+  const frosted = "bg-card/80 text-foreground backdrop-blur-md"
+
+  return (
+    <button
+      type="button"
+      onClick={phase === "off" ? live.start : live.stop}
+      aria-label={
+        phase === "off" ? `Watch ${label} live` : `Stop watching ${label} live`
+      }
+      aria-pressed={phase !== "off"}
+      data-testid="media-live"
+      data-live={phase}
+      className="relative isolate flex min-h-0 w-full flex-1 cursor-pointer flex-col justify-center rounded-md text-center focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
+    >
+      <span
+        className={cn(
+          "relative z-10 flex min-h-0 w-full flex-1 flex-col justify-center",
+          onAir && "invisible",
+        )}
+      >
+        {children}
+      </span>
+      {stream ? (
+        <video
+          ref={(element) => {
+            if (element && element.srcObject !== stream)
+              element.srcObject = stream
+          }}
+          autoPlay
+          muted
+          playsInline
+          onPlaying={live.played}
+          className={cn("absolute inset-0 h-full w-full rounded-md", fit)}
+        />
+      ) : null}
+      {onAir ? (
+        <span
+          className={cn(
+            chip,
+            "right-2 top-2 border-transparent bg-primary text-primary-foreground",
+          )}
+        >
+          Live
+          <X className="size-3.5" aria-hidden />
+        </span>
+      ) : (
+        <span className={cn(chip, frosted, "right-2 top-2")}>
+          {phase === "connecting" ? (
+            <LoaderCircle
+              className="size-3.5 motion-safe:animate-spin"
+              aria-hidden
+            />
+          ) : (
+            <Radio className="size-3.5" aria-hidden />
+          )}
+          {phase === "connecting" ? "Connecting" : "Live"}
+        </span>
+      )}
+      {note ? (
+        <output
+          className={cn(chip, frosted, "bottom-2 left-2 text-muted-foreground")}
+          data-testid="media-live-note"
+        >
+          {note}
+        </output>
+      ) : null}
+    </button>
+  )
+}

frontend/src/components/Dashboard/useLiveView.ts

@@ -0,0 +1,153 @@
+import { useQuery } from "@tanstack/react-query"
+import { useCallback, useEffect, useRef, useState } from "react"
+
+import { LIVE_PORT, reportedPorts } from "@/components/Devices/ports"
+import { devicesQueryOptions } from "@/components/Devices/queries"
+import {
+  type LiveSignal,
+  listenLive,
+  sendLive,
+} from "@/components/Flow/useFlowSocket"
+import { appRoute } from "@/lib/portal"
+
+/** How long a live view gets from the tap to its first frame. */
+const FIRST_FRAME_MS = 10_000
+/** How long the note after a fallback stays. */
+const NOTE_MS = 4000
+
+const BUSY = "Live view busy, showing frames"
+const FAILED = "Live view unavailable, showing frames"
+
+export type LivePhase = "off" | "connecting" | "playing"
+
+/**
+ * The device whose camera publishes `message` and can stream it live, or "".
+ *
+ * A device offers it by reporting a `live` port, and only while it is online.
+ * Read off the device list, which a wall panel's credential cannot read, so a
+ * panel never offers it.
+ */
+export function useLiveDevice(message: string): string {
+  const enabled = Boolean(message) && !appRoute().startsWith("/panel")
+  const { data } = useQuery({ ...devicesQueryOptions(), enabled })
+  if (!enabled) return ""
+  const device = data?.data?.find(
+    (one) =>
+      one.online &&
+      (one.bindings ?? []).some(
+        (binding) =>
+          binding.port === "camera" &&
+          binding.direction === "in" &&
+          binding.message === message,
+      ) &&
+      (reportedPorts(one) ?? []).some((port) => port.port === LIVE_PORT),
+  )
+  return device?.id ?? ""
+}
+
+/**
+ * A live view of one device's camera: the browser's own `RTCPeerConnection`
+ * answering the device's offer, signalled over the page's socket.
+ *
+ * Busy, a failure or no first frame within ten seconds ends it with a short
+ * note, and the tile goes on showing frames. Leaving says `bye`.
+ */
+export function useLiveView(device: string) {
+  const [phase, setPhase] = useState<LivePhase>("off")
+  const [stream, setStream] = useState<MediaStream | null>(null)
+  const [note, setNote] = useState("")
+  const running = useRef<{ stop: () => void; played: () => void } | null>(null)
+
+  const stop = useCallback(() => running.current?.stop(), [])
+  const played = useCallback(() => running.current?.played(), [])
+
+  const start = useCallback(() => {
+    if (!device || running.current) return
+    // Not `randomUUID`, which a page on plain HTTP on a LAN does not have.
+    const session = Array.from(
+      crypto.getRandomValues(new Uint8Array(12)),
+      (byte) => byte.toString(16).padStart(2, "0"),
+    ).join("")
+    let pc: RTCPeerConnection | null = null
+    let over = false
+    // The device's candidates that came before its offer was applied.
+    const early: string[] = []
+
+    const end = (why: string, tell: boolean) => {
+      if (over) return
+      over = true
+      clearTimeout(timer)
+      unlisten()
+      if (tell) sendLive({ device, session, action: "bye" })
+      pc?.close()
+      running.current = null
+      setStream(null)
+      setPhase("off")
+      setNote(why)
+    }
+    const add = (candidate: string) =>
+      pc
+        ?.addIceCandidate({ candidate, sdpMLineIndex: 0 })
+        .catch(() => undefined)
+
+    const answer = async (offer: LiveSignal) => {
+      const peer = new RTCPeerConnection({
+        iceServers: offer.ice_servers ?? [],
+      })
+      pc = peer
+      peer.ontrack = (event) =>
+        setStream(event.streams[0] ?? new MediaStream([event.track]))
+      peer.onicecandidate = (event) => {
+        // The bare `candidate:` line, which is what the device reads.
+        const candidate = event.candidate?.candidate
+        if (candidate)
+          sendLive({ device, session, action: "candidate", candidate })
+      }
+      peer.onconnectionstatechange = () => {
+        if (peer.connectionState === "failed") end(FAILED, true)
+      }
+      await peer.setRemoteDescription({ type: "offer", sdp: offer.sdp })
+      const reply = await peer.createAnswer()
+      await peer.setLocalDescription(reply)
+      sendLive({ device, session, action: "sdp", sdp: reply.sdp })
+      for (const candidate of early.splice(0)) await add(candidate)
+    }
+
+    const unlisten = listenLive(session, (message) => {
+      if (over) return
+      if (message.action === "busy") end(BUSY, false)
+      else if (message.action === "bye") end(FAILED, false)
+      else if (message.action === "sdp" && message.sdp && !pc)
+        answer(message).catch(() => end(FAILED, true))
+      else if (message.action === "candidate" && message.candidate) {
+        if (pc?.remoteDescription) add(message.candidate)
+        else early.push(message.candidate)
+      }
+    })
+    const timer = setTimeout(() => end(FAILED, true), FIRST_FRAME_MS)
+    running.current = {
+      stop: () => end("", true),
+      played: () => {
+        clearTimeout(timer)
+        setPhase("playing")
+      },
+    }
+    setNote("")
+    setPhase("connecting")
+    if (!sendLive({ device, session, action: "start" })) end(FAILED, false)
+  }, [device])
+
+  // A tile that goes away, or is pointed at another device, ends its view.
+  useEffect(() => {
+    if (!device) return
+    return () => running.current?.stop()
+  }, [device])
+
+  useEffect(() => {
+    if (!note) return
+    const timer = setTimeout(() => setNote(""), NOTE_MS)
+    return () => clearTimeout(timer)
+  }, [note])
+
+  return { phase, stream, note, start, stop, played }
+}

frontend/src/components/Devices/DeviceDetail.tsx

@@ -44,6 +44,7 @@ import {
   type DeviceReport,
   deviceNodeOf,
   freeNodeId,
+  LIVE_PORT,
   type ReportedPort,
   reportedPorts,
   reportOf,
@@ -233,7 +234,7 @@ function Camera({ device }: { device: DevicePublic }) {
     <section className={CARD}>
       <PanelSection
         title="Camera"
-        help="Live while this preview is on screen. The camera sends frames only while someone watches them here or on a dashboard, or a running flow reads them."
+        help="Frames while this preview is on screen, which the camera sends only while someone watches them here or on a dashboard, or a running flow reads them. Tap it for live video when the device streams live."
       >
         <div
           className="flex aspect-video w-full overflow-hidden rounded-md bg-muted text-center text-sm"
@@ -748,7 +749,11 @@ function Capabilities({
                     </span>
                   ) : null}
                 </span>
-                {canEdit && said && wired.length === 0 ? (
+                {said?.port === LIVE_PORT ? (
+                  <span className="shrink-0 text-xs text-muted-foreground">
+                    Live view
+                  </span>
+                ) : canEdit && said && wired.length === 0 ? (
                   <UseInFlow device={device} port={said} size="sm" />
                 ) : null}
               </li>

frontend/src/components/Devices/ports.ts

@@ -3,6 +3,12 @@ import type { DevicePublic, DType, MessageSpec, NodeDef_Input } from "@/client"
 /** The node type a device is in a flow. */
 export const DEVICE_TYPE = "device"
 
+/**
+ * The port a device reports when its camera can stream live over WebRTC.
+ * Signalled on the socket, never wired in a flow.
+ */
+export const LIVE_PORT = "live"
+
 /** One port as a device reports it. */
 export type ReportedPort = {
   port: string
@@ -57,8 +63,9 @@ export function screensOf(devices: DevicePublic[] = []) {
   })
 }
 
-/** Whether a reported port may be wired in `direction`. */
+/** Whether a reported port may be wired in `direction`. `live` never is. */
 export const wiresAs = (port: ReportedPort, direction: "in" | "out") =>
+  port.port !== LIVE_PORT &&
   (port.modes?.length ? port.modes : [port.direction]).includes(direction)
 
 /** What the server accepts as a port name. */

frontend/src/components/Flow/CommandPalette.tsx

@@ -2,7 +2,11 @@ import { useQuery } from "@tanstack/react-query"
 import { useNavigate } from "@tanstack/react-router"
 
 import type { DevicePublic, FlowSummary, NodeTypeInfo } from "@/client"
-import { type ReportedPort, reportedPorts } from "@/components/Devices/ports"
+import {
+  LIVE_PORT,
+  type ReportedPort,
+  reportedPorts,
+} from "@/components/Devices/ports"
 import { devicesQueryOptions } from "@/components/Devices/queries"
 import {
   CommandDialog,
@@ -58,7 +62,9 @@ export function CommandPalette({
   // A shortcut per port a device says it has, "Hallway camera". A device that
   // has not said is added as a Device node and its ports typed in.
   const capabilities = (devicesData?.data ?? []).flatMap((device) =>
-    (reportedPorts(device) ?? []).map((port) => ({ device, port })),
+    (reportedPorts(device) ?? [])
+      .filter((port) => port.port !== LIVE_PORT)
+      .map((port) => ({ device, port })),
   )
 
   const close = (action: () => void) => {

frontend/src/components/Flow/useFlowSocket.ts

@@ -134,6 +134,23 @@ type FlowEvent =
       names: string[]
       ts?: number
     }
+  | LiveSignal
+
+/**
+ * A live view's signalling, both ways on this socket (`fluksio.flow.live`).
+ * The page sends `start`, `sdp`, `candidate` and `bye`, and hears `sdp` (the
+ * device's offer, with the ICE servers to use), `candidate`, `bye` and `busy`.
+ */
+export type LiveSignal = {
+  type: "live"
+  device: string
+  session: string
+  action: "start" | "sdp" | "candidate" | "bye" | "busy"
+  sdp?: string
+  candidate?: string
+  reason?: string
+  ice_servers?: RTCIceServer[]
+}
 
 async function socketUrl(): Promise<string> {
   const base = String(OpenAPI.BASE || window.location.origin)
@@ -176,6 +193,42 @@ let client: QueryClient | null = null
 const authHandlers = new Set<() => void>()
 /** Floor re-reads still owed, per node, so a recovery can cancel its own. */
 const healthTimers = new Map<string, ReturnType<typeof setTimeout>>()
+/** What each live session on this page does with what it hears. */
+const liveListeners = new Map<string, (message: LiveSignal) => void>()
+
+/** Send a live view's signalling. False while the socket is not open. */
+export function sendLive(message: Omit<LiveSignal, "type">): boolean {
+  if (socket?.readyState !== WebSocket.OPEN) return false
+  socket.send(JSON.stringify({ type: "live", ...message }))
+  return true
+}
+
+/**
+ * Hear one live session's signalling, until the returned function is called.
+ * A socket that drops ends every session on it, so each hears `bye` then.
+ */
+export function listenLive(
+  session: string,
+  listener: (message: LiveSignal) => void,
+): () => void {
+  liveListeners.set(session, listener)
+  return () => {
+    if (liveListeners.get(session) === listener) liveListeners.delete(session)
+  }
+}
+
+/** The socket went: so did every live session riding on it. */
+function endLive() {
+  for (const [session, listener] of [...liveListeners]) {
+    listener({
+      type: "live",
+      device: "",
+      session,
+      action: "bye",
+      reason: "socket",
+    })
+  }
+}
 
 /** Everything a node's health shows up in, which is two unrelated keys. */
 function invalidateHealth(flow: string | undefined) {
@@ -436,6 +489,9 @@ function open(url: string) {
         // live value, so nothing here goes through the live store.
         client?.invalidateQueries({ queryKey: runKeys.all })
         break
+      case "live":
+        liveListeners.get(message.session)?.(message)
+        break
       case "run_metric":
         // A batch of readings was written. Only what draws them is refetched —
         // the run's own detail (its metric names and curve hang below that
@@ -480,6 +536,7 @@ function open(url: string) {
     socket = null
     liveStore.setConnected(false)
     connectionStore.setSocketOpen(false)
+    endLive()
     if (watchers === 0) return
     if (event.code === 1008 && refused()) return
     // 1013 is the portal saying the instance is not attached — the one
@@ -504,6 +561,7 @@ function release(onAuthFailure?: () => void) {
   const ws = socket
   socket = null
   ws?.close()
+  endLive()
   liveStore.setConnected(false)
   // Its `onclose` is dropped by the guard above, so say so here instead.
   connectionStore.setSocketOpen(false)