Cameras on devices (openpanel C1): frames in memory, a binary uplink, demand

- Volatile frames are held in memory with a byte-bounded queue instead of the /dev/shm ring scanned on every put; workers keep writing to the directory, which is trimmed on a timer. A volatile upload no longer spools to disk. ARTIFACT_VOLATILE_BYTES=0 turns the ring off. - A device sends camera frames as binary messages on its socket, held and published in one step and credited to its device node; refused frames are logged and dropped. Its snapshot carries demand and the uplink (binary, or http through the portal). - One demand registry for every socket's wanted media, the tunnel's included, plus what running flows read, mapped onto on-demand ports and published as device_demand; at most 8 media names per socket. - Per-message deflate off in serve and dev; a device may fetch the last 8 digests of a message; published dashboards cached per store revision; a revoked device socket is closed with 1008. - Media tiles want bytes only while on screen; the Devices tab previews a wired camera; the device node warns when a flow reads an on-demand port.
2026-10-16 20:43:14 +02:00
parent d79203a5f7
commit ecc4bee4bb
29 changed files with 1425 additions and 220 deletions
@@ -1,3 +1,6 @@
+import { useRef } from "react"
+
+import { useOnScreen } from "@/hooks/useOnScreen"
 import { isRef, useArtifactUrl } from "@/lib/media"
 import { cn } from "@/lib/utils"
 import { useBoundValue } from "./dataContext"
@@ -15,74 +18,126 @@ import type { WidgetProps } from "./widgets"
 */
 export function MediaWidget({ widget }: WidgetProps) {
  const cfg = config(widget)
-  const message = text(cfg.message)
-  const stream = text(cfg.stream_url)
-  const live = useBoundValue(message || undefined)
-  const value = live?.value
-  const ref = isRef(value) ? value : null
-  const url = useArtifactUrl(ref, message || undefined)
-
-  const kind = (ref?.media_type ?? text(cfg.dtype)).split("/")[0]
-  const fit = text(cfg.fit) === "contain" ? "object-contain" : "object-cover"
-  const label = widget.title || ref?.name || message
-
-  // A camera serving its own stream is played from source: the engine carries
-  // references at whatever rate the flow publishes them, which is not video.
-  if (stream && kind !== "audio") {
-    return (
-      <img
-        src={stream}
-        alt={label}
-        className={cn("h-full w-full rounded-md", fit)}
-      />
-    )
-  }
-
-  if (!message && !stream) {
-    return <p className="text-muted-foreground">Pick a message.</p>
-  }
-  if (!ref) {
-    return <p className="text-muted-foreground">Nothing published yet.</p>
-  }
-  if (!url) {
-    return <p className="text-muted-foreground">Loading…</p>
-  }
-
-  if (kind === "audio") {
-    return (
-      <audio
-        // Keyed on the digest so a new clip replaces the element rather than
-        // leaving the last one's playhead on it.
-        key={ref.digest}
-        src={url}
-        controls
-        autoPlay={Boolean(cfg.autoplay)}
-        className="w-full"
-      >
-        <track kind="captions" />
-      </audio>
-    )
-  }
-
-  if (kind === "video") {
-    return (
-      <video
-        key={ref.digest}
-        src={url}
-        controls
-        autoPlay={Boolean(cfg.autoplay)}
-        className={cn("h-full w-full rounded-md", fit)}
-      >
-        <track kind="captions" />
-      </video>
-    )
-  }
-
  return (
-    <img
-      src={url}
-      alt={label}
-      className={cn("h-full w-full rounded-md", fit)}
+    <MediaView
+      message={text(cfg.message)}
+      stream={text(cfg.stream_url)}
+      dtype={text(cfg.dtype)}
+      fit={text(cfg.fit) === "contain" ? "contain" : "cover"}
+      autoplay={Boolean(cfg.autoplay)}
+      label={widget.title}
    />
  )
 }
+
+/**
+ * The tile's body, for anywhere a message's media is shown.
+ *
+ * Frames are asked for only while the tile is on screen, in a tab being
+ * looked at: a camera is only sent while somebody watches it, and a tile
+ * scrolled away or behind another tab is nobody. A clip keeps playing either
+ * way.
+ */
+export function MediaView({
+  message,
+  stream = "",
+  dtype = "",
+  fit = "cover",
+  autoplay = false,
+  label,
+}: {
+  message: string
+  stream?: string
+  dtype?: string
+  fit?: "cover" | "contain"
+  autoplay?: boolean
+  label?: string
+}) {
+  const box = useRef<HTMLDivElement>(null)
+  const shown = useOnScreen(box)
+  const live = useBoundValue(message || undefined)
+  const value = live?.value
+  const ref = isRef(value) ? value : null
+  const kind = (ref?.media_type ?? dtype).split("/")[0]
+  const playing = kind === "audio" || kind === "video"
+  const active = shown || playing
+  const url = useArtifactUrl(
+    active ? ref : null,
+    active ? message || undefined : undefined,
+  )
+
+  const objectFit = fit === "contain" ? "object-contain" : "object-cover"
+  const alt = label || ref?.name || message
+
+  return (
+    <div
+      ref={box}
+      className="flex min-h-0 w-full flex-1 flex-col justify-center"
+    >
+      {body()}
+    </div>
+  )
+
+  function body() {
+    // A camera serving its own stream is played from source: the engine
+    // carries references at whatever rate the flow publishes them, which is
+    // not video.
+    if (stream && kind !== "audio") {
+      return (
+        <img
+          src={stream}
+          alt={alt}
+          className={cn("h-full w-full rounded-md", objectFit)}
+        />
+      )
+    }
+
+    if (!message && !stream) {
+      return <p className="text-muted-foreground">Pick a message.</p>
+    }
+    if (!ref) {
+      return <p className="text-muted-foreground">Nothing published yet.</p>
+    }
+    if (!url) {
+      return <p className="text-muted-foreground">Loading…</p>
+    }
+
+    if (kind === "audio") {
+      return (
+        <audio
+          // Keyed on the digest so a new clip replaces the element rather than
+          // leaving the last one's playhead on it.
+          key={ref.digest}
+          src={url}
+          controls
+          autoPlay={autoplay}
+          className="w-full"
+        >
+          <track kind="captions" />
+        </audio>
+      )
+    }
+
+    if (kind === "video") {
+      return (
+        <video
+          key={ref.digest}
+          src={url}
+          controls
+          autoPlay={autoplay}
+          className={cn("h-full w-full rounded-md", objectFit)}
+        >
+          <track kind="captions" />
+        </video>
+      )
+    }
+
+    return (
+      <img
+        src={url}
+        alt={alt}
+        className={cn("h-full w-full rounded-md", objectFit)}
+      />
+    )
+  }
+}
@@ -19,6 +19,7 @@ import {
  FlowsService,
 } from "@/client"
 import { initials, resolveIcon } from "@/components/Dashboard/icons"
+import { MediaView } from "@/components/Dashboard/MediaWidget"
 import { dashboardsQueryOptions } from "@/components/Dashboard/queries"
 import { flowsQueryOptions } from "@/components/Flow/queries"
 import { InfoTip, PanelSection } from "@/components/Flow/SidePanel"
@@ -139,6 +140,7 @@ export function DeviceDetail({ id }: { id: string }) {
          <section className={cn(CARD, "grid gap-4")}>
            <Basics device={device} canEdit={canEdit} onChange={write} />
          </section>
+          <Camera device={device} />
          <section className={CARD}>
            <Dashboards device={device} canEdit={canEdit} onChange={write} />
          </section>
@@ -216,6 +218,38 @@ function Basics({
  )
 }

+/**
+ * What the device's camera sees, once a flow wires its `camera` port.
+ *
+ * Watching is what turns the camera on: it sends frames only while this
+ * preview, a tile or a flow reading them wants them.
+ */
+function Camera({ device }: { device: DevicePublic }) {
+  const bound = (device.bindings ?? []).find(
+    (binding) => binding.port === "camera" && binding.direction === "in",
+  )
+  if (!bound) return null
+  return (
+    <section className={CARD}>
+      <PanelSection
+        title="Camera"
+        help="Live while this preview is on screen. The camera sends frames only while someone watches them here or on a dashboard, or a running flow reads them."
+      >
+        <div
+          className="flex aspect-video w-full overflow-hidden rounded-md bg-muted text-center text-sm"
+          data-testid="device-camera"
+        >
+          <MediaView
+            message={bound.message}
+            fit="contain"
+            label={`${device.title || device.id} camera`}
+          />
+        </div>
+      </PanelSection>
+    </section>
+  )
+}
+
 /** Which dashboards the device shows, in the order its rail carries them. */
 function Dashboards({
  device,
@@ -511,6 +511,19 @@ function FlowEditorInner({
    }
  }, [key])

+  // What the flow's nodes read, for a device node to say that a camera read
+  // here sends all the time.
+  const readers = useMemo(() => {
+    const found = new Set<string>()
+    for (const node of definitions) {
+      if (node.type === DEVICE_TYPE) continue
+      for (const spec of node.requires ?? []) {
+        if (spec.name) found.add(qualify(flowName, spec.name))
+      }
+    }
+    return found
+  }, [definitions, flowName])
+
  /**
   * Where clicking an endpoint takes you: the thing it stands for.
   *
@@ -1508,6 +1521,7 @@ function FlowEditorInner({
        nodeTypes={nodeTypeInfo ?? []}
        origin={flowDoc.origin}
        suggestions={suggestions}
+        readers={readers}
        expanded={editorExpanded}
        onToggleExpand={() => setEditorExpanded((wide) => !wide)}
        onChange={updateNode}
@@ -56,6 +56,7 @@ import useAuth from "@/hooks/useAuth"
 import useCustomToast from "@/hooks/useCustomToast"
 import { inCodeEditor, useShortcuts } from "@/lib/shortcuts"
 import { cn } from "@/lib/utils"
+import { qualify } from "./deriveEdges"
 import { DtypeValue } from "./FlowBoundary"
 import {
  IDENTIFIER,
@@ -508,11 +509,14 @@ function DeviceSettings({
  node,
  flow,
  suggestions,
+  readers,
  onChange,
 }: {
  node: NodeDef_Input
  flow: string
  suggestions: PortSuggestions
+  /** Qualified messages another node of this flow reads. */
+  readers: ReadonlySet<string>
  onChange: (next: NodeDef_Input) => void
 }) {
  const { data } = useQuery(devicesQueryOptions())
@@ -591,6 +595,7 @@ function DeviceSettings({
        flow={flow}
        config={config}
        reported={reported}
+        readers={readers}
        suggestions={suggestions.provides}
        onChange={ports("provides")}
        onConfig={setConfig}
@@ -619,6 +624,7 @@ function DevicePortList({
  flow,
  config,
  reported,
+  readers,
  suggestions,
  onChange,
  onConfig,
@@ -632,6 +638,8 @@ function DevicePortList({
  config: Record<string, unknown>
  /** What the device says it has, or null when it has not said. */
  reported: ReportedPort[] | null
+  /** Qualified messages another node of this flow reads. */
+  readers?: ReadonlySet<string>
  suggestions: string[]
  /** `renamed` is a port taking another name, or dropped (`to` of null). */
  onChange: (next: MessageSpec[], renamed?: [string, string | null]) => void
@@ -809,6 +817,18 @@ function DevicePortList({
                The device does not report this port.
              </p>
            ) : null}
+            {said?.on_demand &&
+            spec.name &&
+            readers?.has(qualify(flow, spec.name)) ? (
+              <p
+                className="pl-5.5 text-xs text-status-warning"
+                data-testid="device-port-always-on"
+              >
+                A node in this flow reads every frame, so {port} sends all the
+                time, also while nobody watches. Read snapshot or person
+                instead.
+              </p>
+            ) : null}
            {schema ? (
              <div className="pl-5.5">
                <ParamsForm
@@ -1771,6 +1791,7 @@ function PanelBody({
  nodeType,
  origin,
  suggestions,
+  readers,
  expanded,
  onChange,
  onRenameMessage,
@@ -1783,6 +1804,7 @@ function PanelBody({
  nodeType: NodeTypeInfo | undefined
  origin: FlowOrigin | null | undefined
  suggestions: PortSuggestions
+  readers: ReadonlySet<string>
  expanded: boolean
  onChange: (next: NodeDef_Input) => void
  onRenameMessage: (previous: string, next: string) => void
@@ -1954,6 +1976,7 @@ function PanelBody({
            node={node}
            flow={flow}
            suggestions={suggestions}
+            readers={readers}
            onChange={editNode}
          />
        ) : (
@@ -2133,6 +2156,7 @@ export function NodePanel({
  nodeTypes,
  origin,
  suggestions,
+  readers,
  expanded,
  onChange,
  onRenameMessage,
@@ -2148,6 +2172,8 @@ export function NodePanel({
  /** Set when the flow was declared in code elsewhere; its bodies are generated. */
  origin: FlowOrigin | null | undefined
  suggestions: PortSuggestions
+  /** Qualified messages the flow's nodes read, a device node's aside. */
+  readers: ReadonlySet<string>
  expanded: boolean
  onChange: (next: NodeDef_Input) => void
  onRenameMessage: (previous: string, next: string) => void
@@ -2201,6 +2227,7 @@ export function NodePanel({
          nodeType={nodeType}
          origin={origin}
          suggestions={suggestions}
+          readers={readers}
          expanded={expanded}
          onChange={onChange}
          onRenameMessage={onRenameMessage}