Live view over WebRTC (openpanel L1, server and browser)

A device reporting a `live` port streams its camera to the browser's own
RTCPeerConnection. The flows socket carries the signalling as text, so it
crosses the portal: `live` messages on a person's socket, `signal` on a
device's (start, sdp, candidate, bye, busy). The server routes by session,
holds a device to two viewers, ends a session when either socket closes or
no answer comes within 10 s, and gives each socket an outbox of its own
that ends a session rather than drop a message. TURN credentials are
minted per session in coturn's REST scheme (STUN_URLS, TURN_URLS,
TURN_SECRET), the viewer's with the device's offer; GET /devices/{id}/ice
hands them out too. The tunnel keeps a stream's early frames instead of
dropping them past eight and forgets them when the stream or link goes.

A media tile on a device's camera, and the Devices tab's preview, play
live when tapped, pause frames meanwhile, and fall back to frames with a
short note on busy, failure or no picture within 10 s. coturn is an
optional compose service behind the `turn` profile.
This commit is contained in:
Melvin Strobl
2026-10-17 08:37:39 +02:00
parent efd883d092
commit 6a98dcd23f
20 changed files with 1535 additions and 24 deletions
+65 -7
View File
@@ -73,7 +73,9 @@ ALLOWED_PATHS = frozenset(
#: reference to a task that was still running.
MAX_IN_FLIGHT = 256
#: How many of a client's frames are kept for a stream not listening yet.
MAX_EARLY = 8
#: One past it closes the stream rather than losing a frame: a live view's
#: candidates are worth nothing with one missing.
MAX_EARLY = 64
#: How often to look for a config that appeared while the engine was up.
#: Enrolment is something a person does and then waits on, so this is the
#: delay they sit through — short enough not to be worth a restart.
@@ -299,7 +301,20 @@ class CloudConnector:
# A browser speaks as soon as the hub accepts it,
# before this end has checked its credential.
early = self._early.setdefault(stream, [])
early[:] = [*early[-(MAX_EARLY - 1) :], text]
if len(early) < MAX_EARLY:
early.append(text)
else:
logger.warning(
"Closing stream %s: %d frames before it listened",
stream,
MAX_EARLY,
)
self._cancel_stream(stream)
await socket.send(
_dump(
{"op": "ws_close", "id": stream, "code": 1011}
)
)
elif op == "ws_close":
self._cancel_stream(str(frame.get("id") or ""))
elif op == "member_removed":
@@ -321,6 +336,7 @@ class CloudConnector:
task.cancel()
self._calls.clear()
self._streams.clear()
self._early.clear()
return attached
@staticmethod
@@ -631,12 +647,16 @@ class CloudConnector:
"Refusing stream %s: %d already open", stream_id, len(self._streams)
)
return
existing = self._streams.pop(stream_id, None)
if existing is not None:
existing.cancel()
self._cancel_stream(stream_id)
task = asyncio.create_task(self._stream(socket, frame))
self._streams[stream_id] = task
task.add_done_callback(lambda _t: self._streams.pop(stream_id, None))
task.add_done_callback(lambda done: self._stream_done(stream_id, done))
def _stream_done(self, stream_id: str, task: asyncio.Task[None]) -> None:
# Its own entry only: an id the hub reused belongs to the newer task.
if self._streams.get(stream_id) is task:
del self._streams[stream_id]
self._early.pop(stream_id, None)
async def _stream(self, socket: Any, frame: dict[str, Any]) -> None:
"""Feed the engine's live events down the tunnel.
@@ -661,6 +681,7 @@ class CloudConnector:
)
from fluksio.api.socket_auth import redeem_ticket
from fluksio.core.db import engine
from fluksio.flow import live as live_view
from fluksio.flow.demand import demand
from fluksio.flow.devices import presence
from fluksio.flow.events import event_bus
@@ -714,6 +735,11 @@ class CloudConnector:
# are fetched one by one through the portal.
viewer = ("tunnel", stream_id)
wanted: set[str] = set()
# Where a live view this stream is one end of reaches it. Its
# signalling is text, so it crosses, on an outbox of its own.
endpoint = live_view.Endpoint()
outbox: asyncio.Task[str] | None = None
sender: asyncio.Task[dict[str, Any]] | None = None
def heard(text: str) -> None:
nonlocal wanted
@@ -721,6 +747,14 @@ class CloudConnector:
if asked is not None:
wanted = asked
demand.want(viewer, asked)
elif (signal := live_view.parse(text)) is not None:
live_view.sessions.heard(
signal,
endpoint,
person=str(user.id) if only is None else "",
device=device,
devices=getattr(self._app.state, "device_store", None),
)
self._listeners[stream_id] = heard
for text in self._early.pop(stream_id, []):
@@ -728,8 +762,10 @@ class CloudConnector:
if device:
presence.opened(device)
live_view.sessions.attach(device, endpoint)
try:
async with event_bus.subscribe() as queue:
outbox = asyncio.create_task(endpoint.next())
if device and (ports := demand.ports(device)) != told:
await socket.send(
_dump(
@@ -741,7 +777,22 @@ class CloudConnector:
)
)
while True:
event = await queue.get()
sender = asyncio.create_task(queue.get())
done, _ = await asyncio.wait(
{sender, outbox}, return_when=asyncio.FIRST_COMPLETED
)
if outbox in done:
# Signalling, for this stream alone.
if sender not in done:
sender.cancel()
text = outbox.result()
outbox = asyncio.create_task(endpoint.next())
await socket.send(
_dump({"op": "ws_msg", "id": stream_id, "text": text})
)
if sender not in done:
continue
event = sender.result()
if event.get("type") == "device_demand":
# For the device it names, and for nobody else.
if device and event.get("device") == device:
@@ -793,9 +844,13 @@ class CloudConnector:
_dump({"op": "ws_close", "id": stream_id, "code": 1011})
)
finally:
for task in (outbox, sender):
if task is not None:
task.cancel()
self._listeners.pop(stream_id, None)
self._early.pop(stream_id, None)
demand.drop(viewer)
live_view.sessions.detach(endpoint)
if device:
presence.closed(device)
@@ -867,6 +922,9 @@ class CloudConnector:
)
def _cancel_stream(self, stream_id: str) -> None:
# The frames it was holding go with it, including those of a stream
# cancelled before it ever ran its own cleanup.
self._early.pop(stream_id, None)
task = self._streams.pop(stream_id, None)
if task is not None:
task.cancel()