Voice nodes (openpanel V4): stt, tts, wren reply/unhandled, socket publish

- `stt`: one realtime transcription session per utterance on the speech
  server's url, opened at `listening` true or the first chunk, committed at
  false; a whole clip goes to /v1/audio/transcriptions. Returns at once and
  publishes `partial` and `text` when the server answers.
- `tts`: one /v1/audio/speech call per sentence, cached by voice and text in
  node state; a reference is marked `append` while earlier ones still play.
- wren: `reply` says what was applied, `unhandled` carries the prompt when
  nothing was. The install hint names the git pin (PyPI's `wren` is another
  package).
- ai/client: transcribe, speak and transcribe_stream beside stream_completion.
- A device socket publishes scalar `in` ports as {"type": "publish"} text,
  in order with its binary frames.
- Pipeline.publish walks only what its values reach: a node downstream of a
  consumer that published nothing no longer runs again (wren re-acted on
  every microphone chunk).
This commit is contained in:
Melvin Strobl
2026-10-19 21:34:02 +02:00
parent 85dad39973
commit 16383ea6e2
15 changed files with 890 additions and 24 deletions
+76 -10
View File
@@ -46,6 +46,7 @@ from fluksio.flow.controller import NODE_TYPES, FlowController, follow_header
from fluksio.flow.dashboards import DashboardStore
from fluksio.flow.demand import demand
from fluksio.flow.devices import (
Binding,
DeviceDef,
DeviceStore,
DeviceUnavailable,
@@ -1362,6 +1363,7 @@ FRAME_REFUSALS = {
"not_media": "the port it names is not wired as an image or audio",
"media_type": "its media type does not match its port",
"too_large": "it is larger than MAX_FRAME_BYTES",
"not_scalar": "the port it names is not wired as a bool, int, float or str",
"undeclared": "no flow declares its message",
"refused": "its message refused the value",
}
@@ -1393,14 +1395,7 @@ def ingest_frame(app: Any, device: DeviceDef, data: bytes) -> str:
controller: FlowController | None = getattr(app.state, "flow_controller", None)
if devices is None or store is None or controller is None:
return "no_engine"
binding = next(
(
b
for b in devices.bound(device.id)
if b.direction == "in" and b.message == name
),
None,
)
binding = _bound_in(devices, device, name)
if binding is None:
return "not_wired"
if binding.dtype not in (DType.IMAGE, DType.AUDIO):
@@ -1420,6 +1415,59 @@ def ingest_frame(app: Any, device: DeviceDef, data: bytes) -> str:
return ""
def _bound_in(devices: DeviceStore, device: DeviceDef, name: str) -> Binding | None:
"""The device's own ``in`` port wired to ``name``, if one is."""
return next(
(
b
for b in devices.bound(device.id)
if b.direction == "in" and b.message == name
),
None,
)
def published(frame: str) -> dict[str, Any] | None:
"""A value a device publishes on its socket, or None for anything else."""
try:
payload = orjson.loads(frame)
except orjson.JSONDecodeError:
return None
if isinstance(payload, dict) and payload.get("type") == "publish":
return payload
return None
def ingest_publish(app: Any, device: DeviceDef, payload: dict[str, Any]) -> str:
"""Publish a value a device sent on its socket, as ``POST /messages`` would.
Returns the key in ``FRAME_REFUSALS`` it was refused for, or "" once it
is published. Only a message one of the device's own ``in`` ports is wired
to as a scalar, credited to the device node holding that port. Taken in
turn with the device's binary frames, so a ``listening`` false lands after
the last chunk of what was heard.
"""
name = str(payload.get("name") or "")
devices: DeviceStore | None = getattr(app.state, "device_store", None)
controller: FlowController | None = getattr(app.state, "flow_controller", None)
if devices is None or controller is None:
return "no_engine"
binding = _bound_in(devices, device, name)
if binding is None:
return "not_wired"
if binding.dtype not in (DType.BOOL, DType.INT, DType.FLOAT, DType.STR):
return "not_scalar"
try:
controller.publish_message(
name, payload.get("value"), port_source(devices, device, name)
)
except KeyError:
return "undeclared"
except TypeError:
return "refused"
return ""
async def _receive(websocket: WebSocket) -> str | bytes:
"""The client's next frame, text or binary."""
message = await websocket.receive()
@@ -1465,8 +1513,9 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
A client says which media it is drawing (``{"type": "media", "names":
[...]}``) and is pushed those frames. A device also sends its camera's
frames here, as binary frames in the layout it is pushed them in, and is
told when its on-demand ports have demand.
frames here, as binary frames in the layout it is pushed them in, and
the values of its scalar ports as ``publish`` messages, and is told when
its on-demand ports have demand.
A live view's signalling rides here too (``fluksio.flow.live``): ``live``
messages on a person's socket, ``signal`` messages on a device's.
@@ -1628,6 +1677,23 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
if asked is not None:
wanted = asked
demand.want(viewer, wanted)
elif (
device_def is not None
and (value := published(frame)) is not None
):
if credential is not None and await credential.revoked():
await websocket.close(code=1008)
return
reason = await run_in_threadpool(
ingest_publish, websocket.app, device_def, value
)
if reason and reason not in refused:
refused.add(reason)
logger.warning(
"Dropped a value from %s: %s",
device,
FRAME_REFUSALS[reason],
)
elif (signal := live_view.parse(frame)) is not None:
live_view.sessions.heard(
signal,