Voice nodes (openpanel V4): stt, tts, wren reply/unhandled, socket publish
- `stt`: one realtime transcription session per utterance on the speech
server's url, opened at `listening` true or the first chunk, committed at
false; a whole clip goes to /v1/audio/transcriptions. Returns at once and
publishes `partial` and `text` when the server answers.
- `tts`: one /v1/audio/speech call per sentence, cached by voice and text in
node state; a reference is marked `append` while earlier ones still play.
- wren: `reply` says what was applied, `unhandled` carries the prompt when
nothing was. The install hint names the git pin (PyPI's `wren` is another
package).
- ai/client: transcribe, speak and transcribe_stream beside stream_completion.
- A device socket publishes scalar `in` ports as {"type": "publish"} text,
in order with its binary frames.
- Pipeline.publish walks only what its values reach: a node downstream of a
consumer that published nothing no longer runs again (wren re-acted on
every microphone chunk).
This commit is contained in:
@@ -46,6 +46,7 @@ from fluksio.flow.controller import NODE_TYPES, FlowController, follow_header
|
||||
from fluksio.flow.dashboards import DashboardStore
|
||||
from fluksio.flow.demand import demand
|
||||
from fluksio.flow.devices import (
|
||||
Binding,
|
||||
DeviceDef,
|
||||
DeviceStore,
|
||||
DeviceUnavailable,
|
||||
@@ -1362,6 +1363,7 @@ FRAME_REFUSALS = {
|
||||
"not_media": "the port it names is not wired as an image or audio",
|
||||
"media_type": "its media type does not match its port",
|
||||
"too_large": "it is larger than MAX_FRAME_BYTES",
|
||||
"not_scalar": "the port it names is not wired as a bool, int, float or str",
|
||||
"undeclared": "no flow declares its message",
|
||||
"refused": "its message refused the value",
|
||||
}
|
||||
@@ -1393,14 +1395,7 @@ def ingest_frame(app: Any, device: DeviceDef, data: bytes) -> str:
|
||||
controller: FlowController | None = getattr(app.state, "flow_controller", None)
|
||||
if devices is None or store is None or controller is None:
|
||||
return "no_engine"
|
||||
binding = next(
|
||||
(
|
||||
b
|
||||
for b in devices.bound(device.id)
|
||||
if b.direction == "in" and b.message == name
|
||||
),
|
||||
None,
|
||||
)
|
||||
binding = _bound_in(devices, device, name)
|
||||
if binding is None:
|
||||
return "not_wired"
|
||||
if binding.dtype not in (DType.IMAGE, DType.AUDIO):
|
||||
@@ -1420,6 +1415,59 @@ def ingest_frame(app: Any, device: DeviceDef, data: bytes) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def _bound_in(devices: DeviceStore, device: DeviceDef, name: str) -> Binding | None:
|
||||
"""The device's own ``in`` port wired to ``name``, if one is."""
|
||||
return next(
|
||||
(
|
||||
b
|
||||
for b in devices.bound(device.id)
|
||||
if b.direction == "in" and b.message == name
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def published(frame: str) -> dict[str, Any] | None:
|
||||
"""A value a device publishes on its socket, or None for anything else."""
|
||||
try:
|
||||
payload = orjson.loads(frame)
|
||||
except orjson.JSONDecodeError:
|
||||
return None
|
||||
if isinstance(payload, dict) and payload.get("type") == "publish":
|
||||
return payload
|
||||
return None
|
||||
|
||||
|
||||
def ingest_publish(app: Any, device: DeviceDef, payload: dict[str, Any]) -> str:
|
||||
"""Publish a value a device sent on its socket, as ``POST /messages`` would.
|
||||
|
||||
Returns the key in ``FRAME_REFUSALS`` it was refused for, or "" once it
|
||||
is published. Only a message one of the device's own ``in`` ports is wired
|
||||
to as a scalar, credited to the device node holding that port. Taken in
|
||||
turn with the device's binary frames, so a ``listening`` false lands after
|
||||
the last chunk of what was heard.
|
||||
"""
|
||||
name = str(payload.get("name") or "")
|
||||
devices: DeviceStore | None = getattr(app.state, "device_store", None)
|
||||
controller: FlowController | None = getattr(app.state, "flow_controller", None)
|
||||
if devices is None or controller is None:
|
||||
return "no_engine"
|
||||
binding = _bound_in(devices, device, name)
|
||||
if binding is None:
|
||||
return "not_wired"
|
||||
if binding.dtype not in (DType.BOOL, DType.INT, DType.FLOAT, DType.STR):
|
||||
return "not_scalar"
|
||||
try:
|
||||
controller.publish_message(
|
||||
name, payload.get("value"), port_source(devices, device, name)
|
||||
)
|
||||
except KeyError:
|
||||
return "undeclared"
|
||||
except TypeError:
|
||||
return "refused"
|
||||
return ""
|
||||
|
||||
|
||||
async def _receive(websocket: WebSocket) -> str | bytes:
|
||||
"""The client's next frame, text or binary."""
|
||||
message = await websocket.receive()
|
||||
@@ -1465,8 +1513,9 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
|
||||
|
||||
A client says which media it is drawing (``{"type": "media", "names":
|
||||
[...]}``) and is pushed those frames. A device also sends its camera's
|
||||
frames here, as binary frames in the layout it is pushed them in, and is
|
||||
told when its on-demand ports have demand.
|
||||
frames here, as binary frames in the layout it is pushed them in, and
|
||||
the values of its scalar ports as ``publish`` messages, and is told when
|
||||
its on-demand ports have demand.
|
||||
|
||||
A live view's signalling rides here too (``fluksio.flow.live``): ``live``
|
||||
messages on a person's socket, ``signal`` messages on a device's.
|
||||
@@ -1628,6 +1677,23 @@ async def flow_events(websocket: WebSocket, ticket: str = "", token: str = "") -
|
||||
if asked is not None:
|
||||
wanted = asked
|
||||
demand.want(viewer, wanted)
|
||||
elif (
|
||||
device_def is not None
|
||||
and (value := published(frame)) is not None
|
||||
):
|
||||
if credential is not None and await credential.revoked():
|
||||
await websocket.close(code=1008)
|
||||
return
|
||||
reason = await run_in_threadpool(
|
||||
ingest_publish, websocket.app, device_def, value
|
||||
)
|
||||
if reason and reason not in refused:
|
||||
refused.add(reason)
|
||||
logger.warning(
|
||||
"Dropped a value from %s: %s",
|
||||
device,
|
||||
FRAME_REFUSALS[reason],
|
||||
)
|
||||
elif (signal := live_view.parse(frame)) is not None:
|
||||
live_view.sessions.heard(
|
||||
signal,
|
||||
|
||||
Reference in New Issue
Block a user