Voice nodes (openpanel V4): stt, tts, wren reply/unhandled, socket publish
- `stt`: one realtime transcription session per utterance on the speech
server's url, opened at `listening` true or the first chunk, committed at
false; a whole clip goes to /v1/audio/transcriptions. Returns at once and
publishes `partial` and `text` when the server answers.
- `tts`: one /v1/audio/speech call per sentence, cached by voice and text in
node state; a reference is marked `append` while earlier ones still play.
- wren: `reply` says what was applied, `unhandled` carries the prompt when
nothing was. The install hint names the git pin (PyPI's `wren` is another
package).
- ai/client: transcribe, speak and transcribe_stream beside stream_completion.
- A device socket publishes scalar `in` ports as {"type": "publish"} text,
in order with its binary frames.
- Pipeline.publish walks only what its values reach: a node downstream of a
consumer that published nothing no longer runs again (wren re-acted on
every microphone chunk).
This commit is contained in:
@@ -6,16 +6,22 @@ OpenRouter, Ollama, vLLM — so one client covers them without a provider SDK.
|
||||
The admin types the URL; the model never chooses where a request goes.
|
||||
|
||||
The call is streamed, so the answer reaches the browser as it is written.
|
||||
|
||||
Speech is the same dialect's audio half: a realtime transcription session, a
|
||||
whole clip transcribed, and a sentence spoken, as `parrot serve` answers them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from websockets.asyncio.client import connect
|
||||
|
||||
|
||||
class ProviderError(Exception):
|
||||
@@ -152,3 +158,113 @@ async def list_models(api_base: str, api_key: str, timeout: float = 15.0) -> lis
|
||||
raise ProviderError(_detail(response))
|
||||
data = response.json().get("data") or []
|
||||
return sorted(str(m["id"]) for m in data if isinstance(m, dict) and "id" in m)
|
||||
|
||||
|
||||
async def transcribe(
|
||||
api_base: str,
|
||||
audio: bytes,
|
||||
filename: str,
|
||||
media_type: str,
|
||||
*,
|
||||
model: str = "whisper-1",
|
||||
timeout: float = 30.0,
|
||||
) -> str:
|
||||
"""``POST {api_base}/audio/transcriptions``: a whole clip, and what it says."""
|
||||
async with httpx.AsyncClient(timeout=timeout) as http:
|
||||
response = await http.post(
|
||||
api_base.rstrip("/") + "/audio/transcriptions",
|
||||
files={"file": (filename, audio, media_type)},
|
||||
data={"model": model, "response_format": "json"},
|
||||
)
|
||||
if response.is_error:
|
||||
raise ProviderError(_detail(response))
|
||||
return str(response.json().get("text") or "")
|
||||
|
||||
|
||||
async def speak(
|
||||
api_base: str,
|
||||
text: str,
|
||||
voice: str,
|
||||
*,
|
||||
model: str = "tts-1",
|
||||
timeout: float = 30.0,
|
||||
) -> bytes:
|
||||
"""``POST {api_base}/audio/speech``: ``text`` spoken, as WAV."""
|
||||
async with httpx.AsyncClient(timeout=timeout) as http:
|
||||
response = await http.post(
|
||||
api_base.rstrip("/") + "/audio/speech",
|
||||
json={
|
||||
"model": model,
|
||||
"input": text,
|
||||
"voice": voice,
|
||||
"response_format": "wav",
|
||||
},
|
||||
)
|
||||
if response.is_error:
|
||||
raise ProviderError(_detail(response))
|
||||
return response.content
|
||||
|
||||
|
||||
async def transcribe_stream(
|
||||
api_base: str,
|
||||
audio: asyncio.Queue[bytes | None],
|
||||
*,
|
||||
rate: int = 16000,
|
||||
idle_s: float = 10.0,
|
||||
) -> AsyncIterator[tuple[bool, str]]:
|
||||
"""``WS {api_base}/realtime``: one utterance, transcribed while it is spoken.
|
||||
|
||||
Sends each chunk of PCM16 taken off ``audio`` and commits at ``None``.
|
||||
Yields ``(False, delta)`` as words are agreed on, then ``(True,
|
||||
transcript)`` once. A chunk that takes longer than ``idle_s`` to come
|
||||
raises ``TimeoutError``, so an utterance nobody commits ends.
|
||||
"""
|
||||
url = "ws" + api_base.rstrip("/").removeprefix("http") + "/realtime"
|
||||
async with connect(url, open_timeout=idle_s) as ws:
|
||||
update = {"type": "audio/pcm", "rate": rate}
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"type": "transcription",
|
||||
"audio": {"input": {"format": update}},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def append() -> None:
|
||||
try:
|
||||
while (
|
||||
chunk := await asyncio.wait_for(audio.get(), idle_s)
|
||||
) is not None:
|
||||
data = base64.b64encode(chunk).decode()
|
||||
await ws.send(
|
||||
json.dumps({"type": "input_audio_buffer.append", "audio": data})
|
||||
)
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
|
||||
except BaseException:
|
||||
# Ends the reading below, which then says why.
|
||||
await ws.close()
|
||||
raise
|
||||
|
||||
sender = asyncio.create_task(append())
|
||||
sender.add_done_callback(lambda done: done.cancelled() or done.exception())
|
||||
try:
|
||||
async for raw in ws:
|
||||
event = json.loads(raw)
|
||||
kind = str(event.get("type") or "")
|
||||
if kind == "error":
|
||||
raise ProviderError(str((event.get("error") or {}).get("message")))
|
||||
if kind.endswith("input_audio_transcription.delta"):
|
||||
yield False, str(event.get("delta") or "")
|
||||
elif kind.endswith("input_audio_transcription.completed"):
|
||||
yield True, str(event.get("transcript") or "")
|
||||
return
|
||||
finally:
|
||||
sender.cancel()
|
||||
failure = (
|
||||
sender.exception() if sender.done() and not sender.cancelled() else None
|
||||
)
|
||||
raise failure or ProviderError("the session closed before its transcript")
|
||||
|
||||
Reference in New Issue
Block a user