tts speaks MP3: a seventh of WAV's bytes over a slow link to the server

A sentence's length for `append` comes from the Xing header LAME writes into
the first frame. The cache key carries the format, so no WAV is reused.
This commit is contained in:
Melvin Strobl
2026-10-19 21:55:17 +02:00
parent 16383ea6e2
commit f50d8b9570
4 changed files with 35 additions and 22 deletions
+6 -2
View File
@@ -189,7 +189,11 @@ async def speak(
model: str = "tts-1", model: str = "tts-1",
timeout: float = 30.0, timeout: float = 30.0,
) -> bytes: ) -> bytes:
"""``POST {api_base}/audio/speech``: ``text`` spoken, as WAV.""" """``POST {api_base}/audio/speech``: ``text`` spoken, as MP3.
MP3 rather than WAV: a seventh of the bytes, which over a slow link to
the server is most of the wait for a sentence.
"""
async with httpx.AsyncClient(timeout=timeout) as http: async with httpx.AsyncClient(timeout=timeout) as http:
response = await http.post( response = await http.post(
api_base.rstrip("/") + "/audio/speech", api_base.rstrip("/") + "/audio/speech",
@@ -197,7 +201,7 @@ async def speak(
"model": model, "model": model,
"input": text, "input": text,
"voice": voice, "voice": voice,
"response_format": "wav", "response_format": "mp3",
}, },
) )
if response.is_error: if response.is_error:
+20 -7
View File
@@ -10,12 +10,10 @@ from __future__ import annotations
import asyncio import asyncio
import hashlib import hashlib
import io
import logging import logging
import re import re
import threading import threading
import time import time
import wave
from collections.abc import Coroutine, Iterable from collections.abc import Coroutine, Iterable
from concurrent.futures import Future from concurrent.futures import Future
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
@@ -68,6 +66,23 @@ def sentences(text: str) -> list[str]:
return [s for s in (p.strip() for p in re.split(r"(?<=[.!?])\s+", text)) if s] return [s for s in (p.strip() for p in re.split(r"(?<=[.!?])\s+", text)) if s]
def mp3_seconds(data: bytes) -> float:
"""An MP3's length, from the frame count LAME writes into its first frame."""
header = int.from_bytes(data[:4], "big")
version = (header >> 19) & 3 # 3 is MPEG-1, 2 MPEG-2, 0 MPEG-2.5
rates = {
3: (44100, 48000, 32000),
2: (22050, 24000, 16000),
0: (11025, 12000, 8000),
}
rate = rates[version][(header >> 10) & 3]
at = max(data.find(b"Xing", 0, 64), data.find(b"Info", 0, 64))
if at < 0:
raise ValueError("the MP3 has no Xing or Info header to read its length from")
frames = int.from_bytes(data[at + 8 : at + 12], "big")
return round(frames * (1152 if version == 3 else 576) / rate, 3)
class _Speech(Node): class _Speech(Node):
"""What both nodes share: the url, the artifact store and their health.""" """What both nodes share: the url, the artifact store and their health."""
@@ -334,19 +349,17 @@ class TtsNode(_Speech):
def _spoken(self, url: str, sentence: str) -> dict[str, Any]: def _spoken(self, url: str, sentence: str) -> dict[str, Any]:
"""The reference to ``sentence`` spoken, from the cache or the server.""" """The reference to ``sentence`` spoken, from the cache or the server."""
key = hashlib.sha256(f"{self.cfg.voice}\n{sentence}".encode()).hexdigest() key = hashlib.sha256(f"mp3\n{self.cfg.voice}\n{sentence}".encode()).hexdigest()
# Kept in state, so the sweep keeps the bytes it refers to. # Kept in state, so the sweep keeps the bytes it refers to.
spoken = dict(self.recall("spoken") or {}) spoken = dict(self.recall("spoken") or {})
reference = spoken.get(key) reference = spoken.get(key)
if reference is not None and self._artifacts.has(reference["digest"]): if reference is not None and self._artifacts.has(reference["digest"]):
return dict(reference) return dict(reference)
data = asyncio.run(client.speak(url + "/v1", sentence, self.cfg.voice)) data = asyncio.run(client.speak(url + "/v1", sentence, self.cfg.voice))
with wave.open(io.BytesIO(data)) as clip:
seconds = round(clip.getnframes() / clip.getframerate(), 3)
stored: dict[str, Any] = self._artifacts.put( stored: dict[str, Any] = self._artifacts.put(
[data], name="speech.wav", media_type="audio/wav" [data], name="speech.mp3", media_type="audio/mpeg"
) )
reference = stored | {"meta": {"seconds": seconds}} reference = stored | {"meta": {"seconds": mp3_seconds(data)}}
spoken.pop(key, None) spoken.pop(key, None)
spoken[key] = reference spoken[key] = reference
self.remember("spoken", dict(list(spoken.items())[-SPOKEN:])) self.remember("spoken", dict(list(spoken.items())[-SPOKEN:]))
+8 -12
View File
@@ -2,8 +2,6 @@
import asyncio import asyncio
import hashlib import hashlib
import io
import wave
from typing import Any from typing import Any
from fluksio.ai import client from fluksio.ai import client
@@ -105,14 +103,11 @@ def test_an_utterance_is_one_session_committed_when_listening_ends(monkeypatch):
asyncio.run(scenario()) asyncio.run(scenario())
def _wav(seconds: float) -> bytes: def _mp3(frames: int) -> bytes:
out = io.BytesIO() """The first frame LAME writes: MPEG-2 layer III at 24 kHz, mono, and the
with wave.open(out, "wb") as clip: Xing header counting the frames, 576 samples each."""
clip.setnchannels(1) xing = b"Xing" + (15).to_bytes(4, "big") + frames.to_bytes(4, "big")
clip.setsampwidth(2) return bytes.fromhex("fff384c4") + bytes(9) + xing + bytes(100)
clip.setframerate(24000)
clip.writeframes(b"\0\0" * int(24000 * seconds))
return out.getvalue()
def test_a_sentence_is_spoken_once_and_the_rest_of_a_reply_queues(monkeypatch): def test_a_sentence_is_spoken_once_and_the_rest_of_a_reply_queues(monkeypatch):
@@ -121,7 +116,7 @@ def test_a_sentence_is_spoken_once_and_the_rest_of_a_reply_queues(monkeypatch):
async def speak(api_base, text, voice): async def speak(api_base, text, voice):
spoken.append(text) spoken.append(text)
return _wav(len(spoken)) return _mp3(25 * len(spoken))
monkeypatch.setattr(client, "speak", speak) monkeypatch.setattr(client, "speak", speak)
monkeypatch.setattr(TtsNode, "_say", lambda self, speech: published.append(speech)) monkeypatch.setattr(TtsNode, "_say", lambda self, speech: published.append(speech))
@@ -144,4 +139,5 @@ def test_a_sentence_is_spoken_once_and_the_rest_of_a_reply_queues(monkeypatch):
assert spoken == ["Kitchen light on.", "Heating off."] assert spoken == ["Kitchen light on.", "Heating off."]
assert ["append" in speech for speech in published] == [False, True, False] assert ["append" in speech for speech in published] == [False, True, False]
assert published[2] == published[0] assert published[2] == published[0]
assert published[1]["meta"] == {"seconds": 2.0} assert published[1]["meta"] == {"seconds": 1.2}
assert published[1]["media_type"] == "audio/mpeg"
+1 -1
View File
@@ -453,7 +453,7 @@ such as one that sends whole clips.
| Output | Type | Notes | | Output | Type | Notes |
|---|---|---| |---|---|---|
| `speech` | `audio` | one WAV reference per sentence | | `speech` | `audio` | one MP3 reference per sentence |
| Setting | Default | Notes | | Setting | Default | Notes |
|---|---|---| |---|---|---|