Melvin Strobl
16383ea6e2
- `stt`: one realtime transcription session per utterance on the speech
server's url, opened at `listening` true or the first chunk, committed at
false; a whole clip goes to /v1/audio/transcriptions. Returns at once and
publishes `partial` and `text` when the server answers.
- `tts`: one /v1/audio/speech call per sentence, cached by voice and text in
node state; a reference is marked `append` while earlier ones still play.
- wren: `reply` says what was applied, `unhandled` carries the prompt when
nothing was. The install hint names the git pin (PyPI's `wren` is another
package).
- ai/client: transcribe, speak and transcribe_stream beside stream_completion.
- A device socket publishes scalar `in` ports as {"type": "publish"} text,
in order with its binary frames.
- Pipeline.publish walks only what its values reach: a node downstream of a
consumer that published nothing no longer runs again (wren re-acted on
every microphone chunk).
148 lines
4.8 KiB
Python
148 lines
4.8 KiB
Python
"""The speech nodes: one session per utterance, and each sentence spoken once."""
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import io
|
|
import wave
|
|
from typing import Any
|
|
|
|
from fluksio.ai import client
|
|
from fluksio.flow.messages import DType, MessageSpec
|
|
from fluksio.flow.nodes.speech import SttNode, TtsNode, sentences
|
|
from fluksio.flow.pipeline import Pipeline
|
|
|
|
URL = "http://speech:8100"
|
|
|
|
|
|
class Store:
|
|
"""The artifact store's surface the nodes use."""
|
|
|
|
def __init__(self) -> None:
|
|
self.data: dict[str, bytes] = {}
|
|
|
|
def put(self, chunks: list[bytes], name: str, media_type: str) -> dict[str, Any]:
|
|
data = b"".join(chunks)
|
|
digest = "sha256:" + hashlib.sha256(data).hexdigest()
|
|
self.data[digest] = data
|
|
return {"digest": digest, "size": len(data), "media_type": media_type}
|
|
|
|
def has(self, digest: str) -> bool:
|
|
return digest in self.data
|
|
|
|
def read(self, digest: str) -> list[bytes]:
|
|
return [self.data[digest]]
|
|
|
|
|
|
def test_sentences_are_cut_after_their_ending():
|
|
assert sentences(" Kitchen light on. Heating off! Is it?\nYes") == [
|
|
"Kitchen light on.",
|
|
"Heating off!",
|
|
"Is it?",
|
|
"Yes",
|
|
]
|
|
|
|
|
|
def test_an_utterance_is_one_session_committed_when_listening_ends(monkeypatch):
|
|
heard: list[bytes] = []
|
|
|
|
async def session(api_base, audio, *, rate, idle_s):
|
|
assert (api_base, rate) == (f"{URL}/v1", 16000)
|
|
while (chunk := await audio.get()) is not None:
|
|
heard.append(chunk)
|
|
yield False, "turn on"
|
|
yield False, " the"
|
|
yield True, " Turn on the light."
|
|
|
|
monkeypatch.setattr(client, "transcribe_stream", session)
|
|
|
|
async def scenario() -> None:
|
|
node = SttNode(
|
|
requires=[
|
|
MessageSpec(name="mic", port="audio", dtype=DType.AUDIO),
|
|
MessageSpec(name="listening", dtype=DType.BOOL),
|
|
MessageSpec(name="url", dtype=DType.STR),
|
|
],
|
|
provides=[
|
|
MessageSpec(name="partial", dtype=DType.STR),
|
|
MessageSpec(name="text", dtype=DType.STR),
|
|
],
|
|
)
|
|
node.assign_flow("voice", "stt")
|
|
pipeline = Pipeline([node])
|
|
store = Store()
|
|
chunks = [
|
|
store.put([bytes([n]) * 3200], "voice.mic", "audio/L16;rate=16000")
|
|
for n in range(3)
|
|
]
|
|
# Left over from before the engine started: not part of what is said.
|
|
pipeline.state["voice.mic"] = chunks[0]
|
|
await node.start()
|
|
node._artifacts = store
|
|
|
|
async def hear(audio: dict, listening: bool, url: str = URL) -> None:
|
|
inputs = {"voice.mic": audio, "voice.listening": listening}
|
|
await asyncio.to_thread(node.execute, inputs | {"voice.url": url})
|
|
|
|
await hear(chunks[0], True)
|
|
await hear(chunks[1], True)
|
|
await hear(chunks[2], True)
|
|
await hear(chunks[2], False)
|
|
for _ in range(200):
|
|
if "voice.text" in pipeline.state:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
|
|
assert heard == [bytes([1]) * 3200, bytes([2]) * 3200]
|
|
assert pipeline.state["voice.text"] == "Turn on the light."
|
|
assert pipeline.state["voice.partial"] == "Turn on the light."
|
|
assert node._session is None
|
|
|
|
# No server, no session: the node says so instead.
|
|
await hear(chunks[1], True, url="")
|
|
assert node._session is None
|
|
assert node._health == "degraded"
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def _wav(seconds: float) -> bytes:
|
|
out = io.BytesIO()
|
|
with wave.open(out, "wb") as clip:
|
|
clip.setnchannels(1)
|
|
clip.setsampwidth(2)
|
|
clip.setframerate(24000)
|
|
clip.writeframes(b"\0\0" * int(24000 * seconds))
|
|
return out.getvalue()
|
|
|
|
|
|
def test_a_sentence_is_spoken_once_and_the_rest_of_a_reply_queues(monkeypatch):
|
|
spoken: list[str] = []
|
|
published: list[dict] = []
|
|
|
|
async def speak(api_base, text, voice):
|
|
spoken.append(text)
|
|
return _wav(len(spoken))
|
|
|
|
monkeypatch.setattr(client, "speak", speak)
|
|
monkeypatch.setattr(TtsNode, "_say", lambda self, speech: published.append(speech))
|
|
node = TtsNode(
|
|
requires=[
|
|
MessageSpec(name="reply", port="text", dtype=DType.STR),
|
|
MessageSpec(name="url", dtype=DType.STR),
|
|
],
|
|
provides=[MessageSpec(name="speech", dtype=DType.AUDIO)],
|
|
)
|
|
node.assign_flow("voice", "tts")
|
|
Pipeline([node])
|
|
node._artifacts = Store()
|
|
|
|
node.execute({"voice.reply": "Kitchen light on. Heating off.", "voice.url": URL})
|
|
# Long after both stopped playing.
|
|
node._until = -10.0
|
|
node.execute({"voice.reply": "Kitchen light on.", "voice.url": URL})
|
|
|
|
assert spoken == ["Kitchen light on.", "Heating off."]
|
|
assert ["append" in speech for speech in published] == [False, True, False]
|
|
assert published[2] == published[0]
|
|
assert published[1]["meta"] == {"seconds": 2.0}
|