Voice nodes (openpanel V4): stt, tts, wren reply/unhandled, socket publish

- `stt`: one realtime transcription session per utterance on the speech
  server's url, opened at `listening` true or the first chunk, committed at
  false; a whole clip goes to /v1/audio/transcriptions. Returns at once and
  publishes `partial` and `text` when the server answers.
- `tts`: one /v1/audio/speech call per sentence, cached by voice and text in
  node state; a reference is marked `append` while earlier ones still play.
- wren: `reply` says what was applied, `unhandled` carries the prompt when
  nothing was. The install hint names the git pin (PyPI's `wren` is another
  package).
- ai/client: transcribe, speak and transcribe_stream beside stream_completion.
- A device socket publishes scalar `in` ports as {"type": "publish"} text,
  in order with its binary frames.
- Pipeline.publish walks only what its values reach: a node downstream of a
  consumer that published nothing no longer runs again (wren re-acted on
  every microphone chunk).

backend/tests/flow/test_node_types.py

@@ -105,6 +105,22 @@ FIXTURES: dict[str, dict] = {
         "requires": [MessageSpec(name="prompt", dtype=DType.STR)],
         "provides": [MessageSpec(name="kitchen_light", dtype=DType.BOOL)],
     },
+    "stt": {
+        "params": {},
+        "requires": [
+            MessageSpec(name="mic", port="audio", dtype=DType.AUDIO),
+            MessageSpec(name="url", dtype=DType.STR),
+        ],
+        "provides": [MessageSpec(name="text", dtype=DType.STR)],
+    },
+    "tts": {
+        "params": {"voice": "bm_george"},
+        "requires": [
+            MessageSpec(name="reply", port="text", dtype=DType.STR),
+            MessageSpec(name="url", dtype=DType.STR),
+        ],
+        "provides": [MessageSpec(name="speech", dtype=DType.AUDIO)],
+    },
     "device": {
         "params": {"device": "hall", "config": {"gpio.4": {"pull": "up"}}},
         "requires": [

backend/tests/flow/test_speech.py

@@ -0,0 +1,147 @@
+"""The speech nodes: one session per utterance, and each sentence spoken once."""
+
+import asyncio
+import hashlib
+import io
+import wave
+from typing import Any
+
+from fluksio.ai import client
+from fluksio.flow.messages import DType, MessageSpec
+from fluksio.flow.nodes.speech import SttNode, TtsNode, sentences
+from fluksio.flow.pipeline import Pipeline
+
+URL = "http://speech:8100"
+
+
+class Store:
+    """The artifact store's surface the nodes use."""
+
+    def __init__(self) -> None:
+        self.data: dict[str, bytes] = {}
+
+    def put(self, chunks: list[bytes], name: str, media_type: str) -> dict[str, Any]:
+        data = b"".join(chunks)
+        digest = "sha256:" + hashlib.sha256(data).hexdigest()
+        self.data[digest] = data
+        return {"digest": digest, "size": len(data), "media_type": media_type}
+
+    def has(self, digest: str) -> bool:
+        return digest in self.data
+
+    def read(self, digest: str) -> list[bytes]:
+        return [self.data[digest]]
+
+
+def test_sentences_are_cut_after_their_ending():
+    assert sentences(" Kitchen light on.  Heating off! Is it?\nYes") == [
+        "Kitchen light on.",
+        "Heating off!",
+        "Is it?",
+        "Yes",
+    ]
+
+
+def test_an_utterance_is_one_session_committed_when_listening_ends(monkeypatch):
+    heard: list[bytes] = []
+
+    async def session(api_base, audio, *, rate, idle_s):
+        assert (api_base, rate) == (f"{URL}/v1", 16000)
+        while (chunk := await audio.get()) is not None:
+            heard.append(chunk)
+        yield False, "turn on"
+        yield False, " the"
+        yield True, " Turn on the light."
+
+    monkeypatch.setattr(client, "transcribe_stream", session)
+
+    async def scenario() -> None:
+        node = SttNode(
+            requires=[
+                MessageSpec(name="mic", port="audio", dtype=DType.AUDIO),
+                MessageSpec(name="listening", dtype=DType.BOOL),
+                MessageSpec(name="url", dtype=DType.STR),
+            ],
+            provides=[
+                MessageSpec(name="partial", dtype=DType.STR),
+                MessageSpec(name="text", dtype=DType.STR),
+            ],
+        )
+        node.assign_flow("voice", "stt")
+        pipeline = Pipeline([node])
+        store = Store()
+        chunks = [
+            store.put([bytes([n]) * 3200], "voice.mic", "audio/L16;rate=16000")
+            for n in range(3)
+        ]
+        # Left over from before the engine started: not part of what is said.
+        pipeline.state["voice.mic"] = chunks[0]
+        await node.start()
+        node._artifacts = store
+
+        async def hear(audio: dict, listening: bool, url: str = URL) -> None:
+            inputs = {"voice.mic": audio, "voice.listening": listening}
+            await asyncio.to_thread(node.execute, inputs | {"voice.url": url})
+
+        await hear(chunks[0], True)
+        await hear(chunks[1], True)
+        await hear(chunks[2], True)
+        await hear(chunks[2], False)
+        for _ in range(200):
+            if "voice.text" in pipeline.state:
+                break
+            await asyncio.sleep(0.01)
+
+        assert heard == [bytes([1]) * 3200, bytes([2]) * 3200]
+        assert pipeline.state["voice.text"] == "Turn on the light."
+        assert pipeline.state["voice.partial"] == "Turn on the light."
+        assert node._session is None
+
+        # No server, no session: the node says so instead.
+        await hear(chunks[1], True, url="")
+        assert node._session is None
+        assert node._health == "degraded"
+
+    asyncio.run(scenario())
+
+
+def _wav(seconds: float) -> bytes:
+    out = io.BytesIO()
+    with wave.open(out, "wb") as clip:
+        clip.setnchannels(1)
+        clip.setsampwidth(2)
+        clip.setframerate(24000)
+        clip.writeframes(b"\0\0" * int(24000 * seconds))
+    return out.getvalue()
+
+
+def test_a_sentence_is_spoken_once_and_the_rest_of_a_reply_queues(monkeypatch):
+    spoken: list[str] = []
+    published: list[dict] = []
+
+    async def speak(api_base, text, voice):
+        spoken.append(text)
+        return _wav(len(spoken))
+
+    monkeypatch.setattr(client, "speak", speak)
+    monkeypatch.setattr(TtsNode, "_say", lambda self, speech: published.append(speech))
+    node = TtsNode(
+        requires=[
+            MessageSpec(name="reply", port="text", dtype=DType.STR),
+            MessageSpec(name="url", dtype=DType.STR),
+        ],
+        provides=[MessageSpec(name="speech", dtype=DType.AUDIO)],
+    )
+    node.assign_flow("voice", "tts")
+    Pipeline([node])
+    node._artifacts = Store()
+
+    node.execute({"voice.reply": "Kitchen light on. Heating off.", "voice.url": URL})
+    # Long after both stopped playing.
+    node._until = -10.0
+    node.execute({"voice.reply": "Kitchen light on.", "voice.url": URL})
+
+    assert spoken == ["Kitchen light on.", "Heating off."]
+    assert ["append" in speech for speech in published] == [False, True, False]
+    assert published[2] == published[0]
+    assert published[1]["meta"] == {"seconds": 2.0}

backend/tests/flow/test_wren.py

@@ -165,3 +165,27 @@ def test_the_model_runs_on_one_blas_thread(fake_wren, monkeypatch):
 
     assert threads
     assert set(threads) == {1}
+
+
+def test_a_reply_says_what_was_applied_and_the_rest_is_unhandled(fake_wren):
+    node = WrenNode(
+        requires=[MessageSpec(name="command", port="prompt", dtype=DType.STR)],
+        provides=[
+            MessageSpec(name="kitchen_light", dtype=DType.BOOL),
+            MessageSpec(name="reply", dtype=DType.STR),
+            MessageSpec(name="unhandled", dtype=DType.STR),
+        ],
+        params={"apply": True},
+    )
+    node.assign_flow("house", "control")
+    fake_wren.answer = ([FakeCall("lights.on", {"id": "kitchen:light"})], 0.99)
+
+    out = node.execute({"house.command": "turn on the kitchen light"})
+
+    assert out == {"house.kitchen_light": True, "house.reply": "Kitchen light on."}
+
+    # Nothing wired for the call, or nothing to call at all.
+    for calls in ([FakeCall("lights.on", {"id": "garage:light"})], []):
+        fake_wren.answer = (calls, 0.99)
+        out = node.execute({"house.command": "what is the weather tomorrow"})
+        assert out == {"house.unhandled": "what is the weather tomorrow"}