Merge remote-tracking branch 'origin/main'

backend/fluksio/ai/client.py

@@ -194,12 +194,19 @@ async def transcribe(
     *,
     model: str = "whisper-1",
     timeout: float = 30.0,
+    prompt: str = "",
 ) -> str:
-    """``POST {api_base}/audio/transcriptions``: a whole clip, and what it says."""
+    """``POST {api_base}/audio/transcriptions``: a whole clip, and what it says.
+
+    ``prompt`` is words the clip is likely to hold, such as room names.
+    """
+    data = {"model": model, "response_format": "json"}
+    if prompt:
+        data["prompt"] = prompt
     response = await _client(api_base).post(
         api_base.rstrip("/") + "/audio/transcriptions",
         files={"file": (filename, audio, media_type)},
-        data={"model": model, "response_format": "json"},
+        data=data,
         timeout=timeout,
     )
     if response.is_error:
@@ -236,25 +243,26 @@ async def transcribe_stream(
     *,
     rate: int = 16000,
     idle_s: float = 10.0,
+    prompt: str = "",
 ) -> AsyncIterator[tuple[bool, str]]:
     """``WS {api_base}/realtime``: one utterance, transcribed while it is spoken.
 
     Sends each chunk of PCM16 taken off ``audio`` and commits at ``None``.
     Yields ``(False, delta)`` as words are agreed on, then ``(True,
     transcript)`` once. A chunk that takes longer than ``idle_s`` to come
-    raises ``TimeoutError``, so an utterance nobody commits ends.
+    raises ``TimeoutError``, so an utterance nobody commits ends. ``prompt``
+    is words the speech is likely to hold, as for ``transcribe``.
     """
     url = "ws" + api_base.rstrip("/").removeprefix("http") + "/realtime"
     async with connect(url, open_timeout=idle_s) as ws:
-        update = {"type": "audio/pcm", "rate": rate}
+        given: dict[str, Any] = {"format": {"type": "audio/pcm", "rate": rate}}
+        if prompt:
+            given["transcription"] = {"prompt": prompt}
         await ws.send(
             json.dumps(
                 {
                     "type": "session.update",
-                    "session": {
+                    "session": {"type": "transcription", "audio": {"input": given}},
-                        "type": "transcription",
-                        "audio": {"input": {"format": update}},
-                    },
                 }
             )
         )

backend/fluksio/flow/nodes/assistant.py

@@ -9,7 +9,7 @@ offered. Every other input is read as context ("outside_temperature: 7.5").
 An answer goes out on ``reply`` a sentence at a time while the model writes
 it, so a ``tts`` node speaks the first before the last is written. A change is
 confirmed from the values set ("Kitchen light off."), without asking the model
-a second time.
+a second time. Talk the panel overheard, such as a TV, gets no answer.
 """
 
 from __future__ import annotations
@@ -47,6 +47,10 @@ LISTENING = "listening"
 UNAVAILABLE = "The assistant isn't available right now."
 #: Said when the model ends a turn without a word or a change.
 CANT = "Sorry, I can't do that."
+#: The model's whole answer to talk not meant for the panel, which is not said.
+#: Asked to answer with nothing instead, Gemma writes tokens nobody sees until
+#: the length limit.
+IGNORED = "Ignored."
 
 #: A server that is not there is given up on after a few seconds, while a
 #: model that has to be loaded first takes about ten.
@@ -60,14 +64,18 @@ _SCHEMAS: dict[DType, dict[str, str]] = {
     DType.STR: {"type": "string"},
 }
 
-_SYSTEM = """You are the voice of a home: someone spoke to a panel on the wall, \
+_SYSTEM = f"""You are the voice of a home: someone spoke to a panel on the wall, \
 and you answer out loud.
 
-- To change something, call its tool without a word: the change is confirmed \
+- To change something in the home, call its tool without a word: the change \
-for you. The tools are all you can change.
+is confirmed for you. The tools are all you can change.
-- If no tool can do what was asked, or you did not understand it, say so in a \
+- If a change was asked for that no tool can make, say so in a few words, such \
-few words, such as "Sorry, I can't do that."
+as "Sorry, I can't do that."
-- Answer questions from the readings below.
+- Answer questions about the home from the readings below, and other questions \
+from what you know.
+- The panel also hears talk that is not meant for it, such as a TV or people \
+talking to each other. If the words are neither a request nor a question for \
+you, answer only "{IGNORED}"
 - Speak plainly: one or two short sentences, no lists, no markdown, no emoji.
 """
 
@@ -308,7 +316,8 @@ class AssistantNode(Node):
         async def say(text: str) -> None:
             nonlocal said
             said = True
-            await asyncio.to_thread(self._say, text)
+            if text.rstrip(".") != IGNORED.rstrip("."):
+                await asyncio.to_thread(self._say, text)
 
         if config is None:
             await say(UNAVAILABLE)

backend/fluksio/flow/nodes/speech.py

@@ -186,7 +186,18 @@ class SttNode(_Speech):
     alone.
     """
 
-    __slots__ = ("_heard", "_listening", "_lock", "_session", "_tasks")
+    class Params(BaseModel):
+        model_config = ConfigDict(extra="allow")
+
+        prompt: str = Field(
+            default="",
+            description=(
+                "Words the speech is likely to hold, such as the names of rooms "
+                "and devices, given to the server as its transcription prompt."
+            ),
+        )
+
+    __slots__ = ("cfg", "_heard", "_listening", "_lock", "_session", "_tasks")
 
     def __init__(
         self,
@@ -195,6 +206,7 @@ class SttNode(_Speech):
         params: dict[str, Any] | None = None,
         name: str | None = None,
     ):
+        self.cfg = self.Params.model_validate(params or {})
         self._lock = threading.Lock()
         #: The chunks of the utterance being spoken, None between utterances.
         self._session: asyncio.Queue[bytes | None] | None = None
@@ -293,7 +305,11 @@ class SttNode(_Speech):
         try:
             async with asyncio.timeout(UTTERANCE_S):
                 async for final, text in client.transcribe_stream(
-                    url + "/v1", chunks, rate=RATE, idle_s=IDLE_S
+                    url + "/v1",
+                    chunks,
+                    rate=RATE,
+                    idle_s=IDLE_S,
+                    prompt=self.cfg.prompt,
                 ):
                     words = text.strip() if final else words + text
                     if final and words:
@@ -317,7 +333,11 @@ class SttNode(_Speech):
     async def _clip(self, url: str, audio: dict[str, Any], data: bytes) -> None:
         try:
             text = await client.transcribe(
-                url + "/v1", data, "speech.wav", str(audio.get("media_type"))
+                url + "/v1",
+                data,
+                "speech.wav",
+                str(audio.get("media_type")),
+                prompt=self.cfg.prompt,
             )
         except Exception as exc:
             logger.warning("'%s' lost a clip: %s: %s", self.id, type(exc).__name__, exc)

backend/tests/flow/test_assistant.py

@@ -11,7 +11,7 @@ import pytest
 from fluksio.ai import client
 from fluksio.ai import config as ai_config
 from fluksio.flow.messages import DType, MessageSpec
-from fluksio.flow.nodes.assistant import UNAVAILABLE, AssistantNode, cut
+from fluksio.flow.nodes.assistant import IGNORED, UNAVAILABLE, AssistantNode, cut
 from fluksio.flow.pipeline import Pipeline
 
 PROVIDER = ai_config.AIConfig(api_base="http://llm:11434/v1", api_key="k", model="m")
@@ -140,6 +140,22 @@ def test_no_provider_or_one_that_does_not_answer_is_said(
         assert node._health == "degraded"
 
 
+def test_overheard_talk_gets_no_answer(monkeypatch: pytest.MonkeyPatch):
+    said: list[dict[str, Any]] = []
+
+    async def completion(**kwargs: Any) -> AsyncIterator[str | dict[str, Any]]:
+        yield IGNORED
+        yield {"role": "assistant", "content": IGNORED}
+
+    monkeypatch.setattr(ai_config, "load", lambda: PROVIDER)
+    monkeypatch.setattr(client, "stream_completion", completion)
+    monkeypatch.setattr(AssistantNode, "emit", lambda self, values: said.append(values))
+
+    _ask(_node(), "I don't know what to get, Francis.")
+
+    assert said == []
+
+
 def test_a_target_with_values_takes_only_those_and_is_confirmed_in_words(
     monkeypatch: pytest.MonkeyPatch,
 ):

backend/tests/flow/test_speech.py

@@ -52,8 +52,8 @@ def test_units_and_clock_times_are_said_as_words():
 def test_an_utterance_is_one_session_committed_when_listening_ends(monkeypatch):
     heard: list[bytes] = []
 
-    async def session(api_base, audio, *, rate, idle_s):
+    async def session(api_base, audio, *, rate, idle_s, prompt):
-        assert (api_base, rate) == (f"{URL}/v1", 16000)
+        assert (api_base, rate, prompt) == (f"{URL}/v1", 16000, "Driveway.")
         while (chunk := await audio.get()) is not None:
             heard.append(chunk)
         yield False, "turn on"
@@ -79,6 +79,7 @@ def test_an_utterance_is_one_session_committed_when_listening_ends(monkeypatch):
                 MessageSpec(name="partial", dtype=DType.STR),
                 MessageSpec(name="text", dtype=DType.STR),
             ],
+            params={"prompt": "Driveway."},
         )
         node.assign_flow("voice", "stt")
         pipeline = Pipeline([node])

docs/reference/node-types.md

@@ -509,10 +509,16 @@ living room setpoint to 22."), so the model is asked once per request. A
 target with `values` is offered only those, each explained to the model, and
 confirmed in its word ("Canopy open."). Any other value is refused.
 
-Every request ends in a reply:
+Questions about the home are answered from the context inputs, other
+questions from what the model knows.
 
-- When no tool fits or the request makes no sense, the model says so, as
+Every request ends in a reply, except talk the panel overheard:
+
+- When no tool can make the change asked for, the model says so, as
   "Sorry, I can't do that.".
+- Words that are neither a request nor a question, such as a TV or people
+  talking to each other, get no reply. The model answers them "Ignored.",
+  which is not said.
 - When no provider is set up, the connection is refused, it takes over 3
   seconds to connect or 15 seconds to answer, the reply is "The assistant
   isn't available right now." and the node reads degraded. A model that has to
@@ -543,6 +549,10 @@ the assistant's prompt, both `reply` outputs go to `tts`, and the device's
 | `partial` | `str` | the words agreed on so far, as they come, then the final text |
 | `text` | `str` | what was said, once per utterance |
 
+| Setting | Default | Notes |
+|---|---|---|
+| `prompt` | empty | words the speech is likely to hold, such as `Living room, kitchen, driveway, appliances.`, sent as the transcription prompt |
+
 The server is any that speaks OpenAI's audio API, such as `parrot serve`. Each
 utterance is one realtime transcription session (`{url}/v1/realtime`), opened
 when `listening` turns true or at the first chunk and committed when it turns
@@ -552,6 +562,11 @@ chunk waits for it. An utterance that comes back empty, fails, or gets no
 chunk for 10 seconds publishes "Didn't catch that." on `partial` alone, so a
 caption shows it and nothing speaks it. A lost one is logged.
 
+A `prompt` naming the installation's rooms and devices helps Whisper with
+words it rarely hears. On the recordings of one wall panel, small.en with such
+a prompt heard 30 of 42 hard commands right instead of 8, as "driveway" for
+"drive bay" and "bathroom" for "best room".
+
 `listening` has to hold a value before a chunk wakes the node. Declare it as a
 flow input with the initial value `false` when a device may never send it,
 such as one that sends whole clips.