diff --git a/backend/fluksio/ai/client.py b/backend/fluksio/ai/client.py index 544d328..da9b553 100644 --- a/backend/fluksio/ai/client.py +++ b/backend/fluksio/ai/client.py @@ -194,12 +194,19 @@ async def transcribe( *, model: str = "whisper-1", timeout: float = 30.0, + prompt: str = "", ) -> str: - """``POST {api_base}/audio/transcriptions``: a whole clip, and what it says.""" + """``POST {api_base}/audio/transcriptions``: a whole clip, and what it says. + + ``prompt`` is words the clip is likely to hold, such as room names. + """ + data = {"model": model, "response_format": "json"} + if prompt: + data["prompt"] = prompt response = await _client(api_base).post( api_base.rstrip("/") + "/audio/transcriptions", files={"file": (filename, audio, media_type)}, - data={"model": model, "response_format": "json"}, + data=data, timeout=timeout, ) if response.is_error: @@ -236,25 +243,26 @@ async def transcribe_stream( *, rate: int = 16000, idle_s: float = 10.0, + prompt: str = "", ) -> AsyncIterator[tuple[bool, str]]: """``WS {api_base}/realtime``: one utterance, transcribed while it is spoken. Sends each chunk of PCM16 taken off ``audio`` and commits at ``None``. Yields ``(False, delta)`` as words are agreed on, then ``(True, transcript)`` once. A chunk that takes longer than ``idle_s`` to come - raises ``TimeoutError``, so an utterance nobody commits ends. + raises ``TimeoutError``, so an utterance nobody commits ends. ``prompt`` + is words the speech is likely to hold, as for ``transcribe``. """ url = "ws" + api_base.rstrip("/").removeprefix("http") + "/realtime" async with connect(url, open_timeout=idle_s) as ws: - update = {"type": "audio/pcm", "rate": rate} + given: dict[str, Any] = {"format": {"type": "audio/pcm", "rate": rate}} + if prompt: + given["transcription"] = {"prompt": prompt} await ws.send( json.dumps( { "type": "session.update", - "session": { - "type": "transcription", - "audio": {"input": {"format": update}}, - }, + "session": {"type": "transcription", "audio": {"input": given}}, } ) ) diff --git a/backend/fluksio/flow/nodes/assistant.py b/backend/fluksio/flow/nodes/assistant.py index b89a6d7..c6af800 100644 --- a/backend/fluksio/flow/nodes/assistant.py +++ b/backend/fluksio/flow/nodes/assistant.py @@ -9,7 +9,7 @@ offered. Every other input is read as context ("outside_temperature: 7.5"). An answer goes out on ``reply`` a sentence at a time while the model writes it, so a ``tts`` node speaks the first before the last is written. A change is confirmed from the values set ("Kitchen light off."), without asking the model -a second time. +a second time. Talk the panel overheard, such as a TV, gets no answer. """ from __future__ import annotations @@ -47,6 +47,10 @@ LISTENING = "listening" UNAVAILABLE = "The assistant isn't available right now." #: Said when the model ends a turn without a word or a change. CANT = "Sorry, I can't do that." +#: The model's whole answer to talk not meant for the panel, which is not said. +#: Asked to answer with nothing instead, Gemma writes tokens nobody sees until +#: the length limit. +IGNORED = "Ignored." #: A server that is not there is given up on after a few seconds, while a #: model that has to be loaded first takes about ten. @@ -60,14 +64,18 @@ _SCHEMAS: dict[DType, dict[str, str]] = { DType.STR: {"type": "string"}, } -_SYSTEM = """You are the voice of a home: someone spoke to a panel on the wall, \ +_SYSTEM = f"""You are the voice of a home: someone spoke to a panel on the wall, \ and you answer out loud. -- To change something, call its tool without a word: the change is confirmed \ -for you. The tools are all you can change. -- If no tool can do what was asked, or you did not understand it, say so in a \ -few words, such as "Sorry, I can't do that." -- Answer questions from the readings below. +- To change something in the home, call its tool without a word: the change \ +is confirmed for you. The tools are all you can change. +- If a change was asked for that no tool can make, say so in a few words, such \ +as "Sorry, I can't do that." +- Answer questions about the home from the readings below, and other questions \ +from what you know. +- The panel also hears talk that is not meant for it, such as a TV or people \ +talking to each other. If the words are neither a request nor a question for \ +you, answer only "{IGNORED}" - Speak plainly: one or two short sentences, no lists, no markdown, no emoji. """ @@ -308,7 +316,8 @@ class AssistantNode(Node): async def say(text: str) -> None: nonlocal said said = True - await asyncio.to_thread(self._say, text) + if text.rstrip(".") != IGNORED.rstrip("."): + await asyncio.to_thread(self._say, text) if config is None: await say(UNAVAILABLE) diff --git a/backend/fluksio/flow/nodes/speech.py b/backend/fluksio/flow/nodes/speech.py index b8c1f8f..02ed592 100644 --- a/backend/fluksio/flow/nodes/speech.py +++ b/backend/fluksio/flow/nodes/speech.py @@ -186,7 +186,18 @@ class SttNode(_Speech): alone. """ - __slots__ = ("_heard", "_listening", "_lock", "_session", "_tasks") + class Params(BaseModel): + model_config = ConfigDict(extra="allow") + + prompt: str = Field( + default="", + description=( + "Words the speech is likely to hold, such as the names of rooms " + "and devices, given to the server as its transcription prompt." + ), + ) + + __slots__ = ("cfg", "_heard", "_listening", "_lock", "_session", "_tasks") def __init__( self, @@ -195,6 +206,7 @@ class SttNode(_Speech): params: dict[str, Any] | None = None, name: str | None = None, ): + self.cfg = self.Params.model_validate(params or {}) self._lock = threading.Lock() #: The chunks of the utterance being spoken, None between utterances. self._session: asyncio.Queue[bytes | None] | None = None @@ -293,7 +305,11 @@ class SttNode(_Speech): try: async with asyncio.timeout(UTTERANCE_S): async for final, text in client.transcribe_stream( - url + "/v1", chunks, rate=RATE, idle_s=IDLE_S + url + "/v1", + chunks, + rate=RATE, + idle_s=IDLE_S, + prompt=self.cfg.prompt, ): words = text.strip() if final else words + text if final and words: @@ -317,7 +333,11 @@ class SttNode(_Speech): async def _clip(self, url: str, audio: dict[str, Any], data: bytes) -> None: try: text = await client.transcribe( - url + "/v1", data, "speech.wav", str(audio.get("media_type")) + url + "/v1", + data, + "speech.wav", + str(audio.get("media_type")), + prompt=self.cfg.prompt, ) except Exception as exc: logger.warning("'%s' lost a clip: %s: %s", self.id, type(exc).__name__, exc) diff --git a/backend/tests/flow/test_assistant.py b/backend/tests/flow/test_assistant.py index 8240d26..7e17479 100644 --- a/backend/tests/flow/test_assistant.py +++ b/backend/tests/flow/test_assistant.py @@ -11,7 +11,7 @@ import pytest from fluksio.ai import client from fluksio.ai import config as ai_config from fluksio.flow.messages import DType, MessageSpec -from fluksio.flow.nodes.assistant import UNAVAILABLE, AssistantNode, cut +from fluksio.flow.nodes.assistant import IGNORED, UNAVAILABLE, AssistantNode, cut from fluksio.flow.pipeline import Pipeline PROVIDER = ai_config.AIConfig(api_base="http://llm:11434/v1", api_key="k", model="m") @@ -140,6 +140,22 @@ def test_no_provider_or_one_that_does_not_answer_is_said( assert node._health == "degraded" +def test_overheard_talk_gets_no_answer(monkeypatch: pytest.MonkeyPatch): + said: list[dict[str, Any]] = [] + + async def completion(**kwargs: Any) -> AsyncIterator[str | dict[str, Any]]: + yield IGNORED + yield {"role": "assistant", "content": IGNORED} + + monkeypatch.setattr(ai_config, "load", lambda: PROVIDER) + monkeypatch.setattr(client, "stream_completion", completion) + monkeypatch.setattr(AssistantNode, "emit", lambda self, values: said.append(values)) + + _ask(_node(), "I don't know what to get, Francis.") + + assert said == [] + + def test_a_target_with_values_takes_only_those_and_is_confirmed_in_words( monkeypatch: pytest.MonkeyPatch, ): diff --git a/backend/tests/flow/test_speech.py b/backend/tests/flow/test_speech.py index 3c9c25c..c327191 100644 --- a/backend/tests/flow/test_speech.py +++ b/backend/tests/flow/test_speech.py @@ -52,8 +52,8 @@ def test_units_and_clock_times_are_said_as_words(): def test_an_utterance_is_one_session_committed_when_listening_ends(monkeypatch): heard: list[bytes] = [] - async def session(api_base, audio, *, rate, idle_s): - assert (api_base, rate) == (f"{URL}/v1", 16000) + async def session(api_base, audio, *, rate, idle_s, prompt): + assert (api_base, rate, prompt) == (f"{URL}/v1", 16000, "Driveway.") while (chunk := await audio.get()) is not None: heard.append(chunk) yield False, "turn on" @@ -79,6 +79,7 @@ def test_an_utterance_is_one_session_committed_when_listening_ends(monkeypatch): MessageSpec(name="partial", dtype=DType.STR), MessageSpec(name="text", dtype=DType.STR), ], + params={"prompt": "Driveway."}, ) node.assign_flow("voice", "stt") pipeline = Pipeline([node]) diff --git a/docs/reference/node-types.md b/docs/reference/node-types.md index 5716f29..1580509 100644 --- a/docs/reference/node-types.md +++ b/docs/reference/node-types.md @@ -509,10 +509,16 @@ living room setpoint to 22."), so the model is asked once per request. A target with `values` is offered only those, each explained to the model, and confirmed in its word ("Canopy open."). Any other value is refused. -Every request ends in a reply: +Questions about the home are answered from the context inputs, other +questions from what the model knows. -- When no tool fits or the request makes no sense, the model says so, as +Every request ends in a reply, except talk the panel overheard: + +- When no tool can make the change asked for, the model says so, as "Sorry, I can't do that.". +- Words that are neither a request nor a question, such as a TV or people + talking to each other, get no reply. The model answers them "Ignored.", + which is not said. - When no provider is set up, the connection is refused, it takes over 3 seconds to connect or 15 seconds to answer, the reply is "The assistant isn't available right now." and the node reads degraded. A model that has to @@ -543,6 +549,10 @@ the assistant's prompt, both `reply` outputs go to `tts`, and the device's | `partial` | `str` | the words agreed on so far, as they come, then the final text | | `text` | `str` | what was said, once per utterance | +| Setting | Default | Notes | +|---|---|---| +| `prompt` | empty | words the speech is likely to hold, such as `Living room, kitchen, driveway, appliances.`, sent as the transcription prompt | + The server is any that speaks OpenAI's audio API, such as `parrot serve`. Each utterance is one realtime transcription session (`{url}/v1/realtime`), opened when `listening` turns true or at the first chunk and committed when it turns @@ -552,6 +562,11 @@ chunk waits for it. An utterance that comes back empty, fails, or gets no chunk for 10 seconds publishes "Didn't catch that." on `partial` alone, so a caption shows it and nothing speaks it. A lost one is logged. +A `prompt` naming the installation's rooms and devices helps Whisper with +words it rarely hears. On the recordings of one wall panel, small.en with such +a prompt heard 30 of 42 hard commands right instead of 8, as "driveway" for +"drive bay" and "bathroom" for "best room". + `listening` has to hold a value before a chunk wakes the node. Declare it as a flow input with the initial value `false` when a device may never send it, such as one that sends whole clips.