diff --git a/backend/fluksio/flow/nodes/speech.py b/backend/fluksio/flow/nodes/speech.py index f061552..b8c1f8f 100644 --- a/backend/fluksio/flow/nodes/speech.py +++ b/backend/fluksio/flow/nodes/speech.py @@ -44,6 +44,15 @@ PLAYING_SLACK_S = 1.0 #: The partial for an utterance that came back empty, often a wake word that #: was not one, or was lost: shown on the caption, never spoken. NOTHING_HEARD = "Didn't catch that." +#: Units read out after a number, the longer first. Kokoro says "22 °C" as +#: "22 degrees C" and "3.4 kWh" letter by letter. +UNITS = { + "°C": "degrees", + "°": "degrees", + "kWh": "kilowatt hours", + "kW": "kilowatts", + "W": "watts", +} def _check(specs: list[MessageSpec], ports: dict[str, DType], node: str) -> None: @@ -70,6 +79,23 @@ def sentences(text: str) -> list[str]: return [s for s in (p.strip() for p in re.split(r"(?<=[.!?])\s+", text)) if s] +def speakable(text: str) -> str: + """``text`` as it is said: a unit after a number in words, and a clock + time as its hour and minutes, since "07:30" is read "zero seven thirty".""" + units = "|".join(map(re.escape, UNITS)) + text = re.sub( + rf"(\d)\s*({units})(?![A-Za-z])", lambda m: f"{m[1]} {UNITS[m[2]]}", text + ) + return re.sub(r"\b([01]?\d|2[0-3]):([0-5]\d)\b", _clock, text) + + +def _clock(time: re.Match[str]) -> str: + hour, minute = int(time[1]), int(time[2]) + if minute == 0: + return f"{hour} o'clock" + return f"{hour} oh {minute}" if minute < 10 else f"{hour} {minute}" + + def mp3_seconds(data: bytes) -> float: """An MP3's length, from the frame count LAME writes into its first frame.""" header = int.from_bytes(data[:4], "big") @@ -358,7 +384,7 @@ class TtsNode(_Speech): if not text.strip() or not self._reachable(url): return None with self._lock: - for sentence in sentences(text): + for sentence in sentences(speakable(text)): reference = self._spoken(url, sentence) now = time.monotonic() if now < self._until + PLAYING_SLACK_S: diff --git a/backend/fluksio/flow/nodes/wren.py b/backend/fluksio/flow/nodes/wren.py index d448037..af6354e 100644 --- a/backend/fluksio/flow/nodes/wren.py +++ b/backend/fluksio/flow/nodes/wren.py @@ -12,7 +12,7 @@ from __future__ import annotations import re import time from collections.abc import Iterable -from typing import Any +from typing import Any, NamedTuple from pydantic import BaseModel, ConfigDict, Field from threadpoolctl import threadpool_limits @@ -60,6 +60,7 @@ KIND_WORDS = { } #: What the node reads, and the types each may be. Only the prompt wakes it. +#: Besides these, ``_temperature`` inputs answer "how warm is it". INPUTS = { "prompt": {DType.STR}, "presence": {DType.STR}, @@ -125,6 +126,84 @@ _SAID = { #: shown, wired or not, or "the bathroom light" lands on a light that is. _TRAINED = {"living room": ("light", "media", "heat")} +#: wren's areas outside, whose temperature is the ``outdoor`` reading. +_OUTSIDE = {"garden", "patio", "garage", "driveway", "balcony"} + +#: Number words, which wren was trained on as digits. +_ONES = ( + "zero one two three four five six seven eight nine ten eleven twelve " + "thirteen fourteen fifteen sixteen seventeen eighteen nineteen" +).split() +_TENS = "twenty thirty forty fifty sixty seventy eighty ninety".split() +_NUMBERS = {w: n for n, w in enumerate(_ONES)} | { + w: 20 + 10 * n for n, w in enumerate(_TENS) +} + +#: A command for a whole group of devices: "turn off all lights". +_GROUP = re.compile(r"\b(all|every|everything|everywhere)\b") +#: What leaves such a command to the model and the assistant: a question, or +#: an exception to the group. +_NOT_GROUP = re.compile( + r"^(is|are|was|were|do|does|did|can|could|what|which|how|why|when|where|who)\b" + r"|\b(except|but|apart from|other than)\b" +) +#: The kind a group command names, by its word. None named is every kind. +_GROUP_KINDS = { + "light": "light", + "lights": "light", + "lamps": "light", + "music": "media", + "media": "media", + "tv": "media", + "shutters": "shutter", + "blinds": "shutter", + "heating": "heat", + "watering": "water", + "sprinklers": "water", +} +#: The call a group command makes on each kind, by its action word. +_GROUP_TOOLS = { + "on": {"light": "lights.on", "media": "media.play", "water": "watering.start"}, + "off": { + "light": "lights.off", + "media": "media.stop", + "heat": "heating.off", + "water": "watering.stop", + }, + "open": {"shutter": "shutters.up"}, + "up": {"shutter": "shutters.up"}, + "raise": {"shutter": "shutters.up"}, + "close": {"shutter": "shutters.down"}, + "down": {"shutter": "shutters.down"}, + "lower": {"shutter": "shutters.down"}, + "stop": { + "media": "media.stop", + "shutter": "shutters.stop", + "water": "watering.stop", + }, +} +#: A group call, said: "all lights off". +_GROUP_SAID = { + "lights.on": "lights on", + "lights.off": "lights off", + "media.play": "media playing", + "media.stop": "media stopped", + "heating.off": "heating off", + "shutters.up": "shutters up", + "shutters.down": "shutters down", + "shutters.stop": "shutters stopped", + "watering.start": "watering on", + "watering.stop": "watering off", +} + + +class _Call(NamedTuple): + """A call the node makes itself, shaped as the model's.""" + + tool: str + args: dict[str, Any] + delay_s: float | None = None + def _device(port: str, areas: dict[str, str]) -> tuple[str, str] | None: """The (area, kind) a port name stands for, or None.""" @@ -197,6 +276,39 @@ def _plain(command: str) -> str: return re.sub(r"[.,!?;:]+(?=\s|$)", "", command).strip().lower() +def _aliased(prompt: str, aliases: dict[str, str]) -> str: + """``prompt`` with the house's words in wren's, whole words in any case: + "the traverse" as "the living room light", "page too" as "patio".""" + if not aliases: + return prompt + words = {k.lower(): v for k, v in aliases.items()} + pattern = "|".join(map(re.escape, sorted(words, key=len, reverse=True))) + return re.sub( + rf"\b({pattern})\b", + lambda m: words[m[1].lower()], + prompt, + flags=re.IGNORECASE, + ) + + +def _digits(command: str) -> str: + """Number words as digits: "in five minutes" kept its delay only as "in 5 + minutes", and "twenty two degrees" read as 21.""" + tens, ones = "|".join(_TENS), "|".join(_ONES[1:10]) + command = re.sub( + rf"\b({tens})[ -]({ones})\b", + lambda m: str(_NUMBERS[m[1]] + _NUMBERS[m[2]]), + command, + ) + words = "|".join(_NUMBERS) + return re.sub(rf"\b({words})\b", lambda m: str(_NUMBERS[m[1]]), command) + + +def _named(areas: Iterable[str], command: str) -> list[str]: + """The areas ``command`` names, in the house's order.""" + return [a for a in areas if re.search(rf"\b{re.escape(a)}\b", command)] + + def _asked(house: Any, command: str) -> set[str]: """The areas a call may land on: those the command names, or else the user's and the living room. @@ -206,8 +318,7 @@ def _asked(house: Any, command: str) -> set[str]: word the model never learned landing on a neighbour: "switch on the appliances" turned on the bathroom light at 0.97. """ - named = {a for a in house.areas if re.search(rf"\b{re.escape(a)}\b", command)} - return named or {house.user_area, "living room"} + return set(_named(house.areas, command)) or {house.user_area, "living room"} def _text(call: Any) -> str: @@ -220,8 +331,15 @@ class WrenNode(Node): """Turn the command on ``prompt`` into values for the devices wired to it. Inputs: ``prompt`` (the command, and the only one that wakes the node), - ``presence`` (the area the user is in), ``outdoor`` (°C) and ``dark`` - (otherwise read off the clock: before 07:00 or from 18:00). + ``presence`` (the area the user is in), ``outdoor`` (°C), ``dark`` + (otherwise read off the clock: before 07:00 or from 18:00) and any + ``_temperature``. A question for a temperature is answered from + that reading, or from ``outdoor`` for an area outside. + + The command reaches the model with the ``aliases`` applied and its number + words as digits. A command for all of a kind ("turn off all lights", + "everything off in the office") is not the model's: the node applies it + to every wired device of that kind itself. Outputs: ``actions``, a list of records, one per call and target, ``reply``, a sentence saying what was applied ("Kitchen light on."), or @@ -246,8 +364,13 @@ class WrenNode(Node): le=1, description="Below this, nothing is written; the calls are still reported.", ) + aliases: dict[str, str] = Field( + default_factory=dict, + description="The house's own words, and words heard wrong, in wren's: " + '{"traverse": "living room light", "page too": "patio"}.', + ) - __slots__ = ("cfg", "devices") + __slots__ = ("cfg", "devices", "readings") # A call is a command to a device; doing it twice is not doing it once. idempotent = False @@ -264,12 +387,18 @@ class WrenNode(Node): inputs = self._normalize_ports(requires) outputs = self._normalize_ports(provides) + areas = {area.replace(" ", "_"): area for area in wren.AREAS} + #: The area each ``_temperature`` input reads, to its port. + self.readings: dict[str, str] = {} for spec in inputs: - allowed = INPUTS.get(spec.port) + reading = areas.get(spec.port.removesuffix("_temperature")) + if reading and spec.port.endswith("_temperature"): + self.readings[reading] = spec.port + allowed = INPUTS["outdoor"] if reading else INPUTS.get(spec.port) if allowed is None: raise ValueError( f"'{spec.port}' is not an input of wren. It reads " - f"{', '.join(INPUTS)}." + f"{', '.join(INPUTS)} and _temperature." ) if spec.dtype not in allowed: raise ValueError( @@ -279,7 +408,6 @@ class WrenNode(Node): if not any(spec.port == "prompt" for spec in inputs): raise ValueError("wren needs a 'prompt' input: the command to act on.") - areas = {area.replace(" ", "_"): area for area in wren.AREAS} #: (area, kind) to the ports standing for that device. self.devices: dict[tuple[str, str], list[str]] = {} for spec in outputs: @@ -364,6 +492,7 @@ class WrenNode(Node): presence: str | None = None, outdoor: float | None = None, dark: bool | None = None, + **readings: float | None, ) -> dict[str, Any] | None: if not prompt.strip(): # An empty command is nothing to do, which is what a flow input @@ -372,19 +501,33 @@ class WrenNode(Node): wren = _wren() specs = {spec.port: spec for spec in self.output_ports} house = self._house(wren, specs, presence, outdoor, dark) - command = _plain(prompt) + prompt = _aliased(prompt, self.cfg.aliases) + command = _digits(_plain(prompt)) asked = _asked(house, command) - # One BLAS thread answers in about half the time for a model this - # small. Scoped to the call, so numpy elsewhere in the process keeps - # its threads. - with threadpool_limits(1): - calls, confidence = wren.act(command, house) + group = self._group(command, house.areas) + if group is not None: + calls, confidence, asked = group, 1.0, set(house.areas) + else: + # One BLAS thread answers in about half the time for a model + # this small. Scoped to the call, so numpy elsewhere in the + # process keeps its threads. + with threadpool_limits(1): + calls, confidence = wren.act(command, house) confidence = round(float(confidence), 3) write = self.cfg.apply and confidence >= self.cfg.min_confidence writes: dict[str, Any] = {} actions: list[dict[str, Any]] = [] said: list[str] = [] + answers: list[str] = [] for call in calls: + area = str(call.args.get("area", "")) + if ( + call.tool == "sensors.temperature" + and area in asked + and confidence >= self.cfg.min_confidence + and (answer := self._temperature(area, outdoor, readings)) + ): + answers.append(answer) area, _, kind = str(call.args.get("id", "")).partition(":") ports = self.devices.get((area, kind), []) if call.tool.split(".")[0] != _TOOLS.get(kind) or area not in asked: @@ -419,19 +562,75 @@ class WrenNode(Node): done = done or applied if done: said.append(_said(call, area)) + if group is not None and said: + said = self._group_said(group, command, house.areas) wired = {spec.port for spec in self.output_ports} if ACTIONS in wired: writes[ACTIONS] = actions - if said and REPLY in wired: + if said: reply = ", ".join(said) - writes[REPLY] = reply[0].upper() + reply[1:] + "." - if not said and UNHANDLED in wired: + answers.insert(0, reply[0].upper() + reply[1:] + ".") + if answers and REPLY in wired: + writes[REPLY] = " ".join(answers) + if not answers and UNHANDLED in wired: writes[UNHANDLED] = prompt - if not said and REPLY in wired and not self._passed_on(): + if not answers and REPLY in wired and not self._passed_on(): writes[REPLY] = CANT return writes or None + def _group(self, command: str, areas: Iterable[str]) -> list[_Call] | None: + """The calls a command for a whole group means, one per wired device, + or None when it is not one: "turn off all lights" is every light. + + Not the model's to answer: it makes one call of it, and "turn off all + lights" switched a living room light on at 0.98. + """ + action = re.search(rf"\b({'|'.join(_GROUP_TOOLS)})\b", command) + if not _GROUP.search(command) or _NOT_GROUP.search(command) or not action: + return None + tools = _GROUP_TOOLS[action[1]] + kinds = {_GROUP_KINDS[w] for w in command.split() if w in _GROUP_KINDS} + named = _named(areas, command) + delay = re.search(r"\bin (\d+) minutes?\b", command) + seconds = int(delay[1]) * 60.0 if delay else None + calls = [ + _Call(tools[kind], {"id": f"{area}:{kind}"}, seconds) + for area, kind in self.devices + if kind in tools + and (not kinds or kind in kinds) + and (not named or area in named) + ] + return calls or None + + @staticmethod + def _group_said( + calls: list[_Call], command: str, areas: Iterable[str] + ) -> list[str]: + """A group command's reply: "all lights off", "lights off in the office".""" + said = list(dict.fromkeys(_GROUP_SAID[call.tool] for call in calls)) + named = _named(areas, command) + if named: + said[-1] += " in the " + " and the ".join(named) + else: + said[0] = "all " + said[0] + if calls[0].delay_s: + minutes = round(calls[0].delay_s / 60) + said[-1] += f" in {minutes} minute{'s' * (minutes != 1)}" + return said + + def _temperature( + self, area: str, outdoor: float | None, readings: dict[str, float | None] + ) -> str | None: + """ "It's 8 degrees outside.", from the reading wired for the area.""" + value = readings.get(self.readings.get(area, "")) + where = f"in the {area}" + if value is None and area in _OUTSIDE: + value, where = outdoor, "outside" + if not isinstance(value, (int, float)) or isinstance(value, bool): + return None + return f"It's {round(value)} degrees {where}." + def _passed_on(self) -> bool: """Whether a node reads ``unhandled``, such as an assistant.""" name = next((s.name for s in self.output_ports if s.port == UNHANDLED), "") diff --git a/backend/tests/flow/conftest.py b/backend/tests/flow/conftest.py index 8c6859b..5af4465 100644 --- a/backend/tests/flow/conftest.py +++ b/backend/tests/flow/conftest.py @@ -41,9 +41,11 @@ class FakeWren: def __init__(self) -> None: self.answer: tuple[list[Any], float] = ([], 1.0) self.houses: list[_House] = [] + self.commands: list[str] = [] def act(self, command: str, house: _House) -> tuple[list[Any], float]: self.houses.append(house) + self.commands.append(command) return self.answer diff --git a/backend/tests/flow/test_speech.py b/backend/tests/flow/test_speech.py index 2fa3bde..3c9c25c 100644 --- a/backend/tests/flow/test_speech.py +++ b/backend/tests/flow/test_speech.py @@ -6,7 +6,7 @@ from typing import Any from fluksio.ai import client from fluksio.flow.messages import DType, MessageSpec -from fluksio.flow.nodes.speech import SttNode, TtsNode, sentences +from fluksio.flow.nodes.speech import SttNode, TtsNode, sentences, speakable from fluksio.flow.pipeline import Pipeline URL = "http://speech:8100" @@ -40,6 +40,15 @@ def test_sentences_are_cut_after_their_ending(): ] +def test_units_and_clock_times_are_said_as_words(): + said = speakable("22 °C and 21.5° at 07:30, 3.4 kWh, 505 W, 2 kW, 7:05 or 18:00.") + assert said == ( + "22 degrees and 21.5 degrees at 7 30, 3.4 kilowatt hours, 505 watts, " + "2 kilowatts, 7 oh 5 or 18 o'clock." + ) + assert speakable("Kitchen light on, 3 Watts.") == "Kitchen light on, 3 Watts." + + def test_an_utterance_is_one_session_committed_when_listening_ends(monkeypatch): heard: list[bytes] = [] diff --git a/backend/tests/flow/test_wren.py b/backend/tests/flow/test_wren.py index c82eec2..0c4babf 100644 --- a/backend/tests/flow/test_wren.py +++ b/backend/tests/flow/test_wren.py @@ -216,3 +216,85 @@ def test_a_call_on_an_area_the_command_does_not_name_is_not_applied(fake_wren): assert out is not None assert "house.bathroom_light" not in out assert [a["applied"] for a in out["house.actions"]] == [False] + + +def _speaking(*targets: tuple[str, DType], inputs=(), **params: object) -> WrenNode: + node = WrenNode( + requires=[MessageSpec(name="command", port="prompt", dtype=DType.STR), *inputs], + provides=[ + *(MessageSpec(name=port, dtype=dtype) for port, dtype in targets), + MessageSpec(name="reply", dtype=DType.STR), + MessageSpec(name="unhandled", dtype=DType.STR), + ], + params={"apply": True, **params}, + ) + node.assign_flow("house", "control") + return node + + +def test_the_house_words_and_number_words_reach_the_model_as_its_own(fake_wren): + node = _speaking(("patio_light", DType.BOOL), aliases={"Page Too": "patio"}) + fake_wren.answer = ([], 0.99) + + out = node.execute({"house.command": "Turn on the page too light in five minutes."}) + + assert fake_wren.commands == ["turn on the patio light in 5 minutes"] + # The assistant reads the house's words too, as they were said otherwise. + assert out == { + "house.unhandled": "Turn on the patio light in five minutes.", + "house.reply": CANT, + } + + +def test_a_command_for_all_of_a_kind_reaches_every_wired_one(fake_wren): + lights = [(f"{area}_light", DType.BOOL) for area in ("kitchen", "office")] + node = _speaking(*lights, ("office_shutter", DType.STR)) + + out = node.execute({"house.command": "Turn off all lights."}) + assert out == { + "house.kitchen_light": False, + "house.office_light": False, + "house.reply": "All lights off.", + } + out = node.execute({"house.command": "turn everything off in the office"}) + assert out == { + "house.office_light": False, + "house.reply": "Lights off in the office.", + } + assert fake_wren.commands == [] # the node's, not the model's + + # A question or an exception is the model's again. + fake_wren.answer = ([], 0.99) + node.execute({"house.command": "are all lights off"}) + node.execute({"house.command": "turn off all lights but the kitchen"}) + assert len(fake_wren.commands) == 2 + + +def test_a_question_for_a_temperature_is_answered_from_its_reading(fake_wren): + node = _speaking( + inputs=[ + MessageSpec(name="outside", port="outdoor", dtype=DType.FLOAT), + MessageSpec( + name="kitchen_temp", port="kitchen_temperature", dtype=DType.FLOAT + ), + ] + ) + assert node.readings == {"kitchen": "kitchen_temperature"} + readings = {"house.outside": 7.6, "house.kitchen_temp": 21.2} + + for area, said in [("garden", "outside"), ("kitchen", "in the kitchen")]: + fake_wren.answer = ([FakeCall("sensors.temperature", {"area": area})], 0.99) + command = f"what is the temperature in the {area}" + out = node.execute({"house.command": command, **readings}) + degrees = 8 if area == "garden" else 21 + assert out == {"house.reply": f"It's {degrees} degrees {said}."} + + # No reading for the bedroom, or a model unsure of it: the assistant's. + for area, confidence in [("bedroom", 0.99), ("kitchen", 0.5)]: + fake_wren.answer = ( + [FakeCall("sensors.temperature", {"area": area})], + confidence, + ) + command = f"how warm is it in the {area}" + out = node.execute({"house.command": command, **readings}) + assert out == {"house.unhandled": command, "house.reply": CANT} diff --git a/docs/reference/node-types.md b/docs/reference/node-types.md index e55950b..02bf09a 100644 --- a/docs/reference/node-types.md +++ b/docs/reference/node-types.md @@ -354,6 +354,7 @@ reports a missing dependency. | `presence` | `str` | the area the user is in; anything else reads as away | | `outdoor` | `float` or `int` | °C; 20 without it, clamped to −5…30 | | `dark` | `bool` | without it, dark before 07:00 and from 18:00 | +| `_temperature` | `float` or `int` | °C, read when the command asks how warm the area is | The outputs are `actions`, a list of records, `reply`, `unhandled`, and any number of **targets**. A @@ -391,6 +392,7 @@ reads. |---|---|---| | `apply` | `false` | write to the targets; off, the node only reports | | `min_confidence` | `0.93` | below this nothing is written | +| `aliases` | empty | the house's own words, or words the recogniser hears wrong, in the model's: `{"traverse": "living room light", "page too": "patio"}` | Each record in `actions` contains `message`, `value`, `applied`, `delay_s`, `call`, and `confidence`. Unwired calls have an empty `message` and are not @@ -412,7 +414,22 @@ is reported but not applied, and goes on to `unhandled`. So is a call on an area the command does not name, unless it is the user's area or, with no area named, the living room: "switch on the appliances" is a word the model never learned, and it lands on whichever light it was shown. The command is read -in lower case without sentence marks, as the model was trained. +in lower case without sentence marks and with number words as digits ("in +five minutes" as "in 5 minutes"), as the model was trained. `aliases` are +applied before, whole words in any case, and `unhandled` carries the prompt +with them. + +A command for all of a kind, such as "turn off all lights", "switch every +light on in 5 minutes" or "everything off in the office", is not the +model's. The node applies it to every wired target of that kind, within the +areas it names, and replies "All lights off." or "Lights off in the +office.". A question ("are all lights off") or an exception ("all but the +kitchen") goes to the model as any other command. + +A question for an area's temperature ("how warm is it in the kitchen") is +answered from its `_temperature` input, or from `outdoor` for an area +outside, as "It's 8 degrees outside.". Without a reading it goes on to +`unhandled`. !!! note "What the current model cannot do" @@ -519,7 +536,11 @@ such as one that sends whole clips. | `voice` | `af_heart` | one of Kokoro's voices, such as `bm_george` | Each sentence is one call to `{url}/v1/audio/speech` and published as soon as -it is spoken. Calls to one server share a connection, which Speech to text opens +it is spoken. Units after a number and clock times are written out first, +since a speech model reads them letter by letter: "22 °C" is said "22 +degrees", "3.4 kWh" "3.4 kilowatt hours" and "07:30" "7 30". How a word +itself is pronounced is the server's: `parrot serve --lexicon` gives words +their phonemes. Calls to one server share a connection, which Speech to text opens when an utterance starts, so a reply's first sentence does not wait for one. A sentence spoken before in the same voice is reused rather than spoken again, for the last 256 sentences. A reference carries `"append": true` while the sentences before it are still playing, which queues it on a panel's