Files
app/backend/fluksio/flow/nodes/wren.py
T

710 lines
27 KiB
Python

"""Wren: a typed command in, values for the devices wired to it out.
`Wren <https://github.com/stroblme/wren>`_ is a small language model trained on
one job: read a house — its areas, the devices in each and their states — and
answer a command with calls on those devices. This node is the bridge. Its
output ports *are* the devices, named ``<area>_<kind>``, so the wiring is the
allowlist: the model can reach nothing that is not wired to the node.
"""
from __future__ import annotations
import re
import time
from collections.abc import Iterable
from typing import Any, NamedTuple
from pydantic import BaseModel, ConfigDict, Field
from threadpoolctl import threadpool_limits
from fluksio.flow.messages import DType, MessageSpec
from fluksio.flow.nodes.base import Node
def _wren() -> Any:
"""The model package, which is optional.
Imported when a node is built rather than at boot, so an instance without
it still offers every other type, and a node placed without it says so
on the canvas straight away.
"""
try:
import wren
except ImportError:
# Not `pip install wren`: that name on PyPI is another package.
raise RuntimeError(
"the wren node needs the model: pip install git+https://github.com/"
"stroblme/wren@eae4d6ff9ded68e36624972fd3f4a8f8d210b0c2"
) from None
return wren
#: The last word of a target port, and the kind of device it stands for.
KIND_WORDS = {
"light": "light",
"lamp": "light",
"brightness": "light",
"setpoint": "heat",
"heating": "heat",
"heat": "heat",
"boost": "heat",
"media": "media",
"music": "media",
"tv": "media",
"shutter": "shutter",
"blind": "shutter",
"cover": "shutter",
"water": "water",
"watering": "water",
"sprinkler": "water",
"switch": "switch",
"plug": "switch",
"socket": "switch",
"appliances": "switch",
}
#: What the node reads, and the types each may be. Only the prompt wakes it.
#: Besides these, ``<area>_temperature`` inputs answer "how warm is it".
INPUTS = {
"prompt": {DType.STR},
"presence": {DType.STR},
"outdoor": {DType.FLOAT, DType.INT},
"dark": {DType.BOOL},
}
#: The output the calls are reported on, and the two that say what came of
#: the command. Every other output is a device.
ACTIONS = "actions"
REPLY = "reply"
UNHANDLED = "unhandled"
#: The reply when nothing was done and nothing reads ``unhandled``.
CANT = "Sorry, I can't do that yet."
#: The output for the house's brightness, 0 to 100, which the model's
#: brightness calls write whatever area they name.
BRIGHTNESS = "brightness"
_TARGET_TYPES = {DType.BOOL, DType.INT, DType.FLOAT, DType.STR}
#: The tools that act on each kind.
_TOOLS = {
"light": "lights",
"media": "media",
"heat": "heating",
"shutter": "shutters",
"water": "watering",
"switch": "switches",
}
#: What a call sets a target to, by the target's type: (bool, number, text).
#: None is a call that type has no value for, which is then not applied.
#: `heating.set` carries its own number.
_VALUES: dict[str, tuple[bool | None, float | None, str | None]] = {
"lights.on": (True, 100, "on"),
"lights.off": (False, 0, "off"),
"media.play": (True, None, "play"),
"media.stop": (False, None, "stop"),
"heating.boost": (True, None, "boost"),
"heating.off": (False, None, "off"),
"shutters.up": (True, 100, "up"),
"shutters.down": (False, 0, "down"),
"shutters.stop": (None, None, "stop"),
"watering.start": (True, None, "on"),
"watering.stop": (False, None, "off"),
"switches.on": (True, 1, "on"),
"switches.off": (False, 0, "off"),
}
#: The brightness calls, and what "dim" and "brighter" do: a step from the
#: brightness now, or from a usual one when none is known. Dimmed lights
#: stay on.
_LEVELS = ("lights.set", "lights.dim", "lights.brighten")
_STEP, _DIMMEST, _BRIGHTEST, _USUAL = 30, 10, 100, 70
#: An applied call, said: "kitchen light on". `heating.set` says its number.
_SAID = {
"lights.on": "light on",
"lights.off": "light off",
"media.play": "media playing",
"media.stop": "media stopped",
"heating.boost": "heating boosted",
"heating.off": "heating off",
"shutters.up": "shutters up",
"shutters.down": "shutters down",
"shutters.stop": "shutters stopped",
"watering.start": "watering on",
"watering.stop": "watering off",
"switches.on": "switch on",
"switches.off": "switch off",
}
#: What every area held in training: a light, and in the living room, which
#: always existed, media and heating besides. A house missing them reads as
#: unfamiliar and the confidence drops with it, so they are filled in; a call
#: on one that nothing is wired to is reported and not applied. Every area is
#: shown, wired or not, or "the bathroom light" lands on a light that is.
_TRAINED = {"living room": ("light", "media", "heat")}
#: wren's areas outside, whose temperature is the ``outdoor`` reading.
_OUTSIDE = {"garden", "patio", "garage", "driveway", "balcony"}
#: Number words, which wren was trained on as digits.
_ONES = (
"zero one two three four five six seven eight nine ten eleven twelve "
"thirteen fourteen fifteen sixteen seventeen eighteen nineteen"
).split()
_TENS = "twenty thirty forty fifty sixty seventy eighty ninety".split()
_NUMBERS = {w: n for n, w in enumerate(_ONES)} | {
w: 20 + 10 * n for n, w in enumerate(_TENS)
}
#: A command for a whole group of devices: "turn off all lights".
_GROUP = re.compile(r"\b(all|every|everything|everywhere)\b")
#: What leaves such a command to the model and the assistant: a question, or
#: an exception to the group.
_NOT_GROUP = re.compile(
r"^(is|are|was|were|do|does|did|can|could|what|which|how|why|when|where|who)\b"
r"|\b(except|but|apart from|other than)\b"
)
#: The kind a group command names, by its word. None named is every kind.
_GROUP_KINDS = {
"light": "light",
"lights": "light",
"lamps": "light",
"music": "media",
"media": "media",
"tv": "media",
"shutters": "shutter",
"blinds": "shutter",
"heating": "heat",
"watering": "water",
"sprinklers": "water",
"switches": "switch",
"plugs": "switch",
"appliances": "switch",
}
#: The call a group command makes on each kind, by its action word.
_GROUP_TOOLS = {
"on": {
"light": "lights.on",
"media": "media.play",
"water": "watering.start",
"switch": "switches.on",
},
"off": {
"light": "lights.off",
"media": "media.stop",
"heat": "heating.off",
"water": "watering.stop",
"switch": "switches.off",
},
"open": {"shutter": "shutters.up"},
"up": {"shutter": "shutters.up"},
"raise": {"shutter": "shutters.up"},
"close": {"shutter": "shutters.down"},
"down": {"shutter": "shutters.down"},
"lower": {"shutter": "shutters.down"},
"stop": {
"media": "media.stop",
"shutter": "shutters.stop",
"water": "watering.stop",
},
}
#: A group call, said: "all lights off".
_GROUP_SAID = {
"lights.on": "lights on",
"lights.off": "lights off",
"media.play": "media playing",
"media.stop": "media stopped",
"heating.off": "heating off",
"shutters.up": "shutters up",
"shutters.down": "shutters down",
"shutters.stop": "shutters stopped",
"watering.start": "watering on",
"watering.stop": "watering off",
"switches.on": "switches on",
"switches.off": "switches off",
}
class _Call(NamedTuple):
"""A call the node makes itself, shaped as the model's."""
tool: str
args: dict[str, Any]
delay_s: float | None = None
def _device(port: str, areas: dict[str, str]) -> tuple[str, str] | None:
"""The (area, kind) a port name stands for, or None."""
for slug, area in areas.items():
word = port.removeprefix(f"{slug}_")
if word != port and word in KIND_WORDS:
return area, KIND_WORDS[word]
return None
def _on(value: Any) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, (int, float)):
return value > 0
return str(value).strip().lower() in {"on", "up", "open", "play", "true"}
def _state(kind: str, values: list[Any]) -> str:
"""A device's state as the model reads it, from what its targets hold."""
if kind == "heat":
temps = [
v for v in values if isinstance(v, (int, float)) and not isinstance(v, bool)
]
return f"{round(temps[0])}C" if temps else "20C"
on = bool(values) and _on(values[0])
if kind == "shutter":
return "up" if on else "down"
return "on" if on else "off"
def _value(call: Any, dtype: DType) -> Any:
"""What ``call`` sets a target of this type to, or None."""
if call.tool == "heating.set":
temp = call.args.get("temp")
flag, number, text = (
None,
temp if isinstance(temp, (int, float)) else None,
None,
)
elif call.tool == "lights.set":
level = call.args.get("level")
flag, number, text = (
None,
level if isinstance(level, (int, float)) else None,
None,
)
else:
flag, number, text = _VALUES.get(call.tool, (None, None, None))
if dtype is DType.BOOL:
return flag
if dtype is DType.STR:
return text
if number is None:
return None
return int(number) if dtype is DType.INT else float(number)
def _said(call: Any, area: str) -> str:
"""An applied call as the reply says it: "garage light on in 5 minutes"."""
if call.tool == "heating.set":
said = f"heating to {_value(call, DType.FLOAT):g} degrees"
elif call.tool == "lights.set":
said = f"brightness at {_value(call, DType.FLOAT):g} percent"
else:
said = _SAID.get(call.tool, call.tool)
if call.delay_s:
minutes = round(call.delay_s / 60)
said += f" in {minutes} minute{'s' * (minutes != 1)}"
# The brightness is the house's, not the area's the call names.
return said if call.tool == "lights.set" else f"{area} {said}"
def _plain(command: str) -> str:
"""The command as the model was trained on it: lower case, no sentence marks.
A transcript's "Turn on the kitchen light." reads less sure than the same
words plain, 0.80 against 0.99 with every area shown.
"""
return re.sub(r"[.,!?;:]+(?=\s|$)", "", command).strip().lower()
def _aliased(prompt: str, aliases: dict[str, str]) -> str:
"""``prompt`` with the house's words in wren's, whole words in any case:
"the traverse" as "the living room light", "page too" as "patio"."""
if not aliases:
return prompt
words = {k.lower(): v for k, v in aliases.items()}
pattern = "|".join(map(re.escape, sorted(words, key=len, reverse=True)))
return re.sub(
rf"\b({pattern})\b",
lambda m: words[m[1].lower()],
prompt,
flags=re.IGNORECASE,
)
def _digits(command: str) -> str:
"""Number words as digits: "in five minutes" kept its delay only as "in 5
minutes", and "twenty two degrees" read as 21."""
tens, ones = "|".join(_TENS), "|".join(_ONES[1:10])
command = re.sub(
rf"\b({tens})[ -]({ones})\b",
lambda m: str(_NUMBERS[m[1]] + _NUMBERS[m[2]]),
command,
)
words = "|".join(_NUMBERS)
return re.sub(rf"\b({words})\b", lambda m: str(_NUMBERS[m[1]]), command)
def _named(areas: Iterable[str], command: str) -> list[str]:
"""The areas ``command`` names, in the house's order."""
return [a for a in areas if re.search(rf"\b{re.escape(a)}\b", command)]
def _asked(house: Any, command: str) -> set[str]:
"""The areas a call may land on: those the command names, or else the
user's and the living room.
That is how wren picks the areas it shows as the command's own. The few
it adds besides are there to be told apart, and a call on one of them is a
word the model never learned landing on a neighbour: "switch on the
appliances" turned on the bathroom light at 0.97.
"""
return set(_named(house.areas, command)) or {house.user_area, "living room"}
def _text(call: Any) -> str:
"""The call as the chat shows it, less the device it names."""
args = ", ".join(f"{k}={v}" for k, v in call.args.items() if k != "id")
return f"{call.tool}({args})"
class WrenNode(Node):
"""Turn the command on ``prompt`` into values for the devices wired to it.
Inputs: ``prompt`` (the command, and the only one that wakes the node),
``presence`` (the area the user is in), ``outdoor`` (°C), ``dark``
(otherwise read off the clock: before 07:00 or from 18:00) and any
``<area>_temperature``. A question for a temperature is answered from
that reading, or from ``outdoor`` for an area outside.
The command reaches the model with the ``aliases`` applied and its number
words as digits. A command for all of a kind ("turn off all lights",
"everything off in the office") is not the model's: the node applies it
to every wired device of that kind itself.
Outputs: ``actions``, a list of records, one per call and target,
``reply``, a sentence saying what was applied ("Kitchen light on."), or
that nothing was when no node reads ``unhandled``, ``unhandled``, the
prompt when nothing was, ``brightness``, the house's, for "dim the
lights" or "set the brightness to 30%", and any number of targets
named ``<area>_<kind>``. Targets of one area and kind are one device to
the model; each takes what the call means for its type.
A delayed call is held by the engine's work queue, like the Trigger
node's second value.
"""
class Params(BaseModel):
model_config = ConfigDict(extra="allow")
apply: bool = Field(
default=False,
description="Write to the targets. Off, the node only reports what it would do.",
)
min_confidence: float = Field(
default=0.93,
ge=0,
le=1,
description="Below this, nothing is written; the calls are still reported.",
)
aliases: dict[str, str] = Field(
default_factory=dict,
description="The house's own words, and words heard wrong, in wren's: "
'{"traverse": "living room light", "page too": "patio"}.',
)
__slots__ = ("cfg", "devices", "readings")
# A call is a command to a device; doing it twice is not doing it once.
idempotent = False
def __init__(
self,
requires: MessageSpec | Iterable[MessageSpec] = (),
provides: MessageSpec | Iterable[MessageSpec] = (),
params: dict[str, Any] | None = None,
name: str | None = None,
):
self.cfg = self.Params.model_validate(params or {})
wren = _wren()
inputs = self._normalize_ports(requires)
outputs = self._normalize_ports(provides)
areas = {area.replace(" ", "_"): area for area in wren.AREAS}
#: The area each ``<area>_temperature`` input reads, to its port.
self.readings: dict[str, str] = {}
for spec in inputs:
reading = areas.get(spec.port.removesuffix("_temperature"))
if reading and spec.port.endswith("_temperature"):
self.readings[reading] = spec.port
allowed = INPUTS["outdoor"] if reading else INPUTS.get(spec.port)
if allowed is None:
raise ValueError(
f"'{spec.port}' is not an input of wren. It reads "
f"{', '.join(INPUTS)} and <area>_temperature."
)
if spec.dtype not in allowed:
raise ValueError(
f"'{spec.port}' is {spec.dtype.value}; wren reads it as "
f"{' or '.join(sorted(d.value for d in allowed))}."
)
if not any(spec.port == "prompt" for spec in inputs):
raise ValueError("wren needs a 'prompt' input: the command to act on.")
#: (area, kind) to the ports standing for that device.
self.devices: dict[tuple[str, str], list[str]] = {}
for spec in outputs:
if spec.port == ACTIONS:
if spec.dtype is not DType.LIST or spec.item_dtype is not DType.RECORD:
raise ValueError(f"'{ACTIONS}' is a list of records.")
continue
if spec.port in (REPLY, UNHANDLED):
if spec.dtype is not DType.STR:
raise ValueError(f"'{spec.port}' is a str.")
continue
if spec.port == BRIGHTNESS:
if spec.dtype not in (DType.FLOAT, DType.INT):
raise ValueError(f"'{BRIGHTNESS}' is a float or an int.")
continue
device = _device(spec.port, areas)
if device is None:
raise ValueError(
f"'{spec.port}' names no device wren knows. A target is "
f"<area>_<kind>, the area one of {', '.join(areas)}, the kind "
f"one of {', '.join(KIND_WORDS)}."
)
if device[1] not in wren.KINDS: # switches came with wren's s10
raise ValueError(
f"'{spec.port}' is a {device[1]}, which this wren does not know."
)
if spec.dtype not in _TARGET_TYPES:
raise ValueError(
f"'{spec.port}' is {spec.dtype.value}; a target is bool, int, "
"float or str."
)
self.devices.setdefault(device, []).append(spec.port)
super().__init__(
self._act,
# The prompt is the event; the rest is read when it arrives.
requires=[
spec.model_copy(update={"trigger": spec.port == "prompt"})
for spec in inputs
],
provides=outputs,
params=params,
name=name or "wren",
)
def _house(
self,
wren: Any,
specs: dict[str, MessageSpec],
presence: str | None,
outdoor: float | None,
dark: bool | None,
) -> Any:
"""The house as the model reads it, each device in its current state."""
names = [specs[port].name for ports in self.devices.values() for port in ports]
current = (
self._pipeline.state.get_present(names)
if self._pipeline is not None
else {}
)
areas: dict[str, list[Any]] = {}
for area in wren.AREAS:
wired = {kind for (a, kind) in self.devices if a == area}
kinds = wired | set(_TRAINED.get(area, ("light",)))
areas[area] = []
for kind in (k for k in wren.KINDS if k in kinds):
ports = self.devices.get((area, kind), [])
held = [
current[specs[p].name] for p in ports if specs[p].name in current
]
# The id is ours to choose: the model is shown aliases anyway.
areas[area].append(
wren.Device(kind, f"{area}:{kind}", _state(kind, held))
)
clock = time.localtime()
where = str(presence or "").strip().lower().replace("_", " ")
return wren.House(
time=time.strftime("%H:%M", clock),
# The range the model was trained on; past it a number is unfamiliar.
outdoor_c=min(max(float(20 if outdoor is None else outdoor), -5), 30),
dark=clock.tm_hour < 7 or clock.tm_hour >= 18 if dark is None else dark,
user_area=where if where in wren.AREAS else None,
areas=areas,
)
def _act(
self,
params: dict[str, Any],
prompt: str = "",
presence: str | None = None,
outdoor: float | None = None,
dark: bool | None = None,
**readings: float | None,
) -> dict[str, Any] | None:
if not prompt.strip():
# An empty command is nothing to do, which is what a flow input
# starts as.
return None
wren = _wren()
specs = {spec.port: spec for spec in self.output_ports}
house = self._house(wren, specs, presence, outdoor, dark)
prompt = _aliased(prompt, self.cfg.aliases)
command = _digits(_plain(prompt))
asked = _asked(house, command)
group = self._group(command, house.areas)
if group is not None:
calls, confidence, asked = group, 1.0, set(house.areas)
else:
# One BLAS thread answers in about half the time for a model
# this small. Scoped to the call, so numpy elsewhere in the
# process keeps its threads.
with threadpool_limits(1):
calls, confidence = wren.act(command, house)
confidence = round(float(confidence), 3)
write = self.cfg.apply and confidence >= self.cfg.min_confidence
writes: dict[str, Any] = {}
actions: list[dict[str, Any]] = []
said: list[str] = []
answers: list[str] = []
for call in calls:
area = str(call.args.get("area", ""))
if (
call.tool == "sensors.temperature"
and area in asked
and confidence >= self.cfg.min_confidence
and (answer := self._temperature(area, outdoor, readings))
):
answers.append(answer)
area, _, kind = str(call.args.get("id", "")).partition(":")
ports = self.devices.get((area, kind), [])
if call.tool.split(".")[0] != _TOOLS.get(kind) or area not in asked:
ports = []
if call.tool in _LEVELS:
ports = []
if BRIGHTNESS in specs:
ports = [BRIGHTNESS]
call = self._leveled(call, specs[BRIGHTNESS].name)
hits = [
(specs[port], value)
for port in ports
if (value := _value(call, specs[port].dtype)) is not None
]
record = {
"message": "",
"value": None,
"applied": False,
"delay_s": call.delay_s,
"call": _text(call),
"confidence": confidence,
}
if not hits:
actions.append(record)
done = False
for spec, value in hits:
applied = write
if applied and call.delay_s:
applied = self._pipeline is not None and self._pipeline.defer(
self, self._to_messages({spec.port: value}) or {}, call.delay_s
)
elif applied:
writes[spec.port] = value
actions.append(
record | {"message": spec.name, "value": value, "applied": applied}
)
done = done or applied
if done:
said.append(_said(call, area))
if group is not None and said:
said = self._group_said(group, command, house.areas)
wired = {spec.port for spec in self.output_ports}
if ACTIONS in wired:
writes[ACTIONS] = actions
if said:
reply = ", ".join(dict.fromkeys(said))
answers.insert(0, reply[0].upper() + reply[1:] + ".")
if answers and REPLY in wired:
writes[REPLY] = " ".join(answers)
if not answers and UNHANDLED in wired:
writes[UNHANDLED] = prompt
if not answers and REPLY in wired and not self._passed_on():
writes[REPLY] = CANT
return writes or None
def _leveled(self, call: Any, name: str) -> _Call:
"""A brightness call as the level it sets: "dim" is 30 below now."""
level = call.args.get("level")
if call.tool != "lights.set":
now = (
self._pipeline.state.get_present([name]).get(name)
if self._pipeline is not None
else None
)
if not isinstance(now, (int, float)) or isinstance(now, bool):
now = _USUAL
if call.tool == "lights.dim":
level = max(now - _STEP, min(now, _DIMMEST))
else:
level = min(now + _STEP, _BRIGHTEST)
elif isinstance(level, (int, float)):
level = min(max(level, 0), _BRIGHTEST)
return _Call("lights.set", {"level": level}, call.delay_s)
def _group(self, command: str, areas: Iterable[str]) -> list[_Call] | None:
"""The calls a command for a whole group means, one per wired device,
or None when it is not one: "turn off all lights" is every light.
Not the model's to answer: it makes one call of it, and "turn off all
lights" switched a living room light on at 0.98.
"""
action = re.search(rf"\b({'|'.join(_GROUP_TOOLS)})\b", command)
if not _GROUP.search(command) or _NOT_GROUP.search(command) or not action:
return None
tools = _GROUP_TOOLS[action[1]]
kinds = {_GROUP_KINDS[w] for w in command.split() if w in _GROUP_KINDS}
named = _named(areas, command)
delay = re.search(r"\bin (\d+) minutes?\b", command)
seconds = int(delay[1]) * 60.0 if delay else None
calls = [
_Call(tools[kind], {"id": f"{area}:{kind}"}, seconds)
for area, kind in self.devices
if kind in tools
and (not kinds or kind in kinds)
and (not named or area in named)
]
return calls or None
@staticmethod
def _group_said(
calls: list[_Call], command: str, areas: Iterable[str]
) -> list[str]:
"""A group command's reply: "all lights off", "lights off in the office"."""
said = list(dict.fromkeys(_GROUP_SAID[call.tool] for call in calls))
named = _named(areas, command)
if named:
said[-1] += " in the " + " and the ".join(named)
else:
said[0] = "all " + said[0]
if calls[0].delay_s:
minutes = round(calls[0].delay_s / 60)
said[-1] += f" in {minutes} minute{'s' * (minutes != 1)}"
return said
def _temperature(
self, area: str, outdoor: float | None, readings: dict[str, float | None]
) -> str | None:
""" "It's 8 degrees outside.", from the reading wired for the area."""
value = readings.get(self.readings.get(area, ""))
where = f"in the {area}"
if value is None and area in _OUTSIDE:
value, where = outdoor, "outside"
if not isinstance(value, (int, float)) or isinstance(value, bool):
return None
return f"It's {round(value)} degrees {where}."
def _passed_on(self) -> bool:
"""Whether a node reads ``unhandled``, such as an assistant."""
name = next((s.name for s in self.output_ports if s.port == UNHANDLED), "")
return bool(name and self._pipeline and self._pipeline.consumes.get(name))