451 lines
15 KiB
Python
451 lines
15 KiB
Python
"""The wren node: its wiring is the allowlist, and nothing is written unasked.
|
|
|
|
The model is an optional package, so ``fake_wren`` (conftest) answers in its
|
|
place: what the node does with an answer is what is under test here.
|
|
"""
|
|
|
|
import asyncio
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from fluksio.flow.artifacts import ArtifactStore
|
|
from fluksio.flow.messages import DType, MessageSpec
|
|
from fluksio.flow.nodes.base import Node
|
|
from fluksio.flow.nodes.wren import CANT, WrenNode, _plain
|
|
from fluksio.flow.pipeline import Pipeline
|
|
|
|
PROMPT = MessageSpec(name="command", port="prompt", dtype=DType.STR)
|
|
OUTDOOR = MessageSpec(name="outdoor", port="outdoor_temperature", dtype=DType.FLOAT)
|
|
LIGHT = MessageSpec(name="kitchen_light", dtype=DType.BOOL)
|
|
|
|
|
|
def _node(
|
|
fake_wren,
|
|
*targets: tuple[str, DType],
|
|
inputs=(),
|
|
saying=False,
|
|
trained=None,
|
|
health=None,
|
|
app=None,
|
|
**params: Any,
|
|
) -> WrenNode:
|
|
"""A started node, its model trained on its own targets unless ``trained``
|
|
lists others."""
|
|
node = WrenNode(
|
|
requires=[PROMPT, *inputs],
|
|
provides=[
|
|
MessageSpec(name="actions", dtype=DType.LIST),
|
|
*(MessageSpec(name=port, dtype=dtype) for port, dtype in targets),
|
|
*(
|
|
MessageSpec(name=port, dtype=DType.STR)
|
|
for port in ("reply", "unhandled")
|
|
if saying
|
|
),
|
|
],
|
|
params=params,
|
|
)
|
|
node.assign_flow("house", "control")
|
|
if health is not None:
|
|
node._on_health = lambda _node, status, detail: health.append((status, detail))
|
|
fake_wren.pack["meta"]["targets"] = (
|
|
list(node.targets.values()) if trained is None else trained
|
|
)
|
|
asyncio.run(node.start(app))
|
|
return node
|
|
|
|
|
|
def _record(port: str, value: Any, call: str, **more: Any) -> dict[str, Any]:
|
|
return {
|
|
"message": f"house.{port}",
|
|
"value": value,
|
|
"applied": True,
|
|
"delay_s": None,
|
|
"call": call,
|
|
"confidence": 1.0,
|
|
} | more
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("requires", "provides", "params", "refusal"),
|
|
[
|
|
([], [LIGHT], {}, "prompt"),
|
|
([PROMPT], [MessageSpec(name="plan", dtype=DType.JSON)], {}, "plan"),
|
|
([PROMPT], [LIGHT], {"targets": {"hall_light": {}}}, "hall_light"),
|
|
(
|
|
[PROMPT],
|
|
[LIGHT],
|
|
{"targets": {"kitchen_light": {"values": ["ON"]}}},
|
|
"values",
|
|
),
|
|
([PROMPT], [LIGHT], {"targets": {"kitchen_light": {"range": [0, 1]}}}, "range"),
|
|
],
|
|
)
|
|
def test_a_wiring_wren_cannot_act_on_is_refused(
|
|
fake_wren, requires, provides, params, refusal
|
|
):
|
|
with pytest.raises(ValueError, match=refusal):
|
|
WrenNode(requires=requires, provides=provides, params=params)
|
|
|
|
|
|
def test_only_the_prompt_wakes_it(fake_wren):
|
|
node = WrenNode(requires=[PROMPT, OUTDOOR])
|
|
assert {spec.port: spec.trigger for spec in node.input_ports} == {
|
|
"prompt": True,
|
|
"outdoor_temperature": False,
|
|
}
|
|
|
|
|
|
def test_the_model_reads_the_command_plain():
|
|
assert _plain("Turn on the kitchen light.") == "turn on the kitchen light"
|
|
assert _plain("Heat it to 22.5, please!") == "heat it to 22.5 please"
|
|
|
|
|
|
def test_the_targets_reach_the_model_with_their_values_and_settings(fake_wren):
|
|
node = _node(
|
|
fake_wren,
|
|
("kitchen_light", DType.BOOL),
|
|
("living_room_heating", DType.FLOAT),
|
|
("kitchen_shutter", DType.STR),
|
|
inputs=[OUTDOOR],
|
|
targets={
|
|
"living_room_heating": {"range": [5, 30], "unit": "°C"},
|
|
"kitchen_shutter": {"values": ["UP", "DOWN"]},
|
|
"outdoor_temperature": {"description": "temperature outside"},
|
|
},
|
|
)
|
|
Pipeline([node]).state["house.kitchen_light"] = True
|
|
|
|
node.execute({"house.command": "warm it up", "house.outdoor": 6.2})
|
|
|
|
assert fake_wren.loaded == [fake_wren.runtime.MODEL]
|
|
assert fake_wren.models == [fake_wren.pack]
|
|
# Targets nothing has set yet are off, 0 within their range, or their
|
|
# first value.
|
|
target = fake_wren.Target
|
|
assert fake_wren.targets[0] == [
|
|
target("kitchen_light", "bool", True),
|
|
target("living_room_heating", "float", 5.0, range=[5.0, 30.0], unit="°C"),
|
|
target("kitchen_shutter", "str", "UP", values=["UP", "DOWN"]),
|
|
target(
|
|
"outdoor_temperature",
|
|
"float",
|
|
6.2,
|
|
description="temperature outside",
|
|
readonly=True,
|
|
),
|
|
]
|
|
|
|
|
|
def test_applied_calls_reach_their_targets_and_the_reply_says_so(fake_wren):
|
|
node = _node(
|
|
fake_wren,
|
|
("kitchen_light", DType.BOOL),
|
|
("living_room_heating", DType.FLOAT),
|
|
saying=True,
|
|
apply=True,
|
|
targets={"living_room_heating": {"range": [5, 30], "unit": "°C"}},
|
|
)
|
|
# 0.95 clears the model's own threshold, 0.9 here.
|
|
fake_wren.answer = [
|
|
fake_wren.Call("kitchen_light", True, confidence=0.95),
|
|
fake_wren.Call("living_room_heating", 21),
|
|
]
|
|
|
|
out = node.execute({"house.command": "kitchen light on and the heating to 21"})
|
|
|
|
assert out == {
|
|
"house.kitchen_light": True,
|
|
"house.living_room_heating": 21.0,
|
|
"house.reply": "Kitchen light on, living room heating to 21 °C.",
|
|
"house.actions": [
|
|
_record("kitchen_light", True, "set(kitchen_light, true)", confidence=0.95),
|
|
_record("living_room_heating", 21.0, "set(living_room_heating, 21)"),
|
|
],
|
|
}
|
|
assert isinstance(out["house.living_room_heating"], float)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("params", "confidence"),
|
|
[
|
|
({}, 0.99),
|
|
# Below the model's own threshold, 0.9 here.
|
|
({"apply": True}, 0.85),
|
|
({"apply": True, "min_confidence": 1}, 0.99),
|
|
],
|
|
)
|
|
def test_nothing_is_written_unless_applied_and_sure(fake_wren, params, confidence):
|
|
node = _node(fake_wren, ("kitchen_light", DType.BOOL), **params)
|
|
fake_wren.answer = [fake_wren.Call("kitchen_light", True, confidence=confidence)]
|
|
|
|
out = node.execute({"house.command": "turn on the kitchen light"})
|
|
|
|
assert out == {
|
|
"house.actions": [
|
|
_record(
|
|
"kitchen_light",
|
|
True,
|
|
"set(kitchen_light, true)",
|
|
applied=False,
|
|
confidence=confidence,
|
|
)
|
|
]
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("params", "applied"),
|
|
[
|
|
({}, True),
|
|
# One call short of its target's bar stops the whole command.
|
|
({"targets": {"kitchen_light": {"min_confidence": 0.99}}}, False),
|
|
({"targets": {"kitchen_light": {"pass_on": True}}}, False),
|
|
({"min_confidence": 0.99}, False),
|
|
# A target's own bar is taken over the node's.
|
|
(
|
|
{
|
|
"min_confidence": 0.99,
|
|
"targets": {
|
|
"kitchen_light": {"min_confidence": 0.9},
|
|
"garden_light": {"min_confidence": 0.9},
|
|
},
|
|
},
|
|
True,
|
|
),
|
|
],
|
|
)
|
|
def test_a_command_is_applied_whole_or_not_at_all(fake_wren, params, applied):
|
|
node = _node(
|
|
fake_wren,
|
|
("kitchen_light", DType.BOOL),
|
|
("garden_light", DType.BOOL),
|
|
saying=True,
|
|
apply=True,
|
|
**params,
|
|
)
|
|
fake_wren.answer = [
|
|
fake_wren.Call("garden_light", True, confidence=0.95),
|
|
fake_wren.Call("kitchen_light", True, confidence=0.95),
|
|
]
|
|
command = "garden and kitchen lights on"
|
|
|
|
out = node.execute({"house.command": command})
|
|
|
|
assert out is not None
|
|
assert [a["applied"] for a in out.pop("house.actions")] == [applied, applied]
|
|
assert out == (
|
|
{
|
|
"house.garden_light": True,
|
|
"house.kitchen_light": True,
|
|
"house.reply": "Garden light on, kitchen light on.",
|
|
}
|
|
if applied
|
|
else {"house.unhandled": command, "house.reply": CANT}
|
|
)
|
|
|
|
|
|
def test_a_target_the_model_was_not_trained_on_is_never_applied(fake_wren):
|
|
trained = [
|
|
# As the pack writes it: [5, 30] is the node's (5.0, 30.0), and the
|
|
# unit is not the model's to know.
|
|
{
|
|
"name": "living_room_heating",
|
|
"type": "float",
|
|
"values": None,
|
|
"range": [5, 30],
|
|
"description": "",
|
|
"unit": None,
|
|
"readonly": False,
|
|
},
|
|
# Described otherwise since.
|
|
{
|
|
"name": "canopy",
|
|
"type": "str",
|
|
"values": ["UP", "DOWN", "STOP"],
|
|
"range": None,
|
|
"description": "awning",
|
|
"unit": None,
|
|
"readonly": False,
|
|
},
|
|
]
|
|
health: list[tuple[str, str | None]] = []
|
|
node = _node(
|
|
fake_wren,
|
|
("living_room_heating", DType.FLOAT),
|
|
("canopy", DType.STR),
|
|
("garden_light", DType.BOOL),
|
|
saying=True,
|
|
trained=trained,
|
|
health=health,
|
|
apply=True,
|
|
targets={
|
|
"living_room_heating": {"range": [5, 30], "unit": "°C"},
|
|
"canopy": {
|
|
"values": ["UP", "DOWN", "STOP"],
|
|
"description": "awning over the patio",
|
|
},
|
|
},
|
|
)
|
|
|
|
((status, detail),) = health
|
|
assert status == "degraded"
|
|
assert detail is not None and "retraining for canopy, garden_light" in detail
|
|
|
|
fake_wren.answer = [fake_wren.Call("canopy", "DOWN")]
|
|
out = node.execute({"house.command": "close the awning"})
|
|
# The model is not shown the targets it never learnt.
|
|
assert [t.name for t in fake_wren.targets[0]] == ["living_room_heating"]
|
|
assert out is not None
|
|
assert [a["applied"] for a in out["house.actions"]] == [False]
|
|
assert out["house.unhandled"] == "close the awning"
|
|
|
|
|
|
def test_a_read_is_answered_from_the_target_value(fake_wren):
|
|
node = _node(
|
|
fake_wren,
|
|
inputs=[OUTDOOR],
|
|
saying=True,
|
|
# Not applied: a reading is answered anyway, as it writes nothing.
|
|
targets={"outdoor_temperature": {"unit": "°C"}},
|
|
)
|
|
fake_wren.answer = [fake_wren.Call("outdoor_temperature")]
|
|
command = "how warm is it outside"
|
|
|
|
out = node.execute({"house.command": command, "house.outdoor": 6.2})
|
|
assert out == {
|
|
"house.reply": "Outdoor temperature is 6.2 °C.",
|
|
"house.actions": [
|
|
_record("outdoor", 6.2, "read(outdoor_temperature)"),
|
|
],
|
|
}
|
|
|
|
# Nothing has written the reading yet: there is no answer to give.
|
|
out = node.execute({"house.command": command})
|
|
assert out is not None
|
|
assert out["house.unhandled"] == command
|
|
|
|
|
|
def test_a_read_beside_a_set_not_written_goes_on_whole(fake_wren):
|
|
node = _node(
|
|
fake_wren, ("kitchen_light", DType.BOOL), inputs=[OUTDOOR], saying=True
|
|
)
|
|
fake_wren.answer = [
|
|
fake_wren.Call("outdoor_temperature"),
|
|
fake_wren.Call("kitchen_light", True),
|
|
]
|
|
command = "how warm is it outside and turn on the kitchen light"
|
|
|
|
out = node.execute({"house.command": command, "house.outdoor": 6.2})
|
|
|
|
assert out is not None
|
|
assert [a["applied"] for a in out["house.actions"]] == [False, False]
|
|
assert out["house.unhandled"] == command
|
|
assert "house.reply" not in out or out["house.reply"] == CANT
|
|
|
|
|
|
def test_a_delayed_call_is_held_by_the_work_queue(fake_wren, monkeypatch):
|
|
node = _node(fake_wren, ("kitchen_light", DType.BOOL), saying=True, apply=True)
|
|
deferred: list[tuple[dict[str, Any], float]] = []
|
|
|
|
def defer(self, node, outputs, seconds):
|
|
deferred.append((outputs, seconds))
|
|
return True
|
|
|
|
monkeypatch.setattr(Pipeline, "defer", defer)
|
|
Pipeline([node])
|
|
fake_wren.answer = [
|
|
fake_wren.Call("kitchen_light", False),
|
|
fake_wren.Call("kitchen_light", True, delay_s=420),
|
|
]
|
|
|
|
out = node.execute({"house.command": "kitchen light off for 7 minutes"})
|
|
|
|
assert deferred == [({"house.kitchen_light": True}, 420)]
|
|
assert out is not None
|
|
assert out["house.kitchen_light"] is False
|
|
assert out["house.reply"] == "Kitchen light off, kitchen light on in 7 minutes."
|
|
|
|
|
|
def test_a_trained_model_is_read_from_the_artifact_store(fake_wren, tmp_path):
|
|
store = ArtifactStore(tmp_path)
|
|
digest = store.put([b"WREN"])["digest"]
|
|
app = SimpleNamespace(state=SimpleNamespace(artifact_store=store))
|
|
health: list[tuple[str, str | None]] = []
|
|
|
|
_node(
|
|
fake_wren, ("kitchen_light", DType.BOOL), app=app, health=health, model=digest
|
|
)
|
|
assert fake_wren.loaded == [store.path(digest)]
|
|
assert health == [("ok", None)]
|
|
|
|
# One the store does not have: every command goes on.
|
|
node = _node(
|
|
fake_wren,
|
|
("kitchen_light", DType.BOOL),
|
|
saying=True,
|
|
app=app,
|
|
health=health,
|
|
apply=True,
|
|
model="sha256:" + "0" * 64,
|
|
)
|
|
assert health[-1][0] == "degraded"
|
|
out = node.execute({"house.command": "kitchen light on"})
|
|
assert out == {
|
|
"house.actions": [],
|
|
"house.unhandled": "kitchen light on",
|
|
"house.reply": CANT,
|
|
}
|
|
assert fake_wren.commands == []
|
|
|
|
|
|
def test_the_model_runs_on_one_blas_thread(fake_wren, monkeypatch):
|
|
"""About twice as fast for a model this small; numpy is loaded by `mlp`."""
|
|
from threadpoolctl import threadpool_info
|
|
|
|
threads: list[int] = []
|
|
act = fake_wren.act
|
|
|
|
def counted(command: str, targets: Any, *, model: Any = None) -> Any:
|
|
threads.extend(
|
|
pool["num_threads"]
|
|
for pool in threadpool_info()
|
|
if pool["user_api"] == "blas"
|
|
)
|
|
return act(command, targets, model=model)
|
|
|
|
monkeypatch.setattr(fake_wren, "act", counted)
|
|
node = _node(fake_wren, ("kitchen_light", DType.BOOL))
|
|
node.execute({"house.command": "lights on"})
|
|
|
|
assert threads
|
|
assert set(threads) == {1}
|
|
|
|
|
|
def test_own_words_and_number_words_reach_the_model_as_its_own(fake_wren):
|
|
node = _node(
|
|
fake_wren,
|
|
("garden_light", DType.BOOL),
|
|
saying=True,
|
|
apply=True,
|
|
aliases={"Page Too": "garden"},
|
|
)
|
|
|
|
out = node.execute({"house.command": "Turn on the page too light in five minutes."})
|
|
|
|
assert fake_wren.commands == ["turn on the garden light in 5 minutes"]
|
|
# The assistant reads the own words too, as they were said otherwise.
|
|
passed = "Turn on the garden light in five minutes."
|
|
assert out == {
|
|
"house.actions": [],
|
|
"house.unhandled": passed,
|
|
"house.reply": CANT,
|
|
}
|
|
|
|
# An assistant reading `unhandled` answers instead.
|
|
unhandled = MessageSpec(name="house.unhandled", port="unhandled", dtype=DType.STR)
|
|
Pipeline([node, Node(lambda params, unhandled: None, requires=[unhandled])])
|
|
out = node.execute({"house.command": "Turn on the page too light in five minutes."})
|
|
assert out == {"house.actions": [], "house.unhandled": passed}
|