wren node: calibration setting, passed to wren's training as the installation's calibration records
This commit is contained in:
@@ -269,6 +269,12 @@ class WrenNode(Node):
|
|||||||
'as {"state": {"user_location": "kitchen"}, "command": "lights on", '
|
'as {"state": {"user_location": "kitchen"}, "command": "lights on", '
|
||||||
'"calls": [["kitchen_light", true]]}.',
|
'"calls": [["kitchen_light", true]]}.',
|
||||||
)
|
)
|
||||||
|
calibration: list[dict[str, Any]] = Field(
|
||||||
|
default_factory=list,
|
||||||
|
description="Training only: commands in your own words, like "
|
||||||
|
"examples, that a new model's threshold is chosen on. It never "
|
||||||
|
"trains on them.",
|
||||||
|
)
|
||||||
tests: list[dict[str, Any]] = Field(
|
tests: list[dict[str, Any]] = Field(
|
||||||
default_factory=list,
|
default_factory=list,
|
||||||
description="Training only: commands a new model is scored on, like "
|
description="Training only: commands a new model is scored on, like "
|
||||||
@@ -355,10 +361,11 @@ class WrenNode(Node):
|
|||||||
|
|
||||||
def installation(self) -> dict[str, Any]:
|
def installation(self) -> dict[str, Any]:
|
||||||
"""What wren's training flow learns from: the targets, each with a
|
"""What wren's training flow learns from: the targets, each with a
|
||||||
value, and the ``examples`` and ``tests``."""
|
value, and the ``examples``, ``calibration`` and ``tests``."""
|
||||||
return {
|
return {
|
||||||
"targets": [t | {"value": _unset(t)} for t in self.targets.values()],
|
"targets": [t | {"value": _unset(t)} for t in self.targets.values()],
|
||||||
"examples": self.cfg.examples,
|
"examples": self.cfg.examples,
|
||||||
|
"calibration": self.cfg.calibration,
|
||||||
"tests": self.cfg.tests,
|
"tests": self.cfg.tests,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from tests.flow.conftest import FakeWren
|
|||||||
FLOW = "wren_house"
|
FLOW = "wren_house"
|
||||||
PREFIX = f"{settings.API_V1_STR}/flows/{FLOW}/nodes"
|
PREFIX = f"{settings.API_V1_STR}/flows/{FLOW}/nodes"
|
||||||
EXAMPLES = [{"command": "warmer please", "calls": [["living_room_heating", 22]]}]
|
EXAMPLES = [{"command": "warmer please", "calls": [["living_room_heating", 22]]}]
|
||||||
|
CALIBRATION = [{"command": "make it toasty", "calls": [["living_room_heating", 24]]}]
|
||||||
TESTS = [
|
TESTS = [
|
||||||
{"command": "is it cold outside", "calls": [["outdoor_temperature"]]},
|
{"command": "is it cold outside", "calls": [["outdoor_temperature"]]},
|
||||||
{"command": "good night", "calls": []},
|
{"command": "good night", "calls": []},
|
||||||
@@ -49,6 +50,7 @@ def house(
|
|||||||
"outdoor_temperature": {"description": "temperature outside"},
|
"outdoor_temperature": {"description": "temperature outside"},
|
||||||
},
|
},
|
||||||
"examples": EXAMPLES,
|
"examples": EXAMPLES,
|
||||||
|
"calibration": CALIBRATION,
|
||||||
"tests": TESTS,
|
"tests": TESTS,
|
||||||
},
|
},
|
||||||
"requires": [
|
"requires": [
|
||||||
@@ -121,7 +123,11 @@ def test_train_submits_the_node_as_wren_s_installation(
|
|||||||
"value": 0.0,
|
"value": 0.0,
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
assert (installation["examples"], installation["tests"]) == (EXAMPLES, TESTS)
|
assert (
|
||||||
|
installation["examples"],
|
||||||
|
installation["calibration"],
|
||||||
|
installation["tests"],
|
||||||
|
) == (EXAMPLES, CALIBRATION, TESTS)
|
||||||
|
|
||||||
|
|
||||||
def test_train_says_how_to_register_a_missing_training_flow(
|
def test_train_says_how_to_register_a_missing_training_flow(
|
||||||
|
|||||||
@@ -379,6 +379,7 @@ name of its own, in the node panel's port name field or as
|
|||||||
| `targets` | empty | by target port, what the model is told of it and how far it is trusted (below) |
|
| `targets` | empty | by target port, what the model is told of it and how far it is trusted (below) |
|
||||||
| `aliases` | empty | map your own names to the model's target names, such as `{"desk lamp": "office light"}` |
|
| `aliases` | empty | map your own names to the model's target names, such as `{"desk lamp": "office light"}` |
|
||||||
| `examples` | empty | for training only, commands to learn beyond the ones wren generates (below) |
|
| `examples` | empty | for training only, commands to learn beyond the ones wren generates (below) |
|
||||||
|
| `calibration` | empty | for training only, commands in your own words a new model's threshold is chosen on (below) |
|
||||||
| `tests` | empty | for training only, commands a new model is scored on (below) |
|
| `tests` | empty | for training only, commands a new model is scored on (below) |
|
||||||
|
|
||||||
Every field of a `targets` entry is optional:
|
Every field of a `targets` entry is optional:
|
||||||
@@ -432,9 +433,11 @@ the store with `PUT /artifacts` (see the [API](../code/api.md#artifacts)).
|
|||||||
|
|
||||||
**Train** in the node panel trains a new model for the node as a run of wren's
|
**Train** in the node panel trains a new model for the node as a run of wren's
|
||||||
training flow, `wren_train`. It learns the targets wired to the node, as the
|
training flow, `wren_train`. It learns the targets wired to the node, as the
|
||||||
panel shows them with unpublished edits included, and its `examples`, and it
|
panel shows them with unpublished edits included, and its `examples`. It
|
||||||
scores the new model on its `tests`. Both are lists in wren's installation
|
chooses the new model's confidence threshold on generated commands and on the
|
||||||
format, with targets named by port:
|
node's `calibration`, which it never trains on, and scores the model on its
|
||||||
|
`tests`. All three are lists in wren's installation format, with targets named
|
||||||
|
by port, and no command may be in two of them:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{"state": {"user_location": "kitchen"}, "command": "lights on please",
|
{"state": {"user_location": "kitchen"}, "command": "lights on please",
|
||||||
@@ -443,7 +446,7 @@ format, with targets named by port:
|
|||||||
|
|
||||||
A call is `[target, value]`, `[target, value, delay_s]` or, for a read,
|
A call is `[target, value]`, `[target, value, delay_s]` or, for a read,
|
||||||
`[target]`. A test with `"calls": []` is a command the model must pass on. The
|
`[target]`. A test with `"calls": []` is a command the model must pass on. The
|
||||||
node never reads either setting when it runs, and wren refuses an example its
|
node never reads these settings when it runs, and wren refuses an example its
|
||||||
targets cannot answer before training starts. The panel follows the run and
|
targets cannot answer before training starts. The panel follows the run and
|
||||||
shows the model's exact match, coverage and threshold, and its exact match on
|
shows the model's exact match, coverage and threshold, and its exact match on
|
||||||
the tests, once it ends. **Use this model** sets the new model as `model` in
|
the tests, once it ends. **Use this model** sets the new model as `model` in
|
||||||
|
|||||||
Reference in New Issue
Block a user