wren node: calibration setting, passed to wren's training as the installation's calibration records

backend/fluksio/flow/nodes/wren.py

@@ -269,6 +269,12 @@ class WrenNode(Node):
             'as {"state": {"user_location": "kitchen"}, "command": "lights on", '
             '"calls": [["kitchen_light", true]]}.',
         )
+        calibration: list[dict[str, Any]] = Field(
+            default_factory=list,
+            description="Training only: commands in your own words, like "
+            "examples, that a new model's threshold is chosen on. It never "
+            "trains on them.",
+        )
         tests: list[dict[str, Any]] = Field(
             default_factory=list,
             description="Training only: commands a new model is scored on, like "
@@ -355,10 +361,11 @@ class WrenNode(Node):
 
     def installation(self) -> dict[str, Any]:
         """What wren's training flow learns from: the targets, each with a
-        value, and the ``examples`` and ``tests``."""
+        value, and the ``examples``, ``calibration`` and ``tests``."""
         return {
             "targets": [t | {"value": _unset(t)} for t in self.targets.values()],
             "examples": self.cfg.examples,
+            "calibration": self.cfg.calibration,
             "tests": self.cfg.tests,
         }
 

backend/tests/api/routes/test_wren_train.py

@@ -20,6 +20,7 @@ from tests.flow.conftest import FakeWren
 FLOW = "wren_house"
 PREFIX = f"{settings.API_V1_STR}/flows/{FLOW}/nodes"
 EXAMPLES = [{"command": "warmer please", "calls": [["living_room_heating", 22]]}]
+CALIBRATION = [{"command": "make it toasty", "calls": [["living_room_heating", 24]]}]
 TESTS = [
     {"command": "is it cold outside", "calls": [["outdoor_temperature"]]},
     {"command": "good night", "calls": []},
@@ -49,6 +50,7 @@ def house(
                     "outdoor_temperature": {"description": "temperature outside"},
                 },
                 "examples": EXAMPLES,
+                "calibration": CALIBRATION,
                 "tests": TESTS,
             },
             "requires": [
@@ -121,7 +123,11 @@ def test_train_submits_the_node_as_wren_s_installation(
             "value": 0.0,
         },
     ]
-    assert (installation["examples"], installation["tests"]) == (EXAMPLES, TESTS)
+    assert (
+        installation["examples"],
+        installation["calibration"],
+        installation["tests"],
+    ) == (EXAMPLES, CALIBRATION, TESTS)
 
 
 def test_train_says_how_to_register_a_missing_training_flow(

docs/reference/node-types.md

@@ -379,6 +379,7 @@ name of its own, in the node panel's port name field or as
 | `targets` | empty | by target port, what the model is told of it and how far it is trusted (below) |
 | `aliases` | empty | map your own names to the model's target names, such as `{"desk lamp": "office light"}` |
 | `examples` | empty | for training only, commands to learn beyond the ones wren generates (below) |
+| `calibration` | empty | for training only, commands in your own words a new model's threshold is chosen on (below) |
 | `tests` | empty | for training only, commands a new model is scored on (below) |
 
 Every field of a `targets` entry is optional:
@@ -432,9 +433,11 @@ the store with `PUT /artifacts` (see the [API](../code/api.md#artifacts)).
 
 **Train** in the node panel trains a new model for the node as a run of wren's
 training flow, `wren_train`. It learns the targets wired to the node, as the
-panel shows them with unpublished edits included, and its `examples`, and it
-scores the new model on its `tests`. Both are lists in wren's installation
-format, with targets named by port:
+panel shows them with unpublished edits included, and its `examples`. It
+chooses the new model's confidence threshold on generated commands and on the
+node's `calibration`, which it never trains on, and scores the model on its
+`tests`. All three are lists in wren's installation format, with targets named
+by port, and no command may be in two of them:
 
 ```json
 {"state": {"user_location": "kitchen"}, "command": "lights on please",
@@ -443,7 +446,7 @@ format, with targets named by port:
 
 A call is `[target, value]`, `[target, value, delay_s]` or, for a read,
 `[target]`. A test with `"calls": []` is a command the model must pass on. The
-node never reads either setting when it runs, and wren refuses an example its
+node never reads these settings when it runs, and wren refuses an example its
 targets cannot answer before training starts. The panel follows the run and
 shows the model's exact match, coverage and threshold, and its exact match on
 the tests, once it ends. **Use this model** sets the new model as `model` in