wren node: Train in the panel, as a run of wren's training flow
POST/GET /flows/{name}/nodes/{id}/train submit wren_train on the node's
targets, examples and tests and read its latest run (run.for_node,
cause "train"); "Use this model" writes `model` into the draft.
This commit is contained in:
@@ -0,0 +1,33 @@
|
|||||||
|
"""run.for_node
|
||||||
|
|
||||||
|
A wren node's Train button submits wren's training flow as a run. The node
|
||||||
|
it was pressed on is what ties that run back to the node's panel.
|
||||||
|
|
||||||
|
Revision ID: a3c7e5d91f42
|
||||||
|
Revises: ccb21eacf5d5
|
||||||
|
Create Date: 2026-10-24
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
import sqlmodel.sql.sqltypes
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = "a3c7e5d91f42"
|
||||||
|
down_revision = "ccb21eacf5d5"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.add_column(
|
||||||
|
"run",
|
||||||
|
sa.Column(
|
||||||
|
"for_node", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.drop_column("run", "for_node")
|
||||||
@@ -57,6 +57,7 @@ from fluksio.flow.devices import (
|
|||||||
)
|
)
|
||||||
from fluksio.flow.events import event_bus
|
from fluksio.flow.events import event_bus
|
||||||
from fluksio.flow.messages import MEDIA_FAMILIES, DType, flow_of, qualify
|
from fluksio.flow.messages import MEDIA_FAMILIES, DType, flow_of, qualify
|
||||||
|
from fluksio.flow.nodes.wren import TRAIN_FLOW, WrenNode, pack_meta
|
||||||
from fluksio.flow.pipeline import ValidationIssue
|
from fluksio.flow.pipeline import ValidationIssue
|
||||||
from fluksio.flow.runs import RunRejected
|
from fluksio.flow.runs import RunRejected
|
||||||
from fluksio.flow.schemas import (
|
from fluksio.flow.schemas import (
|
||||||
@@ -84,7 +85,7 @@ from fluksio.flow.store import (
|
|||||||
LibNotFound,
|
LibNotFound,
|
||||||
StaleVersion,
|
StaleVersion,
|
||||||
)
|
)
|
||||||
from fluksio.models import Flavor, Message, Run
|
from fluksio.models import Flavor, Message, Run, RunArtifact, RunNode
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -159,6 +160,24 @@ class TriggerRequest(BaseModel):
|
|||||||
values: dict[str, Any] = {}
|
values: dict[str, Any] = {}
|
||||||
|
|
||||||
|
|
||||||
|
class Training(BaseModel):
|
||||||
|
"""A wren node's latest training run, as its panel shows it."""
|
||||||
|
|
||||||
|
id: str
|
||||||
|
#: queued, running, ok, error, cancelled or abandoned.
|
||||||
|
status: str
|
||||||
|
#: What went wrong, when it failed.
|
||||||
|
error: str = ""
|
||||||
|
#: final_loss, n_params, exact_match, threshold and coverage, once it ended.
|
||||||
|
result: dict[str, Any] = {}
|
||||||
|
#: The trained model's digest, for the node's ``model`` setting.
|
||||||
|
model: str = ""
|
||||||
|
#: From the model's own record: the confidence it recommends, and its
|
||||||
|
#: score on the node's ``tests`` when there were any.
|
||||||
|
threshold: float | None = None
|
||||||
|
test_exact_match: float | None = None
|
||||||
|
|
||||||
|
|
||||||
def _endpoints(controller: FlowController, definition: FlowDef) -> list[Endpoint]:
|
def _endpoints(controller: FlowController, definition: FlowDef) -> list[Endpoint]:
|
||||||
"""Everything wired into this flow from outside it."""
|
"""Everything wired into this flow from outside it."""
|
||||||
flow = definition.name
|
flow = definition.name
|
||||||
@@ -1010,6 +1029,111 @@ def acknowledge_node_error(
|
|||||||
return Message(message=f"Cleared the failure on '{node_id}'")
|
return Message(message=f"Cleared the failure on '{node_id}'")
|
||||||
|
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
# Training a wren node
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _node_def(controller: FlowController, name: str, node_id: str) -> NodeDef:
|
||||||
|
"""A node as the editor shows it: from the draft when there is one."""
|
||||||
|
node = next(
|
||||||
|
(n for n in _read_flow(controller, name).nodes if n.id == node_id), None
|
||||||
|
)
|
||||||
|
if node is None:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
|
||||||
|
)
|
||||||
|
return node
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{name}/nodes/{node_id}/train", response_model=Training, status_code=202)
|
||||||
|
async def train_node(
|
||||||
|
name: str,
|
||||||
|
node_id: str,
|
||||||
|
request: Request,
|
||||||
|
controller: FlowControllerDep,
|
||||||
|
user: CurrentUser,
|
||||||
|
) -> Any:
|
||||||
|
"""Train a new model for a wren node, as a run of wren's training flow.
|
||||||
|
|
||||||
|
It learns the node's targets, its ``examples`` and its ``tests``, as the
|
||||||
|
editor shows them. The node keeps running its model until the new one is
|
||||||
|
set as its ``model``.
|
||||||
|
"""
|
||||||
|
node = _node_def(controller, name, node_id)
|
||||||
|
if node.type != "wren":
|
||||||
|
raise HTTPException(status_code=400, detail=f"'{node_id}' is not a wren node")
|
||||||
|
try:
|
||||||
|
wren = WrenNode(
|
||||||
|
requires=[spec for spec in node.requires if spec.name],
|
||||||
|
provides=[spec for spec in node.provides if spec.name],
|
||||||
|
params=node.params,
|
||||||
|
)
|
||||||
|
except (ValueError, RuntimeError) as exc:
|
||||||
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||||
|
service = getattr(request.app.state, "run_service", None)
|
||||||
|
if service is None:
|
||||||
|
raise HTTPException(status_code=503, detail="Runs are not available")
|
||||||
|
try:
|
||||||
|
run = await run_in_threadpool(
|
||||||
|
service.submit,
|
||||||
|
TRAIN_FLOW,
|
||||||
|
params={"installation": orjson.dumps(wren.installation()).decode()},
|
||||||
|
cause="train",
|
||||||
|
actor=user.email,
|
||||||
|
no_cache=True,
|
||||||
|
for_node=f"{name}.{node_id}",
|
||||||
|
)
|
||||||
|
except FlowNotFound:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail=(
|
||||||
|
f"wren's training flow '{TRAIN_FLOW}' is not registered. Run "
|
||||||
|
"`fluksio sync wren.flow` from an environment with wren[train]."
|
||||||
|
),
|
||||||
|
) from None
|
||||||
|
except RunRejected as exc:
|
||||||
|
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||||
|
return Training(id=run.id, status=run.status)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{name}/nodes/{node_id}/train", response_model=Training | None)
|
||||||
|
def read_training(
|
||||||
|
name: str,
|
||||||
|
node_id: str,
|
||||||
|
request: Request,
|
||||||
|
controller: FlowControllerDep,
|
||||||
|
session: SessionDep,
|
||||||
|
) -> Any:
|
||||||
|
"""A wren node's latest training run, or null when it has none."""
|
||||||
|
_node_def(controller, name, node_id)
|
||||||
|
run = session.exec(
|
||||||
|
select(Run)
|
||||||
|
.where(col(Run.flow) == TRAIN_FLOW, col(Run.for_node) == f"{name}.{node_id}")
|
||||||
|
.order_by(col(Run.created_at).desc())
|
||||||
|
).first()
|
||||||
|
if run is None:
|
||||||
|
return None
|
||||||
|
training = Training(id=run.id, status=run.status, result=run.result)
|
||||||
|
if run.status == "error":
|
||||||
|
failed = session.exec(
|
||||||
|
select(RunNode.error).where(
|
||||||
|
col(RunNode.run_id) == run.id, col(RunNode.error) != ""
|
||||||
|
)
|
||||||
|
).first()
|
||||||
|
training.error = failed or run.status_reason
|
||||||
|
# The model, once the run made one the store still has.
|
||||||
|
pack = session.get(RunArtifact, {"run_id": run.id, "name": f"{TRAIN_FLOW}.pack"})
|
||||||
|
store = getattr(request.app.state, "artifact_store", None)
|
||||||
|
path = store.path(pack.digest) if pack is not None and store else None
|
||||||
|
if pack is not None and path is not None:
|
||||||
|
meta = pack_meta(path)
|
||||||
|
training.model = pack.digest
|
||||||
|
training.threshold = meta.get("threshold")
|
||||||
|
training.test_exact_match = meta.get("test_exact_match")
|
||||||
|
return training
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{name}/state", response_model=FlowStatePublic)
|
@router.get("/{name}/state", response_model=FlowStatePublic)
|
||||||
def read_flow_state(name: str, controller: FlowControllerDep, hidden: HiddenDep) -> Any:
|
def read_flow_state(name: str, controller: FlowControllerDep, hidden: HiddenDep) -> Any:
|
||||||
"""The last value seen on every message of this flow."""
|
"""The last value seen on every message of this flow."""
|
||||||
|
|||||||
@@ -46,6 +46,13 @@ def _wren() -> Any:
|
|||||||
return wren
|
return wren
|
||||||
|
|
||||||
|
|
||||||
|
def pack_meta(path: Any) -> dict[str, Any]:
|
||||||
|
"""What a model records of its training: its targets, threshold and,
|
||||||
|
when it had tests, ``test_exact_match``."""
|
||||||
|
found: dict[str, Any] = _wren().load_pack(path)["meta"]
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
#: The input the command arrives on, and the only one that wakes the node.
|
#: The input the command arrives on, and the only one that wakes the node.
|
||||||
PROMPT = "prompt"
|
PROMPT = "prompt"
|
||||||
#: The output the calls are reported on, and the two that say what came of
|
#: The output the calls are reported on, and the two that say what came of
|
||||||
@@ -55,6 +62,8 @@ REPLY = "reply"
|
|||||||
UNHANDLED = "unhandled"
|
UNHANDLED = "unhandled"
|
||||||
#: The reply when nothing was done and nothing reads ``unhandled``.
|
#: The reply when nothing was done and nothing reads ``unhandled``.
|
||||||
CANT = "Sorry, I can't do that yet."
|
CANT = "Sorry, I can't do that yet."
|
||||||
|
#: wren's training flow, which wren registers: ``fluksio sync wren.flow``.
|
||||||
|
TRAIN_FLOW = "wren_train"
|
||||||
|
|
||||||
_TARGET_TYPES = {DType.BOOL, DType.INT, DType.FLOAT, DType.STR}
|
_TARGET_TYPES = {DType.BOOL, DType.INT, DType.FLOAT, DType.STR}
|
||||||
|
|
||||||
@@ -260,6 +269,17 @@ class WrenNode(Node):
|
|||||||
description="Your own words, and words heard wrong, in the model's: "
|
description="Your own words, and words heard wrong, in the model's: "
|
||||||
'{"traverse": "living room light", "page too": "patio"}.',
|
'{"traverse": "living room light", "page too": "patio"}.',
|
||||||
)
|
)
|
||||||
|
examples: list[dict[str, Any]] = Field(
|
||||||
|
default_factory=list,
|
||||||
|
description="Training only: commands to learn, targets named by port, "
|
||||||
|
'as {"state": {"user_location": "kitchen"}, "command": "lights on", '
|
||||||
|
'"calls": [["kitchen_light", true]]}.',
|
||||||
|
)
|
||||||
|
tests: list[dict[str, Any]] = Field(
|
||||||
|
default_factory=list,
|
||||||
|
description="Training only: commands a new model is scored on, like "
|
||||||
|
'examples. "calls": [] is one it must pass on.',
|
||||||
|
)
|
||||||
|
|
||||||
__slots__ = ("cfg", "targets", "pack", "untrained")
|
__slots__ = ("cfg", "targets", "pack", "untrained")
|
||||||
|
|
||||||
@@ -339,6 +359,15 @@ class WrenNode(Node):
|
|||||||
name=name or "wren",
|
name=name or "wren",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def installation(self) -> dict[str, Any]:
|
||||||
|
"""What wren's training flow learns from: the targets, each with a
|
||||||
|
value, and the ``examples`` and ``tests``."""
|
||||||
|
return {
|
||||||
|
"targets": [t | {"value": _unset(t)} for t in self.targets.values()],
|
||||||
|
"examples": self.cfg.examples,
|
||||||
|
"tests": self.cfg.tests,
|
||||||
|
}
|
||||||
|
|
||||||
async def start(self, app: FastAPI | None = None) -> None:
|
async def start(self, app: FastAPI | None = None) -> None:
|
||||||
store = getattr(app.state, "artifact_store", None) if app else None
|
store = getattr(app.state, "artifact_store", None) if app else None
|
||||||
self.pack = self._load(store)
|
self.pack = self._load(store)
|
||||||
|
|||||||
@@ -951,6 +951,7 @@ class RunService:
|
|||||||
idempotency_key: str | None = None,
|
idempotency_key: str | None = None,
|
||||||
parent_id: str | None = None,
|
parent_id: str | None = None,
|
||||||
passed: list[str] | None = None,
|
passed: list[str] | None = None,
|
||||||
|
for_node: str | None = None,
|
||||||
) -> Run:
|
) -> Run:
|
||||||
"""Journal a run and wake an engine up for it. Never blocks on it.
|
"""Journal a run and wake an engine up for it. Never blocks on it.
|
||||||
|
|
||||||
@@ -993,6 +994,7 @@ class RunService:
|
|||||||
actor=actor,
|
actor=actor,
|
||||||
idempotency_key=idempotency_key,
|
idempotency_key=idempotency_key,
|
||||||
parent_id=parent_id,
|
parent_id=parent_id,
|
||||||
|
for_node=for_node,
|
||||||
)
|
)
|
||||||
with Session(db_engine) as session:
|
with Session(db_engine) as session:
|
||||||
session.add(run)
|
session.add(run)
|
||||||
@@ -1249,6 +1251,7 @@ class RunService:
|
|||||||
# What its parent's caller chose. A parent from before that was
|
# What its parent's caller chose. A parent from before that was
|
||||||
# recorded falls back to the names its params carry.
|
# recorded falls back to the names its params carry.
|
||||||
passed=source.passed,
|
passed=source.passed,
|
||||||
|
for_node=source.for_node,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _flow_of(self, run_id: str) -> str | None:
|
def _flow_of(self, run_id: str) -> str | None:
|
||||||
|
|||||||
@@ -388,8 +388,11 @@ class Run(SQLModel, table=True):
|
|||||||
external_id: str | None = Field(
|
external_id: str | None = Field(
|
||||||
default=None, unique=True, index=True, max_length=255
|
default=None, unique=True, index=True, max_length=255
|
||||||
)
|
)
|
||||||
#: Where it was asked for: api, cli, sdk, hook, sweep or import.
|
#: Where it was asked for: api, cli, sdk, hook, sweep, import or train.
|
||||||
cause: str = Field(default="api", max_length=32)
|
cause: str = Field(default="api", max_length=32)
|
||||||
|
#: The node it was asked for from, as `flow.node`: the wren node a
|
||||||
|
#: training run is for. Null for every other run.
|
||||||
|
for_node: str | None = Field(default=None, max_length=255)
|
||||||
#: Re-execute every node, whatever the stage cache holds for it.
|
#: Re-execute every node, whatever the stage cache holds for it.
|
||||||
no_cache: bool = False
|
no_cache: bool = False
|
||||||
#: Run the unpublished working copy rather than what is published. Read
|
#: Run the unpublished working copy rather than what is published. Read
|
||||||
|
|||||||
@@ -0,0 +1,186 @@
|
|||||||
|
"""Train on a wren node: its wiring and settings go to wren's training flow.
|
||||||
|
|
||||||
|
The model package is optional, so ``FakeWren`` (tests/flow/conftest.py)
|
||||||
|
stands in for it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from collections.abc import Generator
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from sqlmodel import Session
|
||||||
|
|
||||||
|
from fluksio.core.config import settings
|
||||||
|
from fluksio.core.db import engine as db_engine
|
||||||
|
from fluksio.models import Run, RunArtifact
|
||||||
|
from tests.flow.conftest import FakeWren
|
||||||
|
|
||||||
|
FLOW = "wren_house"
|
||||||
|
PREFIX = f"{settings.API_V1_STR}/flows/{FLOW}/nodes"
|
||||||
|
EXAMPLES = [{"command": "warmer please", "calls": [["living_room_heating", 22]]}]
|
||||||
|
TESTS = [
|
||||||
|
{"command": "is it cold outside", "calls": [["outdoor_temperature"]]},
|
||||||
|
{"command": "good night", "calls": []},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def fake_wren(monkeypatch: pytest.MonkeyPatch) -> FakeWren:
|
||||||
|
fake = FakeWren()
|
||||||
|
monkeypatch.setattr("fluksio.flow.nodes.wren._wren", lambda: fake)
|
||||||
|
return fake
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
|
def house(
|
||||||
|
client: TestClient, superuser_token_headers: dict[str, str]
|
||||||
|
) -> Generator[None, None, None]:
|
||||||
|
"""Two wren nodes, so a run for one is not the other's, and a function."""
|
||||||
|
prompt = {"name": "command", "dtype": "str", "port": "prompt"}
|
||||||
|
nodes = [
|
||||||
|
{
|
||||||
|
"id": "control",
|
||||||
|
"type": "wren",
|
||||||
|
"params": {
|
||||||
|
"targets": {
|
||||||
|
"living_room_heating": {"range": [5, 30], "unit": "°C"},
|
||||||
|
"outdoor_temperature": {"description": "temperature outside"},
|
||||||
|
},
|
||||||
|
"examples": EXAMPLES,
|
||||||
|
"tests": TESTS,
|
||||||
|
},
|
||||||
|
"requires": [
|
||||||
|
prompt,
|
||||||
|
{"name": "outdoor", "dtype": "float", "port": "outdoor_temperature"},
|
||||||
|
],
|
||||||
|
"provides": [
|
||||||
|
{"name": "setpoint", "dtype": "float", "port": "living_room_heating"}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "lights",
|
||||||
|
"type": "wren",
|
||||||
|
"requires": [prompt],
|
||||||
|
"provides": [{"name": "kitchen_light", "dtype": "bool"}],
|
||||||
|
},
|
||||||
|
{"id": "sensor", "type": "python"},
|
||||||
|
]
|
||||||
|
url = f"{settings.API_V1_STR}/flows/{FLOW}"
|
||||||
|
saved = client.put(
|
||||||
|
url, headers=superuser_token_headers, json={"name": FLOW, "nodes": nodes}
|
||||||
|
)
|
||||||
|
assert saved.status_code == 200, saved.text
|
||||||
|
yield
|
||||||
|
client.delete(url, headers=superuser_token_headers)
|
||||||
|
|
||||||
|
|
||||||
|
def test_train_submits_the_node_as_wren_s_installation(
|
||||||
|
client, superuser_token_headers, monkeypatch
|
||||||
|
):
|
||||||
|
"""The draft's ports, targets, examples and tests, never from the cache."""
|
||||||
|
seen: dict[str, object] = {}
|
||||||
|
|
||||||
|
class Recorder:
|
||||||
|
def submit(self, name, **kwargs):
|
||||||
|
seen.update(kwargs, flow=name)
|
||||||
|
return Run(id="train-1", flow=name, created_at=datetime.now(UTC))
|
||||||
|
|
||||||
|
monkeypatch.setattr(client.app.state, "run_service", Recorder())
|
||||||
|
answer = client.post(f"{PREFIX}/control/train", headers=superuser_token_headers)
|
||||||
|
|
||||||
|
assert answer.status_code == 202, answer.text
|
||||||
|
assert answer.json()["id"] == "train-1"
|
||||||
|
assert (seen["flow"], seen["no_cache"], seen["cause"], seen["for_node"]) == (
|
||||||
|
"wren_train",
|
||||||
|
True,
|
||||||
|
"train",
|
||||||
|
f"{FLOW}.control",
|
||||||
|
)
|
||||||
|
installation = json.loads(seen["params"]["installation"]) # type: ignore[index]
|
||||||
|
assert installation["targets"] == [
|
||||||
|
{
|
||||||
|
"name": "living_room_heating",
|
||||||
|
"type": "float",
|
||||||
|
"values": None,
|
||||||
|
"range": [5, 30],
|
||||||
|
"description": "",
|
||||||
|
"unit": "°C",
|
||||||
|
"readonly": False,
|
||||||
|
"value": 5.0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "outdoor_temperature",
|
||||||
|
"type": "float",
|
||||||
|
"values": None,
|
||||||
|
"range": None,
|
||||||
|
"description": "temperature outside",
|
||||||
|
"unit": None,
|
||||||
|
"readonly": True,
|
||||||
|
"value": 0.0,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
assert (installation["examples"], installation["tests"]) == (EXAMPLES, TESTS)
|
||||||
|
|
||||||
|
|
||||||
|
def test_train_says_how_to_register_a_missing_training_flow(
|
||||||
|
client, superuser_token_headers
|
||||||
|
):
|
||||||
|
answer = client.post(f"{PREFIX}/control/train", headers=superuser_token_headers)
|
||||||
|
|
||||||
|
assert answer.status_code == 409
|
||||||
|
assert "fluksio sync wren.flow" in answer.json()["detail"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_only_a_wren_node_trains(client, superuser_token_headers):
|
||||||
|
answer = client.post(f"{PREFIX}/sensor/train", headers=superuser_token_headers)
|
||||||
|
assert answer.status_code == 400
|
||||||
|
missing = client.post(f"{PREFIX}/nobody/train", headers=superuser_token_headers)
|
||||||
|
assert missing.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_node_reads_its_own_latest_training(
|
||||||
|
client, superuser_token_headers, fake_wren
|
||||||
|
):
|
||||||
|
"""With the model's digest and what its record says of it."""
|
||||||
|
pack = client.app.state.artifact_store.put([b"weights"], name="wren.bin")
|
||||||
|
fake_wren.pack = {"meta": {"threshold": 0.93, "test_exact_match": 0.5}}
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
with Session(db_engine) as session:
|
||||||
|
for run_id, node, ago in (
|
||||||
|
("train-old", "control", 3),
|
||||||
|
("train-new", "control", 2),
|
||||||
|
("train-lights", "lights", 1),
|
||||||
|
):
|
||||||
|
session.add(
|
||||||
|
Run(
|
||||||
|
id=run_id,
|
||||||
|
flow="wren_train",
|
||||||
|
status="ok",
|
||||||
|
for_node=f"{FLOW}.{node}",
|
||||||
|
result={"exact_match": 0.98, "coverage": 0.9},
|
||||||
|
created_at=now - timedelta(minutes=ago),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.add(
|
||||||
|
RunArtifact(
|
||||||
|
run_id="train-new", name="wren_train.pack", digest=pack["digest"]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
training = client.get(f"{PREFIX}/control/train", headers=superuser_token_headers)
|
||||||
|
|
||||||
|
assert training.status_code == 200
|
||||||
|
assert training.json() == {
|
||||||
|
"id": "train-new",
|
||||||
|
"status": "ok",
|
||||||
|
"error": "",
|
||||||
|
"result": {"exact_match": 0.98, "coverage": 0.9},
|
||||||
|
"model": pack["digest"],
|
||||||
|
"threshold": 0.93,
|
||||||
|
"test_exact_match": 0.5,
|
||||||
|
}
|
||||||
|
lights = client.get(f"{PREFIX}/lights/train", headers=superuser_token_headers)
|
||||||
|
assert lights.json()["id"] == "train-lights"
|
||||||
@@ -378,6 +378,8 @@ name of its own, in the node panel's port name field or as
|
|||||||
| `min_confidence` | empty | the confidence a call needs; empty takes the model's own threshold |
|
| `min_confidence` | empty | the confidence a call needs; empty takes the model's own threshold |
|
||||||
| `targets` | empty | by target port, what the model is told of it and how far it is trusted (below) |
|
| `targets` | empty | by target port, what the model is told of it and how far it is trusted (below) |
|
||||||
| `aliases` | empty | your own words, or words the recogniser hears wrong, in the model's: `{"traverse": "living room light", "page too": "patio"}` |
|
| `aliases` | empty | your own words, or words the recogniser hears wrong, in the model's: `{"traverse": "living room light", "page too": "patio"}` |
|
||||||
|
| `examples` | empty | for training only, commands to learn beyond the ones wren generates (below) |
|
||||||
|
| `tests` | empty | for training only, commands a new model is scored on (below) |
|
||||||
|
|
||||||
Every field of a `targets` entry is optional:
|
Every field of a `targets` entry is optional:
|
||||||
|
|
||||||
@@ -427,6 +429,34 @@ it runs without that warning only on ports named and described as there. A
|
|||||||
and every command goes on to `unhandled`. A model trained with wren goes into
|
and every command goes on to `unhandled`. A model trained with wren goes into
|
||||||
the store with `PUT /artifacts` (see the [API](../code/api.md#artifacts)).
|
the store with `PUT /artifacts` (see the [API](../code/api.md#artifacts)).
|
||||||
|
|
||||||
|
**Train** in the node panel trains a new model for the node as a run of wren's
|
||||||
|
training flow, `wren_train`. It learns the targets wired to the node, as the
|
||||||
|
panel shows them with unpublished edits included, and its `examples`, and it
|
||||||
|
scores the new model on its `tests`. Both are lists in wren's installation
|
||||||
|
format, with targets named by port:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"state": {"user_location": "kitchen"}, "command": "lights on please",
|
||||||
|
"calls": [["kitchen_light", true]]}
|
||||||
|
```
|
||||||
|
|
||||||
|
A call is `[target, value]`, `[target, value, delay_s]` or, for a read,
|
||||||
|
`[target]`. A test with `"calls": []` is a command the model must pass on. The
|
||||||
|
node never reads either setting when it runs, and wren refuses an example its
|
||||||
|
targets cannot answer before training starts. The panel follows the run and
|
||||||
|
shows the model's exact match, coverage and threshold, and its exact match on
|
||||||
|
the tests, once it ends. **Use this model** sets the new model as `model` in
|
||||||
|
the draft, and publishing puts it live. `POST` and `GET` on
|
||||||
|
`/flows/{name}/nodes/{id}/train` start a training run and read the latest one.
|
||||||
|
|
||||||
|
Training needs wren's `train` extra, which brings JAX, where the run executes:
|
||||||
|
in **Modules** (see [Packages](../code/nodes.md#packages)) or on a worker. List
|
||||||
|
`wren[train] @ https://github.com/stroblme/wren/archive/main.tar.gz` there
|
||||||
|
(an archive, since the engine image has no git). The flow comes from
|
||||||
|
wren and is registered once with `fluksio sync wren.flow`, run from an
|
||||||
|
environment with `wren[train]` and fluksio. Until then Train answers that the
|
||||||
|
flow is missing.
|
||||||
|
|
||||||
The command is read in lower case without sentence marks and with number words
|
The command is read in lower case without sentence marks and with number words
|
||||||
as digits ("in five minutes" as "in 5 minutes"), as the model was trained.
|
as digits ("in five minutes" as "in 5 minutes"), as the model was trained.
|
||||||
`aliases` are applied before, whole words in any case, and `unhandled` carries
|
`aliases` are applied before, whole words in any case, and `unhandled` carries
|
||||||
|
|||||||
@@ -4505,6 +4505,61 @@ export const TokenRequestSchema = {
|
|||||||
title: 'TokenRequest'
|
title: 'TokenRequest'
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
|
export const TrainingSchema = {
|
||||||
|
properties: {
|
||||||
|
id: {
|
||||||
|
type: 'string',
|
||||||
|
title: 'Id'
|
||||||
|
},
|
||||||
|
status: {
|
||||||
|
type: 'string',
|
||||||
|
title: 'Status'
|
||||||
|
},
|
||||||
|
error: {
|
||||||
|
type: 'string',
|
||||||
|
title: 'Error',
|
||||||
|
default: ''
|
||||||
|
},
|
||||||
|
result: {
|
||||||
|
additionalProperties: true,
|
||||||
|
type: 'object',
|
||||||
|
title: 'Result',
|
||||||
|
default: {}
|
||||||
|
},
|
||||||
|
model: {
|
||||||
|
type: 'string',
|
||||||
|
title: 'Model',
|
||||||
|
default: ''
|
||||||
|
},
|
||||||
|
threshold: {
|
||||||
|
anyOf: [
|
||||||
|
{
|
||||||
|
type: 'number'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'null'
|
||||||
|
}
|
||||||
|
],
|
||||||
|
title: 'Threshold'
|
||||||
|
},
|
||||||
|
test_exact_match: {
|
||||||
|
anyOf: [
|
||||||
|
{
|
||||||
|
type: 'number'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'null'
|
||||||
|
}
|
||||||
|
],
|
||||||
|
title: 'Test Exact Match'
|
||||||
|
}
|
||||||
|
},
|
||||||
|
type: 'object',
|
||||||
|
required: ['id', 'status'],
|
||||||
|
title: 'Training',
|
||||||
|
description: "A wren node's latest training run, as its panel shows it."
|
||||||
|
} as const;
|
||||||
|
|
||||||
export const TriggerRequestSchema = {
|
export const TriggerRequestSchema = {
|
||||||
properties: {
|
properties: {
|
||||||
values: {
|
values: {
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -1471,6 +1471,21 @@ export type TokenRequest = {
|
|||||||
labels?: Array<(string)>;
|
labels?: Array<(string)>;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A wren node's latest training run, as its panel shows it.
|
||||||
|
*/
|
||||||
|
export type Training = {
|
||||||
|
id: string;
|
||||||
|
status: string;
|
||||||
|
error?: string;
|
||||||
|
result?: {
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
model?: string;
|
||||||
|
threshold?: (number | null);
|
||||||
|
test_exact_match?: (number | null);
|
||||||
|
};
|
||||||
|
|
||||||
export type TriggerRequest = {
|
export type TriggerRequest = {
|
||||||
values?: {
|
values?: {
|
||||||
[key: string]: unknown;
|
[key: string]: unknown;
|
||||||
@@ -2112,6 +2127,20 @@ export type FlowsAcknowledgeNodeErrorData = {
|
|||||||
|
|
||||||
export type FlowsAcknowledgeNodeErrorResponse = (Message);
|
export type FlowsAcknowledgeNodeErrorResponse = (Message);
|
||||||
|
|
||||||
|
export type FlowsTrainNodeData = {
|
||||||
|
name: string;
|
||||||
|
nodeId: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type FlowsTrainNodeResponse = (Training);
|
||||||
|
|
||||||
|
export type FlowsReadTrainingData = {
|
||||||
|
name: string;
|
||||||
|
nodeId: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type FlowsReadTrainingResponse = ((Training | null));
|
||||||
|
|
||||||
export type FlowsReadFlowStateData = {
|
export type FlowsReadFlowStateData = {
|
||||||
name: string;
|
name: string;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"
|
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"
|
||||||
|
import { Link } from "@tanstack/react-router"
|
||||||
import {
|
import {
|
||||||
Activity,
|
Activity,
|
||||||
CornerDownRight,
|
CornerDownRight,
|
||||||
@@ -33,6 +34,8 @@ import {
|
|||||||
wiresAs,
|
wiresAs,
|
||||||
} from "@/components/Devices/ports"
|
} from "@/components/Devices/ports"
|
||||||
import { devicesQueryOptions } from "@/components/Devices/queries"
|
import { devicesQueryOptions } from "@/components/Devices/queries"
|
||||||
|
import { isLive, shortId } from "@/components/Runs/queries"
|
||||||
|
import { RunStatusBadge } from "@/components/Runs/RunStatus"
|
||||||
import { Button } from "@/components/ui/button"
|
import { Button } from "@/components/ui/button"
|
||||||
import {
|
import {
|
||||||
Command,
|
Command,
|
||||||
@@ -56,6 +59,7 @@ import useAuth from "@/hooks/useAuth"
|
|||||||
import useCustomToast from "@/hooks/useCustomToast"
|
import useCustomToast from "@/hooks/useCustomToast"
|
||||||
import { inCodeEditor, useShortcuts } from "@/lib/shortcuts"
|
import { inCodeEditor, useShortcuts } from "@/lib/shortcuts"
|
||||||
import { cn } from "@/lib/utils"
|
import { cn } from "@/lib/utils"
|
||||||
|
import { handleError } from "@/utils"
|
||||||
import { qualify } from "./deriveEdges"
|
import { qualify } from "./deriveEdges"
|
||||||
import { DtypeValue } from "./FlowBoundary"
|
import { DtypeValue } from "./FlowBoundary"
|
||||||
import {
|
import {
|
||||||
@@ -1593,6 +1597,119 @@ function SharingSection({
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A fraction as a percentage, or a dash when there is none. */
|
||||||
|
const percent = (value: unknown) =>
|
||||||
|
typeof value === "number"
|
||||||
|
? value.toLocaleString(undefined, {
|
||||||
|
style: "percent",
|
||||||
|
maximumFractionDigits: 1,
|
||||||
|
})
|
||||||
|
: "–"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Train a new model for a wren node and offer it as the node's `model`. Using
|
||||||
|
* it is an edit like any other, so it waits in the draft until publishing.
|
||||||
|
*/
|
||||||
|
function TrainingSection({
|
||||||
|
flow,
|
||||||
|
node,
|
||||||
|
onUse,
|
||||||
|
}: {
|
||||||
|
flow: string
|
||||||
|
node: NodeDef_Input
|
||||||
|
onUse: (model: string) => void
|
||||||
|
}) {
|
||||||
|
const queryClient = useQueryClient()
|
||||||
|
// Why it would not start, such as wren's training flow not being registered.
|
||||||
|
const [problem, setProblem] = useState<string | null>(null)
|
||||||
|
const queryKey = ["wren-training", flow, node.id]
|
||||||
|
const { data: training } = useQuery({
|
||||||
|
queryKey,
|
||||||
|
queryFn: () => FlowsService.readTraining({ name: flow, nodeId: node.id }),
|
||||||
|
// Training takes minutes, so only a run still going is read again.
|
||||||
|
refetchInterval: (query) =>
|
||||||
|
query.state.data && isLive(query.state.data.status) ? 5_000 : false,
|
||||||
|
})
|
||||||
|
const train = useMutation({
|
||||||
|
mutationFn: () => FlowsService.trainNode({ name: flow, nodeId: node.id }),
|
||||||
|
onMutate: () => setProblem(null),
|
||||||
|
onSuccess: () => queryClient.invalidateQueries({ queryKey }),
|
||||||
|
onError: handleError.bind(setProblem),
|
||||||
|
})
|
||||||
|
const running = training ? isLive(training.status) : false
|
||||||
|
const model = training?.model
|
||||||
|
const inUse = Boolean(model) && node.params?.model === model
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="grid gap-2" data-testid="wren-training">
|
||||||
|
<div className="flex items-center justify-between gap-3">
|
||||||
|
<div className="flex items-center gap-1">
|
||||||
|
<span className={SECTION}>Training</span>
|
||||||
|
<InfoTip label="Training">
|
||||||
|
Trains a new model on the targets wired here and on the examples and
|
||||||
|
tests, as a run of wren's training flow. The node keeps its model
|
||||||
|
until you use the new one and publish.
|
||||||
|
</InfoTip>
|
||||||
|
</div>
|
||||||
|
<Button
|
||||||
|
variant="outline"
|
||||||
|
size="sm"
|
||||||
|
className="h-8"
|
||||||
|
disabled={train.isPending || running}
|
||||||
|
onClick={() => train.mutate()}
|
||||||
|
data-testid="wren-train"
|
||||||
|
>
|
||||||
|
Train
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
{problem ? <p className="text-xs text-destructive">{problem}</p> : null}
|
||||||
|
{training ? (
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<RunStatusBadge
|
||||||
|
run={{ status: training.status, status_reason: "", labels: [] }}
|
||||||
|
/>
|
||||||
|
<Link
|
||||||
|
to="/runs/$id"
|
||||||
|
params={{ id: training.id }}
|
||||||
|
className="font-mono text-sm hover:underline"
|
||||||
|
>
|
||||||
|
{shortId(training.id)}
|
||||||
|
</Link>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
{training?.error ? (
|
||||||
|
<p className="text-xs text-destructive">{training.error}</p>
|
||||||
|
) : null}
|
||||||
|
{training && model ? (
|
||||||
|
<>
|
||||||
|
<dl className="grid gap-1 text-sm">
|
||||||
|
{[
|
||||||
|
["Exact match", percent(training.result?.exact_match)],
|
||||||
|
["Coverage", percent(training.result?.coverage)],
|
||||||
|
["Threshold", training.threshold?.toFixed(2) ?? "–"],
|
||||||
|
["Test exact match", percent(training.test_exact_match)],
|
||||||
|
].map(([label, value]) => (
|
||||||
|
<div key={label} className="flex justify-between gap-3">
|
||||||
|
<dt className="text-muted-foreground">{label}</dt>
|
||||||
|
<dd className="font-mono">{value}</dd>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</dl>
|
||||||
|
<Button
|
||||||
|
size="sm"
|
||||||
|
className="h-8 justify-self-start"
|
||||||
|
disabled={inUse}
|
||||||
|
onClick={() => onUse(model)}
|
||||||
|
data-testid="wren-use-model"
|
||||||
|
>
|
||||||
|
{inUse ? "In use" : "Use this model"}
|
||||||
|
</Button>
|
||||||
|
</>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/** The value a fresh output returns, so the scaffold runs as written. */
|
/** The value a fresh output returns, so the scaffold runs as written. */
|
||||||
const PLACEHOLDER: Record<DType, string> = {
|
const PLACEHOLDER: Record<DType, string> = {
|
||||||
float: "0.0",
|
float: "0.0",
|
||||||
@@ -2037,6 +2154,15 @@ function PanelBody({
|
|||||||
{nodeType?.plugin && user?.is_superuser ? (
|
{nodeType?.plugin && user?.is_superuser ? (
|
||||||
<StoredStateSection nodeType={nodeType.type} />
|
<StoredStateSection nodeType={nodeType.type} />
|
||||||
) : null}
|
) : null}
|
||||||
|
{node.type === "wren" ? (
|
||||||
|
<TrainingSection
|
||||||
|
flow={flow}
|
||||||
|
node={node}
|
||||||
|
onUse={(model) =>
|
||||||
|
editNode({ ...node, params: { ...node.params, model } })
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
) : null}
|
||||||
{hasSource ? (
|
{hasSource ? (
|
||||||
<div className="grid gap-1.5">
|
<div className="grid gap-1.5">
|
||||||
<div className="flex items-center gap-1">
|
<div className="flex items-center gap-1">
|
||||||
|
|||||||
Reference in New Issue
Block a user