wren node: Train in the panel, as a run of wren's training flow
POST/GET /flows/{name}/nodes/{id}/train submit wren_train on the node's
targets, examples and tests and read its latest run (run.for_node,
cause "train"); "Use this model" writes `model` into the draft.
This commit is contained in:
@@ -57,6 +57,7 @@ from fluksio.flow.devices import (
|
||||
)
|
||||
from fluksio.flow.events import event_bus
|
||||
from fluksio.flow.messages import MEDIA_FAMILIES, DType, flow_of, qualify
|
||||
from fluksio.flow.nodes.wren import TRAIN_FLOW, WrenNode, pack_meta
|
||||
from fluksio.flow.pipeline import ValidationIssue
|
||||
from fluksio.flow.runs import RunRejected
|
||||
from fluksio.flow.schemas import (
|
||||
@@ -84,7 +85,7 @@ from fluksio.flow.store import (
|
||||
LibNotFound,
|
||||
StaleVersion,
|
||||
)
|
||||
from fluksio.models import Flavor, Message, Run
|
||||
from fluksio.models import Flavor, Message, Run, RunArtifact, RunNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -159,6 +160,24 @@ class TriggerRequest(BaseModel):
|
||||
values: dict[str, Any] = {}
|
||||
|
||||
|
||||
class Training(BaseModel):
|
||||
"""A wren node's latest training run, as its panel shows it."""
|
||||
|
||||
id: str
|
||||
#: queued, running, ok, error, cancelled or abandoned.
|
||||
status: str
|
||||
#: What went wrong, when it failed.
|
||||
error: str = ""
|
||||
#: final_loss, n_params, exact_match, threshold and coverage, once it ended.
|
||||
result: dict[str, Any] = {}
|
||||
#: The trained model's digest, for the node's ``model`` setting.
|
||||
model: str = ""
|
||||
#: From the model's own record: the confidence it recommends, and its
|
||||
#: score on the node's ``tests`` when there were any.
|
||||
threshold: float | None = None
|
||||
test_exact_match: float | None = None
|
||||
|
||||
|
||||
def _endpoints(controller: FlowController, definition: FlowDef) -> list[Endpoint]:
|
||||
"""Everything wired into this flow from outside it."""
|
||||
flow = definition.name
|
||||
@@ -1010,6 +1029,111 @@ def acknowledge_node_error(
|
||||
return Message(message=f"Cleared the failure on '{node_id}'")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Training a wren node
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _node_def(controller: FlowController, name: str, node_id: str) -> NodeDef:
|
||||
"""A node as the editor shows it: from the draft when there is one."""
|
||||
node = next(
|
||||
(n for n in _read_flow(controller, name).nodes if n.id == node_id), None
|
||||
)
|
||||
if node is None:
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"No node named '{node_id}' in flow '{name}'"
|
||||
)
|
||||
return node
|
||||
|
||||
|
||||
@router.post("/{name}/nodes/{node_id}/train", response_model=Training, status_code=202)
|
||||
async def train_node(
|
||||
name: str,
|
||||
node_id: str,
|
||||
request: Request,
|
||||
controller: FlowControllerDep,
|
||||
user: CurrentUser,
|
||||
) -> Any:
|
||||
"""Train a new model for a wren node, as a run of wren's training flow.
|
||||
|
||||
It learns the node's targets, its ``examples`` and its ``tests``, as the
|
||||
editor shows them. The node keeps running its model until the new one is
|
||||
set as its ``model``.
|
||||
"""
|
||||
node = _node_def(controller, name, node_id)
|
||||
if node.type != "wren":
|
||||
raise HTTPException(status_code=400, detail=f"'{node_id}' is not a wren node")
|
||||
try:
|
||||
wren = WrenNode(
|
||||
requires=[spec for spec in node.requires if spec.name],
|
||||
provides=[spec for spec in node.provides if spec.name],
|
||||
params=node.params,
|
||||
)
|
||||
except (ValueError, RuntimeError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
service = getattr(request.app.state, "run_service", None)
|
||||
if service is None:
|
||||
raise HTTPException(status_code=503, detail="Runs are not available")
|
||||
try:
|
||||
run = await run_in_threadpool(
|
||||
service.submit,
|
||||
TRAIN_FLOW,
|
||||
params={"installation": orjson.dumps(wren.installation()).decode()},
|
||||
cause="train",
|
||||
actor=user.email,
|
||||
no_cache=True,
|
||||
for_node=f"{name}.{node_id}",
|
||||
)
|
||||
except FlowNotFound:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
f"wren's training flow '{TRAIN_FLOW}' is not registered. Run "
|
||||
"`fluksio sync wren.flow` from an environment with wren[train]."
|
||||
),
|
||||
) from None
|
||||
except RunRejected as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
return Training(id=run.id, status=run.status)
|
||||
|
||||
|
||||
@router.get("/{name}/nodes/{node_id}/train", response_model=Training | None)
|
||||
def read_training(
|
||||
name: str,
|
||||
node_id: str,
|
||||
request: Request,
|
||||
controller: FlowControllerDep,
|
||||
session: SessionDep,
|
||||
) -> Any:
|
||||
"""A wren node's latest training run, or null when it has none."""
|
||||
_node_def(controller, name, node_id)
|
||||
run = session.exec(
|
||||
select(Run)
|
||||
.where(col(Run.flow) == TRAIN_FLOW, col(Run.for_node) == f"{name}.{node_id}")
|
||||
.order_by(col(Run.created_at).desc())
|
||||
).first()
|
||||
if run is None:
|
||||
return None
|
||||
training = Training(id=run.id, status=run.status, result=run.result)
|
||||
if run.status == "error":
|
||||
failed = session.exec(
|
||||
select(RunNode.error).where(
|
||||
col(RunNode.run_id) == run.id, col(RunNode.error) != ""
|
||||
)
|
||||
).first()
|
||||
training.error = failed or run.status_reason
|
||||
# The model, once the run made one the store still has.
|
||||
pack = session.get(RunArtifact, {"run_id": run.id, "name": f"{TRAIN_FLOW}.pack"})
|
||||
store = getattr(request.app.state, "artifact_store", None)
|
||||
path = store.path(pack.digest) if pack is not None and store else None
|
||||
if pack is not None and path is not None:
|
||||
meta = pack_meta(path)
|
||||
training.model = pack.digest
|
||||
training.threshold = meta.get("threshold")
|
||||
training.test_exact_match = meta.get("test_exact_match")
|
||||
return training
|
||||
|
||||
|
||||
@router.get("/{name}/state", response_model=FlowStatePublic)
|
||||
def read_flow_state(name: str, controller: FlowControllerDep, hidden: HiddenDep) -> Any:
|
||||
"""The last value seen on every message of this flow."""
|
||||
|
||||
Reference in New Issue
Block a user