Assistant node, and the Control tab as one conversation (openpanel V5)

run_turn takes its tools, system prompt and tool caller; the chat passes the
MCP tools as before. The assistant node offers one tool per wired target,
reads its other inputs as context, streams its reply a sentence at a time,
warms the provider on `listening` and says so when the provider is not there.
Completions share the per-server HTTP client. wren replies when nothing reads
`unhandled`, stt shows a note when nothing was heard. /ai/control lists one
conversation per entry node with the messages to watch.
This commit is contained in:
Melvin Strobl
2026-10-19 23:23:25 +02:00
parent 4df7bdb057
commit a286cf4ceb
21 changed files with 937 additions and 184 deletions
+7
View File
@@ -41,6 +41,7 @@ from fluksio.flow.executor import ExecutionService
from fluksio.flow.messages import MessageSpec, flow_of, qualify
from fluksio.flow.nodes import (
RESERVED_SETTINGS,
AssistantNode,
ChangeNode,
DebounceNode,
DelayNode,
@@ -385,6 +386,12 @@ NODE_TYPES: dict[str, NodeType] = {
cls=WrenNode,
params_schema=_schema_of(WrenNode),
),
"assistant": NodeType(
title="Assistant",
description="Answer a request with the AI provider, acting on what is wired to it.",
cls=AssistantNode,
params_schema=_schema_of(AssistantNode),
),
"stt": NodeType(
title="Speech to text",
description="Turn a microphone's audio into text, through a speech server.",
+2
View File
@@ -4,6 +4,7 @@ Split by the outside world each one talks to. Importing from
``fluksio.flow.nodes`` keeps working, which is what every caller does.
"""
from fluksio.flow.nodes.assistant import AssistantNode
from fluksio.flow.nodes.base import RESERVED_SETTINGS, Node
from fluksio.flow.nodes.debounce import DebounceNode
from fluksio.flow.nodes.delay import DelayNode
@@ -23,6 +24,7 @@ from fluksio.flow.nodes.websocket import WebSocketNode
from fluksio.flow.nodes.wren import WrenNode
__all__ = [
"AssistantNode",
"ChangeNode",
"DebounceNode",
"DelayNode",
+350
View File
@@ -0,0 +1,350 @@
"""Assistant: a request answered by the instance's language model.
The request arrives on ``prompt``, typically what a wren node could not do.
Every output but ``reply`` and ``actions`` is a target the model may set,
offered as one tool named after its port and typed by its message, so the
wiring is the allowlist, as for wren: the MCP authoring tools are never
offered. Every other input is read as context ("outside_temperature: 7.5").
The answer goes out on ``reply`` a sentence at a time while the model writes
it, so a ``tts`` node speaks the first before the last is written.
"""
from __future__ import annotations
import asyncio
import contextlib
import dataclasses
import json
import logging
import re
import threading
from collections.abc import Coroutine, Iterable
from concurrent.futures import Future
from typing import TYPE_CHECKING, Any
import httpx
from pydantic import BaseModel, ConfigDict, Field
from fluksio.ai import agent, client
from fluksio.ai import config as ai_config
from fluksio.flow.messages import DType, MessageSpec
from fluksio.flow.nodes.base import Node
if TYPE_CHECKING:
from fastapi import FastAPI
logger = logging.getLogger(__name__)
REPLY = "reply"
ACTIONS = "actions"
#: An optional input: true warms the provider while the request is spoken.
LISTENING = "listening"
#: Said when there is no provider, or it does not answer.
UNAVAILABLE = "The assistant isn't available right now."
#: Said when the model ends a turn without a word.
DONE, CANT = "Done.", "Sorry, I can't do that."
#: A server that is not there is given up on after a few seconds, while a
#: model that has to be loaded first takes about ten.
TIMEOUT = httpx.Timeout(15.0, connect=3.0)
#: Rounds of tool calls one request may take.
ROUNDS = 3
#: A target's type, as its tool's parameter.
_SCHEMAS: dict[DType, dict[str, str]] = {
DType.BOOL: {"type": "boolean"},
DType.INT: {"type": "integer"},
DType.FLOAT: {"type": "number"},
DType.STR: {"type": "string"},
}
_SYSTEM = """You are the voice of a home: someone spoke to a panel on the wall, \
and you answer out loud.
- To change something, call its tool. The tools are all you can change.
- After a change, say what you did in one short sentence.
- If no tool can do what was asked, or you did not understand it, say so in a \
few words, such as "Sorry, I can't do that."
- Answer questions from the readings below.
- Speak plainly: one or two short sentences, no lists, no markdown, no emoji.
"""
_END = re.compile(r"(?<=[.!?])\s+")
def cut(text: str) -> tuple[list[str], str]:
"""The sentences ``text`` has finished, and the one still being written."""
*done, rest = _END.split(text)
return [s.strip() for s in done if s.strip()], rest
class AssistantNode(Node):
"""Answer the request on ``prompt``, acting only on the targets wired to it.
Inputs: ``prompt`` (the request, which wakes the node), ``listening``
(optional; turning true opens the connection and loads the model while
the request is still spoken) and any other, read as context.
Outputs: ``reply``, the answer a sentence at a time, ``actions``, a list
of records, one per value set, and any number of targets, each a tool.
When the provider is not set up or does not answer, the reply says so.
"""
class Params(BaseModel):
model_config = ConfigDict(extra="allow")
model: str = Field(
default="", description="The model to ask. Empty asks the instance's."
)
extra_body: dict[str, Any] = Field(
default_factory=lambda: {
"temperature": 0,
"reasoning_effort": "none",
"keep_alive": "30m",
},
description=(
"Added to every request as it is. The default answers a request "
"the same way each time, turns thinking off and keeps the model "
"loaded for 30 minutes on Ollama; drop what a provider refuses."
),
)
__slots__ = ("cfg", "_listening", "_lock", "_loop")
# A tool call is a command to a device, as wren's are.
idempotent = False
def __init__(
self,
requires: MessageSpec | Iterable[MessageSpec] = (),
provides: MessageSpec | Iterable[MessageSpec] = (),
params: dict[str, Any] | None = None,
name: str | None = None,
):
self.cfg = self.Params.model_validate(params or {})
self._lock = threading.Lock()
#: What ``listening`` held at the last run, to tell its change apart.
self._listening: bool | None = None
self._loop: asyncio.AbstractEventLoop | None = None
inputs = self._normalize_ports(requires)
outputs = self._normalize_ports(provides)
typed = {"prompt": DType.STR, LISTENING: DType.BOOL}
for spec in inputs:
if spec.port in typed and spec.dtype is not typed[spec.port]:
raise ValueError(f"'{spec.port}' is a {typed[spec.port].value}.")
if not any(spec.port == "prompt" for spec in inputs):
raise ValueError("assistant needs a 'prompt' input: the request.")
for spec in outputs:
if spec.port == REPLY and spec.dtype is not DType.STR:
raise ValueError(f"'{REPLY}' is a str.")
if spec.port == ACTIONS and (
spec.dtype is not DType.LIST or spec.item_dtype is not DType.RECORD
):
raise ValueError(f"'{ACTIONS}' is a list of records.")
if spec.port not in (REPLY, ACTIONS) and spec.dtype not in _SCHEMAS:
raise ValueError(
f"'{spec.port}' is {spec.dtype.value}; a target is bool, int, "
"float or str."
)
super().__init__(
self._answer,
requires=[
spec.model_copy(update={"trigger": spec.port in ("prompt", LISTENING)})
for spec in inputs
],
provides=outputs,
params=params,
name=name or "assistant",
)
async def start(self, app: FastAPI | None = None) -> None:
self._loop = asyncio.get_running_loop()
names = [spec.name for spec in self.input_ports if spec.port == LISTENING]
if names and self._pipeline is not None:
held = await asyncio.to_thread(self._pipeline.state.get_present, names)
self._listening = held.get(names[0])
def _targets(self) -> list[MessageSpec]:
return [
spec
for spec in self.output_ports
if spec.name and spec.port not in (REPLY, ACTIONS)
]
def _tools(self) -> list[dict[str, Any]]:
return [
{
"type": "function",
"function": {
"name": spec.port,
"description": f"Set the {spec.port.replace('_', ' ')}.",
"parameters": {
"type": "object",
"properties": {"value": _SCHEMAS[spec.dtype]},
"required": ["value"],
},
},
}
for spec in self._targets()
]
def _system(self, context: dict[str, Any]) -> str:
"""The instructions, then what the inputs read and the targets hold."""
targets = self._targets()
held = (
self._pipeline.state.get_present([spec.name for spec in targets])
if self._pipeline is not None
else {}
)
lines = [_SYSTEM, "Readings:"]
lines += [f"{port}: {json.dumps(value)}" for port, value in context.items()]
lines += ["", "What the tools set, now:"]
lines += [
f"{spec.port}: {json.dumps(held[spec.name])}"
for spec in targets
if spec.name in held
]
return "\n".join(lines)
def _config(self) -> ai_config.AIConfig | None:
config = ai_config.load()
if config is None or not config.ready:
return None
return dataclasses.replace(
config, model=self.cfg.model or config.model, max_rounds=ROUNDS
)
def _call(self, work: Coroutine[Any, Any, Any]) -> Future[Any]:
"""Run on the engine's loop, where the provider's connection is kept."""
if self._loop is None:
work.close()
raise RuntimeError(f"{self.name} only runs in a live flow")
return asyncio.run_coroutine_threadsafe(work, self._loop)
def _answer(
self,
params: dict[str, Any],
prompt: str = "",
listening: bool | None = None,
**context: Any,
) -> dict[str, Any] | None:
with self._lock:
was, self._listening = self._listening, listening
config = self._config()
if listening != was:
# Not a request: the device started or stopped listening.
if listening and config is not None:
self._call(self._warm(config))
return None
if not prompt.strip():
return None
actions: list[dict[str, Any]] = []
problem = self._call(
self._turn(config, prompt, self._system(context), actions)
).result()
if problem:
self.report_health("degraded", problem)
elif self._health != "ok":
self.report_health("ok")
wired = {spec.port for spec in self.output_ports if spec.name}
return {ACTIONS: actions} if ACTIONS in wired else None
async def _warm(self, config: ai_config.AIConfig) -> None:
"""A request for one token: the connection opens and the model loads.
Nothing more is worth sending ahead: a model with a sliding window,
such as Gemma, reads the whole prompt again on every request anyway.
"""
with contextlib.suppress(Exception):
async for _ in client.stream_completion(
api_base=config.api_base,
api_key=config.api_key,
model=config.model,
messages=[{"role": "user", "content": "Hello."}],
tools=[],
timeout=60.0,
extra={**self.cfg.extra_body, "max_tokens": 1},
):
pass
async def _turn(
self,
config: ai_config.AIConfig | None,
prompt: str,
system: str,
actions: list[dict[str, Any]],
) -> str:
"""Ask, act and say. Returns what went wrong, or an empty string."""
said = False
async def say(text: str) -> None:
nonlocal said
said = True
await asyncio.to_thread(self._say, text)
if config is None:
await say(UNAVAILABLE)
return "no AI provider is set up"
rest, problem = "", ""
try:
async for event in agent.run_turn(
config,
[],
prompt,
tools=self._tools(),
system=system,
call_tool=self._caller(actions),
extra=self.cfg.extra_body,
timeout=TIMEOUT,
):
if event["type"] == "delta":
done, rest = cut(rest + event["content"])
for sentence in done:
await say(sentence)
elif event["type"] == "message" and rest.strip():
await say(rest.strip())
rest = ""
elif event["type"] == "error":
problem = str(event["detail"])
except Exception as exc: # noqa: BLE001 - said, then reported as the health
problem = f"{type(exc).__name__}: {exc}"
if problem:
logger.warning("'%s' got no answer: %s", self.id, problem)
if not said:
await say(UNAVAILABLE if problem else DONE if actions else CANT)
return problem
def _caller(self, actions: list[dict[str, Any]]) -> agent.ToolCaller:
"""Set the target a call names, and record it in ``actions``."""
specs = {spec.port: spec for spec in self._targets()}
async def call(name: str, arguments: dict[str, Any]) -> str:
spec = specs.get(name)
if spec is None:
return json.dumps({"error": f"There is no tool named {name}."})
value = arguments.get("value")
if spec.dtype is DType.FLOAT and type(value) is int:
value = float(value)
try:
spec.check(value)
except TypeError as exc:
return json.dumps({"error": str(exc)})
await asyncio.to_thread(self.emit, {name: value})
actions.append(
{
"message": spec.name,
"value": value,
"applied": True,
"delay_s": None,
"call": f"{name}({json.dumps(value)})",
}
)
return json.dumps({name: value})
return call
def _say(self, text: str) -> None:
if any(spec.port == REPLY and spec.name for spec in self.output_ports):
self.emit({REPLY: text})
+12 -3
View File
@@ -41,6 +41,9 @@ SPOKEN = 256
#: How long after its estimated end a sentence still counts as playing: the
#: device fetches it before it starts.
PLAYING_SLACK_S = 1.0
#: The partial for an utterance that came back empty, often a wake word that
#: was not one, or was lost: shown on the caption, never spoken.
NOTHING_HEARD = "Didn't catch that."
def _check(specs: list[MessageSpec], ports: dict[str, DType], node: str) -> None:
@@ -152,8 +155,9 @@ class SttNode(_Speech):
One realtime session per utterance, opened when ``listening`` turns true
or at the first chunk, and committed when it turns false. A clip goes to
the server whole. Either way the node returns at once, since a device's
next frame waits for it, and publishes once the server answers. An
utterance that fails publishes nothing and is logged.
next frame waits for it, and publishes once the server answers. Nothing
heard, or an utterance lost and logged, is a short note on ``partial``
alone.
"""
__slots__ = ("_heard", "_listening", "_lock", "_session", "_tasks")
@@ -268,12 +272,15 @@ class SttNode(_Speech):
words = text.strip() if final else words + text
if final and words:
await asyncio.to_thread(self._say, partial=words, text=words)
elif final:
await asyncio.to_thread(self._say, partial=NOTHING_HEARD)
elif words:
await asyncio.to_thread(self._say, partial=words)
except Exception as exc:
logger.warning(
"'%s' lost an utterance: %s: %s", self.id, type(exc).__name__, exc
)
await asyncio.to_thread(self._say, partial=NOTHING_HEARD)
finally:
# Ended without a commit, by a timeout or a failure: the next chunk
# opens a session of its own rather than feeding this dead one.
@@ -288,9 +295,11 @@ class SttNode(_Speech):
)
except Exception as exc:
logger.warning("'%s' lost a clip: %s: %s", self.id, type(exc).__name__, exc)
return
text = ""
if text.strip():
await asyncio.to_thread(self._say, partial=text.strip(), text=text.strip())
else:
await asyncio.to_thread(self._say, partial=NOTHING_HEARD)
class TtsNode(_Speech):
+12 -2
View File
@@ -71,6 +71,8 @@ INPUTS = {
ACTIONS = "actions"
REPLY = "reply"
UNHANDLED = "unhandled"
#: The reply when nothing was done and nothing reads ``unhandled``.
CANT = "Sorry, I can't do that yet."
_TARGET_TYPES = {DType.BOOL, DType.INT, DType.FLOAT, DType.STR}
@@ -198,8 +200,9 @@ class WrenNode(Node):
(otherwise read off the clock: before 07:00 or from 18:00).
Outputs: ``actions``, a list of records, one per call and target,
``reply``, a sentence saying what was applied ("Kitchen light on."),
``unhandled``, the prompt when nothing was, and any number of targets
``reply``, a sentence saying what was applied ("Kitchen light on."), or
that nothing was when no node reads ``unhandled``, ``unhandled``, the
prompt when nothing was, and any number of targets
named ``<area>_<kind>``. Targets of one area and kind are one device to
the model; each takes what the call means for its type.
A delayed call is held by the engine's work queue, like the Trigger
@@ -401,4 +404,11 @@ class WrenNode(Node):
writes[REPLY] = reply[0].upper() + reply[1:] + "."
if not said and UNHANDLED in wired:
writes[UNHANDLED] = prompt
if not said and REPLY in wired and not self._passed_on():
writes[REPLY] = CANT
return writes or None
def _passed_on(self) -> bool:
"""Whether a node reads ``unhandled``, such as an assistant."""
name = next((s.name for s in self.output_ports if s.port == UNHANDLED), "")
return bool(name and self._pipeline and self._pipeline.consumes.get(name))