Assistant node, and the Control tab as one conversation (openpanel V5)
run_turn takes its tools, system prompt and tool caller; the chat passes the MCP tools as before. The assistant node offers one tool per wired target, reads its other inputs as context, streams its reply a sentence at a time, warms the provider on `listening` and says so when the provider is not there. Completions share the per-server HTTP client. wren replies when nothing reads `unhandled`, stt shows a note when nothing was heard. /ai/control lists one conversation per entry node with the messages to watch.
This commit is contained in:
@@ -41,6 +41,7 @@ from fluksio.flow.executor import ExecutionService
|
||||
from fluksio.flow.messages import MessageSpec, flow_of, qualify
|
||||
from fluksio.flow.nodes import (
|
||||
RESERVED_SETTINGS,
|
||||
AssistantNode,
|
||||
ChangeNode,
|
||||
DebounceNode,
|
||||
DelayNode,
|
||||
@@ -385,6 +386,12 @@ NODE_TYPES: dict[str, NodeType] = {
|
||||
cls=WrenNode,
|
||||
params_schema=_schema_of(WrenNode),
|
||||
),
|
||||
"assistant": NodeType(
|
||||
title="Assistant",
|
||||
description="Answer a request with the AI provider, acting on what is wired to it.",
|
||||
cls=AssistantNode,
|
||||
params_schema=_schema_of(AssistantNode),
|
||||
),
|
||||
"stt": NodeType(
|
||||
title="Speech to text",
|
||||
description="Turn a microphone's audio into text, through a speech server.",
|
||||
|
||||
@@ -4,6 +4,7 @@ Split by the outside world each one talks to. Importing from
|
||||
``fluksio.flow.nodes`` keeps working, which is what every caller does.
|
||||
"""
|
||||
|
||||
from fluksio.flow.nodes.assistant import AssistantNode
|
||||
from fluksio.flow.nodes.base import RESERVED_SETTINGS, Node
|
||||
from fluksio.flow.nodes.debounce import DebounceNode
|
||||
from fluksio.flow.nodes.delay import DelayNode
|
||||
@@ -23,6 +24,7 @@ from fluksio.flow.nodes.websocket import WebSocketNode
|
||||
from fluksio.flow.nodes.wren import WrenNode
|
||||
|
||||
__all__ = [
|
||||
"AssistantNode",
|
||||
"ChangeNode",
|
||||
"DebounceNode",
|
||||
"DelayNode",
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
"""Assistant: a request answered by the instance's language model.
|
||||
|
||||
The request arrives on ``prompt``, typically what a wren node could not do.
|
||||
Every output but ``reply`` and ``actions`` is a target the model may set,
|
||||
offered as one tool named after its port and typed by its message, so the
|
||||
wiring is the allowlist, as for wren: the MCP authoring tools are never
|
||||
offered. Every other input is read as context ("outside_temperature: 7.5").
|
||||
|
||||
The answer goes out on ``reply`` a sentence at a time while the model writes
|
||||
it, so a ``tts`` node speaks the first before the last is written.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Coroutine, Iterable
|
||||
from concurrent.futures import Future
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from fluksio.ai import agent, client
|
||||
from fluksio.ai import config as ai_config
|
||||
from fluksio.flow.messages import DType, MessageSpec
|
||||
from fluksio.flow.nodes.base import Node
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import FastAPI
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REPLY = "reply"
|
||||
ACTIONS = "actions"
|
||||
#: An optional input: true warms the provider while the request is spoken.
|
||||
LISTENING = "listening"
|
||||
|
||||
#: Said when there is no provider, or it does not answer.
|
||||
UNAVAILABLE = "The assistant isn't available right now."
|
||||
#: Said when the model ends a turn without a word.
|
||||
DONE, CANT = "Done.", "Sorry, I can't do that."
|
||||
|
||||
#: A server that is not there is given up on after a few seconds, while a
|
||||
#: model that has to be loaded first takes about ten.
|
||||
TIMEOUT = httpx.Timeout(15.0, connect=3.0)
|
||||
#: Rounds of tool calls one request may take.
|
||||
ROUNDS = 3
|
||||
|
||||
#: A target's type, as its tool's parameter.
|
||||
_SCHEMAS: dict[DType, dict[str, str]] = {
|
||||
DType.BOOL: {"type": "boolean"},
|
||||
DType.INT: {"type": "integer"},
|
||||
DType.FLOAT: {"type": "number"},
|
||||
DType.STR: {"type": "string"},
|
||||
}
|
||||
|
||||
_SYSTEM = """You are the voice of a home: someone spoke to a panel on the wall, \
|
||||
and you answer out loud.
|
||||
|
||||
- To change something, call its tool. The tools are all you can change.
|
||||
- After a change, say what you did in one short sentence.
|
||||
- If no tool can do what was asked, or you did not understand it, say so in a \
|
||||
few words, such as "Sorry, I can't do that."
|
||||
- Answer questions from the readings below.
|
||||
- Speak plainly: one or two short sentences, no lists, no markdown, no emoji.
|
||||
"""
|
||||
|
||||
_END = re.compile(r"(?<=[.!?])\s+")
|
||||
|
||||
|
||||
def cut(text: str) -> tuple[list[str], str]:
|
||||
"""The sentences ``text`` has finished, and the one still being written."""
|
||||
*done, rest = _END.split(text)
|
||||
return [s.strip() for s in done if s.strip()], rest
|
||||
|
||||
|
||||
class AssistantNode(Node):
|
||||
"""Answer the request on ``prompt``, acting only on the targets wired to it.
|
||||
|
||||
Inputs: ``prompt`` (the request, which wakes the node), ``listening``
|
||||
(optional; turning true opens the connection and loads the model while
|
||||
the request is still spoken) and any other, read as context.
|
||||
|
||||
Outputs: ``reply``, the answer a sentence at a time, ``actions``, a list
|
||||
of records, one per value set, and any number of targets, each a tool.
|
||||
When the provider is not set up or does not answer, the reply says so.
|
||||
"""
|
||||
|
||||
class Params(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
model: str = Field(
|
||||
default="", description="The model to ask. Empty asks the instance's."
|
||||
)
|
||||
extra_body: dict[str, Any] = Field(
|
||||
default_factory=lambda: {
|
||||
"temperature": 0,
|
||||
"reasoning_effort": "none",
|
||||
"keep_alive": "30m",
|
||||
},
|
||||
description=(
|
||||
"Added to every request as it is. The default answers a request "
|
||||
"the same way each time, turns thinking off and keeps the model "
|
||||
"loaded for 30 minutes on Ollama; drop what a provider refuses."
|
||||
),
|
||||
)
|
||||
|
||||
__slots__ = ("cfg", "_listening", "_lock", "_loop")
|
||||
|
||||
# A tool call is a command to a device, as wren's are.
|
||||
idempotent = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
requires: MessageSpec | Iterable[MessageSpec] = (),
|
||||
provides: MessageSpec | Iterable[MessageSpec] = (),
|
||||
params: dict[str, Any] | None = None,
|
||||
name: str | None = None,
|
||||
):
|
||||
self.cfg = self.Params.model_validate(params or {})
|
||||
self._lock = threading.Lock()
|
||||
#: What ``listening`` held at the last run, to tell its change apart.
|
||||
self._listening: bool | None = None
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
inputs = self._normalize_ports(requires)
|
||||
outputs = self._normalize_ports(provides)
|
||||
typed = {"prompt": DType.STR, LISTENING: DType.BOOL}
|
||||
for spec in inputs:
|
||||
if spec.port in typed and spec.dtype is not typed[spec.port]:
|
||||
raise ValueError(f"'{spec.port}' is a {typed[spec.port].value}.")
|
||||
if not any(spec.port == "prompt" for spec in inputs):
|
||||
raise ValueError("assistant needs a 'prompt' input: the request.")
|
||||
for spec in outputs:
|
||||
if spec.port == REPLY and spec.dtype is not DType.STR:
|
||||
raise ValueError(f"'{REPLY}' is a str.")
|
||||
if spec.port == ACTIONS and (
|
||||
spec.dtype is not DType.LIST or spec.item_dtype is not DType.RECORD
|
||||
):
|
||||
raise ValueError(f"'{ACTIONS}' is a list of records.")
|
||||
if spec.port not in (REPLY, ACTIONS) and spec.dtype not in _SCHEMAS:
|
||||
raise ValueError(
|
||||
f"'{spec.port}' is {spec.dtype.value}; a target is bool, int, "
|
||||
"float or str."
|
||||
)
|
||||
super().__init__(
|
||||
self._answer,
|
||||
requires=[
|
||||
spec.model_copy(update={"trigger": spec.port in ("prompt", LISTENING)})
|
||||
for spec in inputs
|
||||
],
|
||||
provides=outputs,
|
||||
params=params,
|
||||
name=name or "assistant",
|
||||
)
|
||||
|
||||
async def start(self, app: FastAPI | None = None) -> None:
|
||||
self._loop = asyncio.get_running_loop()
|
||||
names = [spec.name for spec in self.input_ports if spec.port == LISTENING]
|
||||
if names and self._pipeline is not None:
|
||||
held = await asyncio.to_thread(self._pipeline.state.get_present, names)
|
||||
self._listening = held.get(names[0])
|
||||
|
||||
def _targets(self) -> list[MessageSpec]:
|
||||
return [
|
||||
spec
|
||||
for spec in self.output_ports
|
||||
if spec.name and spec.port not in (REPLY, ACTIONS)
|
||||
]
|
||||
|
||||
def _tools(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": spec.port,
|
||||
"description": f"Set the {spec.port.replace('_', ' ')}.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"value": _SCHEMAS[spec.dtype]},
|
||||
"required": ["value"],
|
||||
},
|
||||
},
|
||||
}
|
||||
for spec in self._targets()
|
||||
]
|
||||
|
||||
def _system(self, context: dict[str, Any]) -> str:
|
||||
"""The instructions, then what the inputs read and the targets hold."""
|
||||
targets = self._targets()
|
||||
held = (
|
||||
self._pipeline.state.get_present([spec.name for spec in targets])
|
||||
if self._pipeline is not None
|
||||
else {}
|
||||
)
|
||||
lines = [_SYSTEM, "Readings:"]
|
||||
lines += [f"{port}: {json.dumps(value)}" for port, value in context.items()]
|
||||
lines += ["", "What the tools set, now:"]
|
||||
lines += [
|
||||
f"{spec.port}: {json.dumps(held[spec.name])}"
|
||||
for spec in targets
|
||||
if spec.name in held
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
def _config(self) -> ai_config.AIConfig | None:
|
||||
config = ai_config.load()
|
||||
if config is None or not config.ready:
|
||||
return None
|
||||
return dataclasses.replace(
|
||||
config, model=self.cfg.model or config.model, max_rounds=ROUNDS
|
||||
)
|
||||
|
||||
def _call(self, work: Coroutine[Any, Any, Any]) -> Future[Any]:
|
||||
"""Run on the engine's loop, where the provider's connection is kept."""
|
||||
if self._loop is None:
|
||||
work.close()
|
||||
raise RuntimeError(f"{self.name} only runs in a live flow")
|
||||
return asyncio.run_coroutine_threadsafe(work, self._loop)
|
||||
|
||||
def _answer(
|
||||
self,
|
||||
params: dict[str, Any],
|
||||
prompt: str = "",
|
||||
listening: bool | None = None,
|
||||
**context: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
with self._lock:
|
||||
was, self._listening = self._listening, listening
|
||||
config = self._config()
|
||||
if listening != was:
|
||||
# Not a request: the device started or stopped listening.
|
||||
if listening and config is not None:
|
||||
self._call(self._warm(config))
|
||||
return None
|
||||
if not prompt.strip():
|
||||
return None
|
||||
actions: list[dict[str, Any]] = []
|
||||
problem = self._call(
|
||||
self._turn(config, prompt, self._system(context), actions)
|
||||
).result()
|
||||
if problem:
|
||||
self.report_health("degraded", problem)
|
||||
elif self._health != "ok":
|
||||
self.report_health("ok")
|
||||
wired = {spec.port for spec in self.output_ports if spec.name}
|
||||
return {ACTIONS: actions} if ACTIONS in wired else None
|
||||
|
||||
async def _warm(self, config: ai_config.AIConfig) -> None:
|
||||
"""A request for one token: the connection opens and the model loads.
|
||||
|
||||
Nothing more is worth sending ahead: a model with a sliding window,
|
||||
such as Gemma, reads the whole prompt again on every request anyway.
|
||||
"""
|
||||
with contextlib.suppress(Exception):
|
||||
async for _ in client.stream_completion(
|
||||
api_base=config.api_base,
|
||||
api_key=config.api_key,
|
||||
model=config.model,
|
||||
messages=[{"role": "user", "content": "Hello."}],
|
||||
tools=[],
|
||||
timeout=60.0,
|
||||
extra={**self.cfg.extra_body, "max_tokens": 1},
|
||||
):
|
||||
pass
|
||||
|
||||
async def _turn(
|
||||
self,
|
||||
config: ai_config.AIConfig | None,
|
||||
prompt: str,
|
||||
system: str,
|
||||
actions: list[dict[str, Any]],
|
||||
) -> str:
|
||||
"""Ask, act and say. Returns what went wrong, or an empty string."""
|
||||
said = False
|
||||
|
||||
async def say(text: str) -> None:
|
||||
nonlocal said
|
||||
said = True
|
||||
await asyncio.to_thread(self._say, text)
|
||||
|
||||
if config is None:
|
||||
await say(UNAVAILABLE)
|
||||
return "no AI provider is set up"
|
||||
rest, problem = "", ""
|
||||
try:
|
||||
async for event in agent.run_turn(
|
||||
config,
|
||||
[],
|
||||
prompt,
|
||||
tools=self._tools(),
|
||||
system=system,
|
||||
call_tool=self._caller(actions),
|
||||
extra=self.cfg.extra_body,
|
||||
timeout=TIMEOUT,
|
||||
):
|
||||
if event["type"] == "delta":
|
||||
done, rest = cut(rest + event["content"])
|
||||
for sentence in done:
|
||||
await say(sentence)
|
||||
elif event["type"] == "message" and rest.strip():
|
||||
await say(rest.strip())
|
||||
rest = ""
|
||||
elif event["type"] == "error":
|
||||
problem = str(event["detail"])
|
||||
except Exception as exc: # noqa: BLE001 - said, then reported as the health
|
||||
problem = f"{type(exc).__name__}: {exc}"
|
||||
if problem:
|
||||
logger.warning("'%s' got no answer: %s", self.id, problem)
|
||||
if not said:
|
||||
await say(UNAVAILABLE if problem else DONE if actions else CANT)
|
||||
return problem
|
||||
|
||||
def _caller(self, actions: list[dict[str, Any]]) -> agent.ToolCaller:
|
||||
"""Set the target a call names, and record it in ``actions``."""
|
||||
specs = {spec.port: spec for spec in self._targets()}
|
||||
|
||||
async def call(name: str, arguments: dict[str, Any]) -> str:
|
||||
spec = specs.get(name)
|
||||
if spec is None:
|
||||
return json.dumps({"error": f"There is no tool named {name}."})
|
||||
value = arguments.get("value")
|
||||
if spec.dtype is DType.FLOAT and type(value) is int:
|
||||
value = float(value)
|
||||
try:
|
||||
spec.check(value)
|
||||
except TypeError as exc:
|
||||
return json.dumps({"error": str(exc)})
|
||||
await asyncio.to_thread(self.emit, {name: value})
|
||||
actions.append(
|
||||
{
|
||||
"message": spec.name,
|
||||
"value": value,
|
||||
"applied": True,
|
||||
"delay_s": None,
|
||||
"call": f"{name}({json.dumps(value)})",
|
||||
}
|
||||
)
|
||||
return json.dumps({name: value})
|
||||
|
||||
return call
|
||||
|
||||
def _say(self, text: str) -> None:
|
||||
if any(spec.port == REPLY and spec.name for spec in self.output_ports):
|
||||
self.emit({REPLY: text})
|
||||
@@ -41,6 +41,9 @@ SPOKEN = 256
|
||||
#: How long after its estimated end a sentence still counts as playing: the
|
||||
#: device fetches it before it starts.
|
||||
PLAYING_SLACK_S = 1.0
|
||||
#: The partial for an utterance that came back empty, often a wake word that
|
||||
#: was not one, or was lost: shown on the caption, never spoken.
|
||||
NOTHING_HEARD = "Didn't catch that."
|
||||
|
||||
|
||||
def _check(specs: list[MessageSpec], ports: dict[str, DType], node: str) -> None:
|
||||
@@ -152,8 +155,9 @@ class SttNode(_Speech):
|
||||
One realtime session per utterance, opened when ``listening`` turns true
|
||||
or at the first chunk, and committed when it turns false. A clip goes to
|
||||
the server whole. Either way the node returns at once, since a device's
|
||||
next frame waits for it, and publishes once the server answers. An
|
||||
utterance that fails publishes nothing and is logged.
|
||||
next frame waits for it, and publishes once the server answers. Nothing
|
||||
heard, or an utterance lost and logged, is a short note on ``partial``
|
||||
alone.
|
||||
"""
|
||||
|
||||
__slots__ = ("_heard", "_listening", "_lock", "_session", "_tasks")
|
||||
@@ -268,12 +272,15 @@ class SttNode(_Speech):
|
||||
words = text.strip() if final else words + text
|
||||
if final and words:
|
||||
await asyncio.to_thread(self._say, partial=words, text=words)
|
||||
elif final:
|
||||
await asyncio.to_thread(self._say, partial=NOTHING_HEARD)
|
||||
elif words:
|
||||
await asyncio.to_thread(self._say, partial=words)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"'%s' lost an utterance: %s: %s", self.id, type(exc).__name__, exc
|
||||
)
|
||||
await asyncio.to_thread(self._say, partial=NOTHING_HEARD)
|
||||
finally:
|
||||
# Ended without a commit, by a timeout or a failure: the next chunk
|
||||
# opens a session of its own rather than feeding this dead one.
|
||||
@@ -288,9 +295,11 @@ class SttNode(_Speech):
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("'%s' lost a clip: %s: %s", self.id, type(exc).__name__, exc)
|
||||
return
|
||||
text = ""
|
||||
if text.strip():
|
||||
await asyncio.to_thread(self._say, partial=text.strip(), text=text.strip())
|
||||
else:
|
||||
await asyncio.to_thread(self._say, partial=NOTHING_HEARD)
|
||||
|
||||
|
||||
class TtsNode(_Speech):
|
||||
|
||||
@@ -71,6 +71,8 @@ INPUTS = {
|
||||
ACTIONS = "actions"
|
||||
REPLY = "reply"
|
||||
UNHANDLED = "unhandled"
|
||||
#: The reply when nothing was done and nothing reads ``unhandled``.
|
||||
CANT = "Sorry, I can't do that yet."
|
||||
|
||||
_TARGET_TYPES = {DType.BOOL, DType.INT, DType.FLOAT, DType.STR}
|
||||
|
||||
@@ -198,8 +200,9 @@ class WrenNode(Node):
|
||||
(otherwise read off the clock: before 07:00 or from 18:00).
|
||||
|
||||
Outputs: ``actions``, a list of records, one per call and target,
|
||||
``reply``, a sentence saying what was applied ("Kitchen light on."),
|
||||
``unhandled``, the prompt when nothing was, and any number of targets
|
||||
``reply``, a sentence saying what was applied ("Kitchen light on."), or
|
||||
that nothing was when no node reads ``unhandled``, ``unhandled``, the
|
||||
prompt when nothing was, and any number of targets
|
||||
named ``<area>_<kind>``. Targets of one area and kind are one device to
|
||||
the model; each takes what the call means for its type.
|
||||
A delayed call is held by the engine's work queue, like the Trigger
|
||||
@@ -401,4 +404,11 @@ class WrenNode(Node):
|
||||
writes[REPLY] = reply[0].upper() + reply[1:] + "."
|
||||
if not said and UNHANDLED in wired:
|
||||
writes[UNHANDLED] = prompt
|
||||
if not said and REPLY in wired and not self._passed_on():
|
||||
writes[REPLY] = CANT
|
||||
return writes or None
|
||||
|
||||
def _passed_on(self) -> bool:
|
||||
"""Whether a node reads ``unhandled``, such as an assistant."""
|
||||
name = next((s.name for s in self.output_ports if s.port == UNHANDLED), "")
|
||||
return bool(name and self._pipeline and self._pipeline.consumes.get(name))
|
||||
|
||||
Reference in New Issue
Block a user