Assistant streams its answer token by token
ai/client.py streams the completion and puts the chunks back together (Assembler), covering OpenAI's piecewise tool-call arguments, Ollama's whole calls, OpenRouter's comment lines, unnumbered calls and in-stream errors. The turn emits a delta event per piece and a message per round as before, and the chat grows the last answer as deltas arrive. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx
This commit is contained in:
@@ -119,8 +119,10 @@ async def run_turn(
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""One user turn, yielding progress events as it goes.
|
||||
|
||||
A round is appended to *messages* only once every tool in it has answered,
|
||||
so a turn cut off part-way leaves a transcript the provider still accepts.
|
||||
A round's text arrives as ``delta`` events while the model writes it, and
|
||||
once more whole as ``message`` when the round is done. A round is appended
|
||||
to *messages* only once every tool in it has answered, so a turn cut off
|
||||
part-way leaves a transcript the provider still accepts.
|
||||
"""
|
||||
messages.append({"role": "user", "content": text})
|
||||
tools = await tool_specs()
|
||||
@@ -128,18 +130,22 @@ async def run_turn(
|
||||
token = caller_authorization.set(authorization)
|
||||
try:
|
||||
for _ in range(config.max_rounds):
|
||||
choice: dict[str, Any] = {}
|
||||
try:
|
||||
response = await client.chat_completion(
|
||||
async for part in client.stream_completion(
|
||||
api_base=config.api_base,
|
||||
api_key=config.api_key,
|
||||
model=config.model,
|
||||
messages=[system, *messages],
|
||||
tools=tools,
|
||||
)
|
||||
):
|
||||
if isinstance(part, str):
|
||||
yield {"type": "delta", "content": part}
|
||||
else:
|
||||
choice = part
|
||||
except (client.ProviderError, httpx.HTTPError) as exc:
|
||||
yield {"type": "error", "detail": f"The provider answered {exc}"}
|
||||
return
|
||||
choice = response["choices"][0]["message"]
|
||||
content = choice.get("content") or ""
|
||||
calls = choice.get("tool_calls") or []
|
||||
assistant: dict[str, Any] = {"role": "assistant", "content": content}
|
||||
|
||||
@@ -4,10 +4,14 @@ Chat completions with function tools is the dialect every hosted provider and
|
||||
every local server speaks — OpenAI, Anthropic's compatibility endpoint,
|
||||
OpenRouter, Ollama, vLLM — so one client covers them without a provider SDK.
|
||||
The admin types the URL; the model never chooses where a request goes.
|
||||
|
||||
The call is streamed, so the answer reaches the browser as it is written.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -39,7 +43,69 @@ def _detail(response: httpx.Response) -> str:
|
||||
return f"{response.status_code}: {text[:300]}"
|
||||
|
||||
|
||||
async def chat_completion(
|
||||
class Assembler:
|
||||
"""The streamed chunks put back together into the message a plain call returns.
|
||||
|
||||
The providers stream the same ``data:`` lines and differ only in the
|
||||
margins: OpenRouter interleaves ``:`` comment lines, a thinking model's
|
||||
reasoning arrives in a field of its own and is left out, OpenAI and
|
||||
Anthropic send a tool call's arguments in pieces where Ollama sends the call
|
||||
whole, and some leave out the call's ``index``.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.content = ""
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
|
||||
def feed(self, line: str) -> str:
|
||||
"""One line of the stream; returns the text it adds to the answer."""
|
||||
if not line.startswith("data:"):
|
||||
return ""
|
||||
data = line[5:].strip()
|
||||
if not data or data == "[DONE]":
|
||||
return ""
|
||||
try:
|
||||
chunk = json.loads(data)
|
||||
except ValueError as exc:
|
||||
raise ProviderError(f"sent {data[:300]}") from exc
|
||||
error = chunk.get("error")
|
||||
if error:
|
||||
if isinstance(error, dict):
|
||||
error = error.get("message", error)
|
||||
raise ProviderError(str(error)[:300])
|
||||
choices = chunk.get("choices") or [{}]
|
||||
delta = choices[0].get("delta") or {}
|
||||
for piece in delta.get("tool_calls") or []:
|
||||
self._merge(piece)
|
||||
text: str = delta.get("content") or ""
|
||||
self.content += text
|
||||
return text
|
||||
|
||||
def _merge(self, piece: dict[str, Any]) -> None:
|
||||
index = piece.get("index", len(self.calls))
|
||||
while len(self.calls) <= index:
|
||||
self.calls.append(
|
||||
{
|
||||
"id": "",
|
||||
"type": "function",
|
||||
"function": {"name": "", "arguments": ""},
|
||||
}
|
||||
)
|
||||
call = self.calls[index]
|
||||
call["id"] = piece.get("id") or call["id"]
|
||||
function = piece.get("function") or {}
|
||||
call["function"]["name"] = function.get("name") or call["function"]["name"]
|
||||
call["function"]["arguments"] += function.get("arguments") or ""
|
||||
|
||||
def message(self) -> dict[str, Any]:
|
||||
return {
|
||||
"role": "assistant",
|
||||
"content": self.content or None,
|
||||
"tool_calls": self.calls or None,
|
||||
}
|
||||
|
||||
|
||||
async def stream_completion(
|
||||
*,
|
||||
api_base: str,
|
||||
api_key: str,
|
||||
@@ -47,21 +113,33 @@ async def chat_completion(
|
||||
messages: list[dict[str, Any]],
|
||||
tools: list[dict[str, Any]],
|
||||
timeout: float = 120.0,
|
||||
) -> dict[str, Any]:
|
||||
"""``POST {api_base}/chat/completions``, the whole answer at once."""
|
||||
body: dict[str, Any] = {"model": model, "messages": messages}
|
||||
) -> AsyncIterator[str | dict[str, Any]]:
|
||||
"""``POST {api_base}/chat/completions`` streamed.
|
||||
|
||||
Yields each piece of the answer's text as it arrives, then the whole
|
||||
message. The timeout is per chunk, so a long answer is not cut off.
|
||||
"""
|
||||
body: dict[str, Any] = {"model": model, "messages": messages, "stream": True}
|
||||
if tools:
|
||||
body["tools"] = tools
|
||||
async with httpx.AsyncClient(timeout=timeout) as http:
|
||||
response = await http.post(
|
||||
assembler = Assembler()
|
||||
async with (
|
||||
httpx.AsyncClient(timeout=timeout) as http,
|
||||
http.stream(
|
||||
"POST",
|
||||
api_base.rstrip("/") + "/chat/completions",
|
||||
json=body,
|
||||
headers=_headers(api_key),
|
||||
)
|
||||
if response.is_error:
|
||||
raise ProviderError(_detail(response))
|
||||
result: dict[str, Any] = response.json()
|
||||
return result
|
||||
) as response,
|
||||
):
|
||||
if response.is_error:
|
||||
await response.aread()
|
||||
raise ProviderError(_detail(response))
|
||||
async for line in response.aiter_lines():
|
||||
text = assembler.feed(line)
|
||||
if text:
|
||||
yield text
|
||||
yield assembler.message()
|
||||
|
||||
|
||||
async def list_models(api_base: str, api_key: str, timeout: float = 15.0) -> list[str]:
|
||||
|
||||
@@ -259,8 +259,9 @@ def chat(
|
||||
) -> StreamingResponse:
|
||||
"""One turn, streamed as server-sent events.
|
||||
|
||||
``message``, ``tool_call`` and ``tool_result`` events as the model works,
|
||||
``error`` when something stops it, and ``done`` last either way.
|
||||
``delta`` as the model writes, ``message`` with a round's whole text,
|
||||
``tool_call`` and ``tool_result`` as it works, ``error`` when something
|
||||
stops it, and ``done`` last either way.
|
||||
"""
|
||||
config = ai_config.load()
|
||||
if config is None or not config.ready:
|
||||
|
||||
Reference in New Issue
Block a user