Assistant streams its answer token by token

ai/client.py streams the completion and puts the chunks back together
(Assembler), covering OpenAI's piecewise tool-call arguments, Ollama's whole
calls, OpenRouter's comment lines, unnumbered calls and in-stream errors.
The turn emits a delta event per piece and a message per round as before,
and the chat grows the last answer as deltas arrive.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WgBfkm4rCmtd1T4vHWjEPx
This commit is contained in:
2026-10-15 17:37:56 +02:00
co-authored by Claude Opus 5.5
parent 429100a0c7
commit 58b92da8b0
8 changed files with 243 additions and 32 deletions
+11 -5
View File
@@ -119,8 +119,10 @@ async def run_turn(
) -> AsyncIterator[dict[str, Any]]:
"""One user turn, yielding progress events as it goes.
A round is appended to *messages* only once every tool in it has answered,
so a turn cut off part-way leaves a transcript the provider still accepts.
A round's text arrives as ``delta`` events while the model writes it, and
once more whole as ``message`` when the round is done. A round is appended
to *messages* only once every tool in it has answered, so a turn cut off
part-way leaves a transcript the provider still accepts.
"""
messages.append({"role": "user", "content": text})
tools = await tool_specs()
@@ -128,18 +130,22 @@ async def run_turn(
token = caller_authorization.set(authorization)
try:
for _ in range(config.max_rounds):
choice: dict[str, Any] = {}
try:
response = await client.chat_completion(
async for part in client.stream_completion(
api_base=config.api_base,
api_key=config.api_key,
model=config.model,
messages=[system, *messages],
tools=tools,
)
):
if isinstance(part, str):
yield {"type": "delta", "content": part}
else:
choice = part
except (client.ProviderError, httpx.HTTPError) as exc:
yield {"type": "error", "detail": f"The provider answered {exc}"}
return
choice = response["choices"][0]["message"]
content = choice.get("content") or ""
calls = choice.get("tool_calls") or []
assistant: dict[str, Any] = {"role": "assistant", "content": content}
+89 -11
View File
@@ -4,10 +4,14 @@ Chat completions with function tools is the dialect every hosted provider and
every local server speaks — OpenAI, Anthropic's compatibility endpoint,
OpenRouter, Ollama, vLLM — so one client covers them without a provider SDK.
The admin types the URL; the model never chooses where a request goes.
The call is streamed, so the answer reaches the browser as it is written.
"""
from __future__ import annotations
import json
from collections.abc import AsyncIterator
from typing import Any
from urllib.parse import urlparse
@@ -39,7 +43,69 @@ def _detail(response: httpx.Response) -> str:
return f"{response.status_code}: {text[:300]}"
async def chat_completion(
class Assembler:
"""The streamed chunks put back together into the message a plain call returns.
The providers stream the same ``data:`` lines and differ only in the
margins: OpenRouter interleaves ``:`` comment lines, a thinking model's
reasoning arrives in a field of its own and is left out, OpenAI and
Anthropic send a tool call's arguments in pieces where Ollama sends the call
whole, and some leave out the call's ``index``.
"""
def __init__(self) -> None:
self.content = ""
self.calls: list[dict[str, Any]] = []
def feed(self, line: str) -> str:
"""One line of the stream; returns the text it adds to the answer."""
if not line.startswith("data:"):
return ""
data = line[5:].strip()
if not data or data == "[DONE]":
return ""
try:
chunk = json.loads(data)
except ValueError as exc:
raise ProviderError(f"sent {data[:300]}") from exc
error = chunk.get("error")
if error:
if isinstance(error, dict):
error = error.get("message", error)
raise ProviderError(str(error)[:300])
choices = chunk.get("choices") or [{}]
delta = choices[0].get("delta") or {}
for piece in delta.get("tool_calls") or []:
self._merge(piece)
text: str = delta.get("content") or ""
self.content += text
return text
def _merge(self, piece: dict[str, Any]) -> None:
index = piece.get("index", len(self.calls))
while len(self.calls) <= index:
self.calls.append(
{
"id": "",
"type": "function",
"function": {"name": "", "arguments": ""},
}
)
call = self.calls[index]
call["id"] = piece.get("id") or call["id"]
function = piece.get("function") or {}
call["function"]["name"] = function.get("name") or call["function"]["name"]
call["function"]["arguments"] += function.get("arguments") or ""
def message(self) -> dict[str, Any]:
return {
"role": "assistant",
"content": self.content or None,
"tool_calls": self.calls or None,
}
async def stream_completion(
*,
api_base: str,
api_key: str,
@@ -47,21 +113,33 @@ async def chat_completion(
messages: list[dict[str, Any]],
tools: list[dict[str, Any]],
timeout: float = 120.0,
) -> dict[str, Any]:
"""``POST {api_base}/chat/completions``, the whole answer at once."""
body: dict[str, Any] = {"model": model, "messages": messages}
) -> AsyncIterator[str | dict[str, Any]]:
"""``POST {api_base}/chat/completions`` streamed.
Yields each piece of the answer's text as it arrives, then the whole
message. The timeout is per chunk, so a long answer is not cut off.
"""
body: dict[str, Any] = {"model": model, "messages": messages, "stream": True}
if tools:
body["tools"] = tools
async with httpx.AsyncClient(timeout=timeout) as http:
response = await http.post(
assembler = Assembler()
async with (
httpx.AsyncClient(timeout=timeout) as http,
http.stream(
"POST",
api_base.rstrip("/") + "/chat/completions",
json=body,
headers=_headers(api_key),
)
if response.is_error:
raise ProviderError(_detail(response))
result: dict[str, Any] = response.json()
return result
) as response,
):
if response.is_error:
await response.aread()
raise ProviderError(_detail(response))
async for line in response.aiter_lines():
text = assembler.feed(line)
if text:
yield text
yield assembler.message()
async def list_models(api_base: str, api_key: str, timeout: float = 15.0) -> list[str]:
+3 -2
View File
@@ -259,8 +259,9 @@ def chat(
) -> StreamingResponse:
"""One turn, streamed as server-sent events.
``message``, ``tool_call`` and ``tool_result`` events as the model works,
``error`` when something stops it, and ``done`` last either way.
``delta`` as the model writes, ``message`` with a round's whole text,
``tool_call`` and ``tool_result`` as it works, ``error`` when something
stops it, and ``done`` last either way.
"""
config = ai_config.load()
if config is None or not config.ready: