Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Gigaslop oai, message processing and tool calling fixes (#756)

uuuvnb3c021cc52ff
diff --git a/bindings/python/uzu/__init__.pyi b/bindings/python/uzu/__init__.pyi
index 08609d816..5441554bd 100644
--- a/bindings/python/uzu/__init__.pyi
+++ b/bindings/python/uzu/__init__.pyi
@@ -381,6 +381,8 @@ class ChatReplyStats:
@property
def tokens_count_input(self) -> typing.Optional[builtins.int]: ...
@property
+ def tokens_count_input_cached(self) -> typing.Optional[builtins.int]: ...
+ @property
def tokens_count_output(self) -> typing.Optional[builtins.int]: ...
@property
def memory_used_bytes(self) -> typing.Optional[builtins.int]: ...
@@ -392,7 +394,7 @@ class ChatReplyStats:
def tokens_count(self) -> typing.Optional[builtins.int]: ...
@property
def joules_per_token(self) -> typing.Optional[builtins.float]: ...
- def __new__(cls, duration: builtins.float, time_to_first_token: typing.Optional[builtins.float], prefill_tokens_per_second: typing.Optional[builtins.float], generate_tokens_per_second: typing.Optional[builtins.float], tokens_count_input: typing.Optional[builtins.int], tokens_count_output: typing.Optional[builtins.int], memory_used_bytes: typing.Optional[builtins.int], speculator_stats: typing.Optional[ChatReplySpeculatorStats], power_stats: typing.Optional[ChatReplyPowerStats]) -> ChatReplyStats: ...
+ def __new__(cls, duration: builtins.float, time_to_first_token: typing.Optional[builtins.float], prefill_tokens_per_second: typing.Optional[builtins.float], generate_tokens_per_second: typing.Optional[builtins.float], tokens_count_input: typing.Optional[builtins.int], tokens_count_input_cached: typing.Optional[builtins.int], tokens_count_output: typing.Optional[builtins.int], memory_used_bytes: typing.Optional[builtins.int], speculator_stats: typing.Optional[ChatReplySpeculatorStats], power_stats: typing.Optional[ChatReplyPowerStats]) -> ChatReplyStats: ...
class ChatRole:
@typing.final
diff --git a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
index 7c51f5ee9..c9f7e5f2e 100644
--- a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
+++ b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
@@ -1311,6 +1311,7 @@ public struct ChatReplyStats: Equatable, Hashable, Codable {
public var prefillTokensPerSecond: Double?
public var generateTokensPerSecond: Double?
public var tokensCountInput: UInt32?
+ public var tokensCountInputCached: UInt32?
public var tokensCountOutput: UInt32?
public var memoryUsedBytes: Int64?
public var speculatorStats: ChatReplySpeculatorStats?
@@ -1318,12 +1319,13 @@ public struct ChatReplyStats: Equatable, Hashable, Codable {
// Default memberwise initializers are never public by default, so we
// declare one manually.
- public init(duration: Double, timeToFirstToken: Double?, prefillTokensPerSecond: Double?, generateTokensPerSecond: Double?, tokensCountInput: UInt32?, tokensCountOutput: UInt32?, memoryUsedBytes: Int64?, speculatorStats: ChatReplySpeculatorStats?, powerStats: ChatReplyPowerStats?) {
+ public init(duration: Double, timeToFirstToken: Double?, prefillTokensPerSecond: Double?, generateTokensPerSecond: Double?, tokensCountInput: UInt32?, tokensCountInputCached: UInt32?, tokensCountOutput: UInt32?, memoryUsedBytes: Int64?, speculatorStats: ChatReplySpeculatorStats?, powerStats: ChatReplyPowerStats?) {
self.duration = duration
self.timeToFirstToken = timeToFirstToken
self.prefillTokensPerSecond = prefillTokensPerSecond
self.generateTokensPerSecond = generateTokensPerSecond
self.tokensCountInput = tokensCountInput
+ self.tokensCountInputCached = tokensCountInputCached
self.tokensCountOutput = tokensCountOutput
self.memoryUsedBytes = memoryUsedBytes
self.speculatorStats = speculatorStats
@@ -1370,6 +1372,7 @@ public struct FfiConverterTypeChatReplyStats: FfiConverterRustBuffer {
prefillTokensPerSecond: FfiConverterOptionDouble.read(from: &buf),
generateTokensPerSecond: FfiConverterOptionDouble.read(from: &buf),
tokensCountInput: FfiConverterOptionUInt32.read(from: &buf),
+ tokensCountInputCached: FfiConverterOptionUInt32.read(from: &buf),
tokensCountOutput: FfiConverterOptionUInt32.read(from: &buf),
memoryUsedBytes: FfiConverterOptionInt64.read(from: &buf),
speculatorStats: FfiConverterOptionTypeChatReplySpeculatorStats.read(from: &buf),
@@ -1383,6 +1386,7 @@ public struct FfiConverterTypeChatReplyStats: FfiConverterRustBuffer {
FfiConverterOptionDouble.write(value.prefillTokensPerSecond, into: &buf)
FfiConverterOptionDouble.write(value.generateTokensPerSecond, into: &buf)
FfiConverterOptionUInt32.write(value.tokensCountInput, into: &buf)
+ FfiConverterOptionUInt32.write(value.tokensCountInputCached, into: &buf)
FfiConverterOptionUInt32.write(value.tokensCountOutput, into: &buf)
FfiConverterOptionInt64.write(value.memoryUsedBytes, into: &buf)
FfiConverterOptionTypeChatReplySpeculatorStats.write(value.speculatorStats, into: &buf)
diff --git a/bindings/typescript/src/napi/index.d.mts b/bindings/typescript/src/napi/index.d.mts
index 801692db9..92f9c1556 100644
--- a/bindings/typescript/src/napi/index.d.mts
+++ b/bindings/typescript/src/napi/index.d.mts
@@ -310,11 +310,12 @@ export declare class ChatReplyStats {
prefillTokensPerSecond?: number
generateTokensPerSecond?: number
tokensCountInput?: number
+ tokensCountInputCached?: number
tokensCountOutput?: number
memoryUsedBytes?: number
speculatorStats?: ChatReplySpeculatorStats
powerStats?: ChatReplyPowerStats
- constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, powerStats?: ChatReplyPowerStats)
+ constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountInputCached?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, powerStats?: ChatReplyPowerStats)
get tokensCount(): number | null
/** Energy spent per processed token, counting both input and output tokens. */
get joulesPerToken(): number | null
diff --git a/bindings/typescript/src/napi/index.d.ts b/bindings/typescript/src/napi/index.d.ts
index 801692db9..92f9c1556 100644
--- a/bindings/typescript/src/napi/index.d.ts
+++ b/bindings/typescript/src/napi/index.d.ts
@@ -310,11 +310,12 @@ export declare class ChatReplyStats {
prefillTokensPerSecond?: number
generateTokensPerSecond?: number
tokensCountInput?: number
+ tokensCountInputCached?: number
tokensCountOutput?: number
memoryUsedBytes?: number
speculatorStats?: ChatReplySpeculatorStats
powerStats?: ChatReplyPowerStats
- constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, powerStats?: ChatReplyPowerStats)
+ constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountInputCached?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, powerStats?: ChatReplyPowerStats)
get tokensCount(): number | null
/** Energy spent per processed token, counting both input and output tokens. */
get joulesPerToken(): number | null
diff --git a/crates/legacy/backend-remote/src/openai/bridging/chat/stream_chunk.rs b/crates/legacy/backend-remote/src/openai/bridging/chat/stream_chunk.rs
index 21304056d..222ac9c6f 100644
--- a/crates/legacy/backend-remote/src/openai/bridging/chat/stream_chunk.rs
+++ b/crates/legacy/backend-remote/src/openai/bridging/chat/stream_chunk.rs
@@ -53,12 +53,25 @@ struct Usage {
prompt_tokens: Option<u32>,
#[serde(default)]
completion_tokens: Option<u32>,
+ #[serde(default)]
+ prompt_tokens_details: Option<PromptTokensDetails>,
+}
+
+#[derive(Debug, Deserialize)]
+struct PromptTokensDetails {
+ #[serde(default)]
+ cached_tokens: Option<u32>,
}
pub fn build(response: Response) -> Option<StreamChunk> {
let usage = response.usage;
let choice = response.choices.into_iter().next();
- let tokens_input = usage.as_ref().and_then(|usage| usage.prompt_tokens);
+ let tokens_input_cached =
+ usage.as_ref().and_then(|usage| usage.prompt_tokens_details.as_ref()).and_then(|details| details.cached_tokens);
+ let tokens_input = usage
+ .as_ref()
+ .and_then(|usage| usage.prompt_tokens)
+ .map(|tokens| tokens.saturating_sub(tokens_input_cached.unwrap_or(0)));
let tokens_output = usage.as_ref().and_then(|usage| usage.completion_tokens);
let (content, tool_calls, finish_reason) = match choice {
@@ -88,6 +101,7 @@ pub fn build(response: Response) -> Option<StreamChunk> {
&& tool_calls.is_empty()
&& finish_reason.is_none()
&& tokens_input.is_none()
+ && tokens_input_cached.is_none()
&& tokens_output.is_none()
{
return None;
@@ -99,6 +113,7 @@ pub fn build(response: Response) -> Option<StreamChunk> {
tool_calls,
finish_reason,
tokens_input,
+ tokens_input_cached,
tokens_output,
})
}
diff --git a/crates/legacy/backend-remote/src/openai/bridging/responses/stream_chunk.rs b/crates/legacy/backend-remote/src/openai/bridging/responses/stream_chunk.rs
index d7357fbd7..f1a9fba28 100644
--- a/crates/legacy/backend-remote/src/openai/bridging/responses/stream_chunk.rs
+++ b/crates/legacy/backend-remote/src/openai/bridging/responses/stream_chunk.rs
@@ -1,10 +1,15 @@
-use async_openai::types::responses::{OutputItem, ResponseStreamEvent};
+use async_openai::types::responses::{OutputItem, ResponseStreamEvent, ResponseUsage};
use crate::openai::{
bridging::responses::finish_reason,
stream_state::{StreamChunk, ToolCallChunk},
};
+fn usage_parts(usage: &ResponseUsage) -> (Option<u32>, Option<u32>, Option<u32>) {
+ let cached = usage.input_tokens_details.cached_tokens;
+ (Some(usage.input_tokens.saturating_sub(cached)), Some(cached), Some(usage.output_tokens))
+}
+
pub fn build(event: ResponseStreamEvent) -> Option<StreamChunk> {
match event {
ResponseStreamEvent::ResponseOutputTextDelta(event) => Some(StreamChunk {
@@ -40,24 +45,39 @@ pub fn build(event: ResponseStreamEvent) -> Option<StreamChunk> {
}],
..StreamChunk::default()
}),
- ResponseStreamEvent::ResponseCompleted(event) => Some(StreamChunk {
- finish_reason: finish_reason::build(&event.response.status),
- tokens_input: event.response.usage.as_ref().map(|usage| usage.input_tokens),
- tokens_output: event.response.usage.as_ref().map(|usage| usage.output_tokens),
- ..StreamChunk::default()
- }),
- ResponseStreamEvent::ResponseFailed(event) => Some(StreamChunk {
- finish_reason: finish_reason::build(&event.response.status),
- tokens_input: event.response.usage.as_ref().map(|usage| usage.input_tokens),
- tokens_output: event.response.usage.as_ref().map(|usage| usage.output_tokens),
- ..StreamChunk::default()
- }),
- ResponseStreamEvent::ResponseIncomplete(event) => Some(StreamChunk {
- finish_reason: finish_reason::build(&event.response.status),
- tokens_input: event.response.usage.as_ref().map(|usage| usage.input_tokens),
- tokens_output: event.response.usage.as_ref().map(|usage| usage.output_tokens),
- ..StreamChunk::default()
- }),
+ ResponseStreamEvent::ResponseCompleted(event) => {
+ let (tokens_input, tokens_input_cached, tokens_output) =
+ event.response.usage.as_ref().map(usage_parts).unwrap_or_default();
+ Some(StreamChunk {
+ finish_reason: finish_reason::build(&event.response.status),
+ tokens_input,
+ tokens_input_cached,
+ tokens_output,
+ ..StreamChunk::default()
+ })
+ },
+ ResponseStreamEvent::ResponseFailed(event) => {
+ let (tokens_input, tokens_input_cached, tokens_output) =
+ event.response.usage.as_ref().map(usage_parts).unwrap_or_default();
+ Some(StreamChunk {
+ finish_reason: finish_reason::build(&event.response.status),
+ tokens_input,
+ tokens_input_cached,
+ tokens_output,
+ ..StreamChunk::default()
+ })
+ },
+ ResponseStreamEvent::ResponseIncomplete(event) => {
+ let (tokens_input, tokens_input_cached, tokens_output) =
+ event.response.usage.as_ref().map(usage_parts).unwrap_or_default();
+ Some(StreamChunk {
+ finish_reason: finish_reason::build(&event.response.status),
+ tokens_input,
+ tokens_input_cached,
+ tokens_output,
+ ..StreamChunk::default()
+ })
+ },
_ => None,
}
}
diff --git a/crates/legacy/backend-remote/src/openai/stream_state.rs b/crates/legacy/backend-remote/src/openai/stream_state.rs
index f7c125af5..6f69cd090 100644
--- a/crates/legacy/backend-remote/src/openai/stream_state.rs
+++ b/crates/legacy/backend-remote/src/openai/stream_state.rs
@@ -23,6 +23,7 @@ pub struct StreamChunk {
pub tool_calls: Vec<ToolCallChunk>,
pub finish_reason: Option<ChatReplyFinishReason>,
pub tokens_input: Option<u32>,
+ pub tokens_input_cached: Option<u32>,
pub tokens_output: Option<u32>,
}
@@ -35,6 +36,7 @@ pub struct StreamState {
start_moment: Instant,
first_token_moment: Option<Instant>,
tokens_input: Option<u32>,
+ tokens_input_cached: Option<u32>,
tokens_output: Option<u32>,
}
@@ -48,6 +50,7 @@ impl StreamState {
start_moment: Instant::now(),
first_token_moment: None,
tokens_input: None,
+ tokens_input_cached: None,
tokens_output: None,
}
}
@@ -85,6 +88,11 @@ impl StreamState {
processed = true;
}
+ if let Some(tokens) = chunk.tokens_input_cached {
+ self.tokens_input_cached = Some(tokens);
+ processed = true;
+ }
+
if let Some(tokens) = chunk.tokens_output {
self.tokens_output = Some(tokens);
processed = true;
@@ -141,6 +149,7 @@ impl StreamState {
prefill_tokens_per_second,
generate_tokens_per_second,
tokens_count_input: self.tokens_input,
+ tokens_count_input_cached: self.tokens_input_cached,
tokens_count_output: self.tokens_output,
memory_used_bytes: None,
speculator_stats: None,
diff --git a/crates/legacy/cli/src/main.rs b/crates/legacy/cli/src/main.rs
index 4864a58c3..ac57345c1 100644
--- a/crates/legacy/cli/src/main.rs
+++ b/crates/legacy/cli/src/main.rs
@@ -42,6 +42,12 @@ enum Commands {
port: u16,
#[arg(long, default_value = "127.0.0.1")]
host: String,
+ /// Reuse the previous request's state when the new prompt extends it (enabled by default).
+ #[arg(long = "prefix-cache", action = clap::ArgAction::SetTrue, conflicts_with = "no_prefix_cache")]
+ prefix_cache: bool,
+ /// Reset the session on every request, disabling prefix cache reuse.
+ #[arg(long = "no-prefix-cache", action = clap::ArgAction::SetTrue)]
+ no_prefix_cache: bool,
},
Storage {
#[arg(long, value_enum, default_value_t = storage::DownloadManagerCliType::default())]
@@ -67,7 +73,9 @@ async fn main() -> Result<()> {
model,
port,
host,
- }) => server::run_server(model, host, port).await?,
+ prefix_cache,
+ no_prefix_cache,
+ }) => server::run_server(model, host, port, prefix_cache || !no_prefix_cache).await?,
Some(Commands::Storage {
download_manager,
}) => storage::run(download_manager).await?,
diff --git a/crates/legacy/cli/src/server/chat_completions.rs b/crates/legacy/cli/src/server/chat_completions.rs
index 50755dc2b..614b42055 100644
--- a/crates/legacy/cli/src/server/chat_completions.rs
+++ b/crates/legacy/cli/src/server/chat_completions.rs
@@ -1,4 +1,5 @@
use std::{
+ collections::HashMap,
pin::Pin,
str::FromStr,
sync::Arc,
@@ -7,6 +8,7 @@ use std::{
use rocket::{
Request, State,
+ data::{ByteUnit, Data},
futures::Stream,
http::Status,
post,
@@ -21,10 +23,12 @@ use tokio::sync::{Mutex, mpsc};
use tokio_stream::wrappers::UnboundedReceiverStream;
use uuid::Uuid;
use uzu::{
- session::chat::{ChatSession, ChatSessionStreamChunk},
+ session::chat::{ChatSession, ChatSessionStreamChunk, UNPARSED_ARGUMENTS_KEY},
types::{
basic::{Grammar, ReasoningEffort, SamplingMethod},
- session::chat::{ChatMessage, ChatReplyConfig, ChatReplyFinishReason, ChatReplyStats, ChatRole},
+ session::chat::{
+ ChatContentBlock, ChatMessage, ChatReplyConfig, ChatReplyFinishReason, ChatReplyStats, ChatRole,
+ },
},
};
@@ -33,16 +37,45 @@ use crate::{
server::{
ServerState,
chat_tool_calls::{
- OaiTool, OaiToolCall, backfill_tool_result_names, choose_tools, insert_tools_message, reply_tool_calls,
- to_tool_call, tool_call_deltas, tool_call_result_block, withhold_stream_text,
+ OaiTool, OaiToolCall, ToolCallStreamer, backfill_tool_result_names, choose_tools, insert_tools_message,
+ oai_tool_call, reply_tool_calls, to_tool_call, tool_call_result_block, withhold_stream_text,
},
+ request_log::RequestLog,
},
};
+fn deserialize_message_content<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
+where
+ D: serde::Deserializer<'de>,
+{
+ match Option::<serde_json::Value>::deserialize(deserializer)? {
+ None => Ok(None),
+ Some(serde_json::Value::String(text)) => Ok(Some(text)),
+ Some(serde_json::Value::Array(parts)) => {
+ let mut text = String::new();
+ for part in &parts {
+ match part.get("type").and_then(serde_json::Value::as_str) {
+ Some("text") => {
+ text.push_str(part.get("text").and_then(serde_json::Value::as_str).unwrap_or_default());
+ },
+ Some(other) => {
+ return Err(serde::de::Error::custom(format!(
+ "unsupported message content part type: {other}"
+ )));
+ },
+ None => return Err(serde::de::Error::custom("message content part is missing a type")),
+ }
+ }
+ Ok(Some(text))
+ },
+ Some(_) => Err(serde::de::Error::custom("message content must be a string or an array of content parts")),
+ }
+}
+
#[derive(Serialize, Deserialize, Clone)]
pub struct OaiMessage {
pub role: String,
- #[serde(default)]
+ #[serde(default, deserialize_with = "deserialize_message_content")]
pub content: Option<String>,
// vLLM-style reasoning channel.
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -112,12 +145,19 @@ pub struct ChatCompletionChoice {
pub finish_reason: String,
}
+#[derive(Serialize, Clone)]
+pub struct ChatCompletionPromptTokensDetails {
+ pub cached_tokens: u32,
+}
+
#[derive(Serialize, Clone, Default)]
pub struct ChatCompletionUsage {
pub prompt_tokens: u32,
pub completion_tokens: u32,
pub total_tokens: u32,
#[serde(skip_serializing_if = "Option::is_none")]
+ pub prompt_tokens_details: Option<ChatCompletionPromptTokensDetails>,
+ #[serde(skip_serializing_if = "Option::is_none")]
pub spec_verify_ct: Option<u32>,
}
@@ -138,6 +178,8 @@ struct StreamDelta {
#[serde(skip_serializing_if = "Option::is_none")]
content: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
+ reasoning_content: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
tool_calls: Option<Vec<OaiToolCall>>,
}
@@ -202,6 +244,12 @@ fn to_chat_messages(messages: &[OaiMessage]) -> Vec<ChatMessage> {
.map(|message| {
let role = ChatRole::from_str(&message.role).unwrap_or(ChatRole::User {});
let mut chat_message = ChatMessage::for_role(role);
+ // Block order matches how the session stores generated replies
+ // (reasoning, then text, then tool calls) so a client that echoes
+ // reasoning back keeps the message prefix intact.
+ if let Some(reasoning) = message.reasoning_content.as_ref().filter(|reasoning| !reasoning.is_empty()) {
+ chat_message = chat_message.with_reasoning(reasoning.clone());
+ }
if let Some(identifier) = &message.tool_call_id {
let result = tool_call_result_block(identifier, message.content.clone().unwrap_or_default());
chat_message = chat_message.with_block(result);
@@ -354,8 +402,15 @@ pub(crate) fn build_messages(
if let Some((effort, source)) = requested_reasoning_effort(request)? {
let fulfilled = thinking_support.fulfill_requested_effort(effort).map_err(|detail| source.error(detail))?;
if let Some(effort) = fulfilled {
- // The engine reads the effort from a reasoning_effort block carried on a system message.
- messages.insert(0, ChatMessage::system().with_reasoning_effort(effort));
+ // The engine reads the effort from a reasoning_effort block carried on a system
+ // message. Merge into a leading system message when the client sent one:
+ // templates reject two system messages in a row.
+ match messages.first_mut() {
+ Some(first) if first.role == (ChatRole::System {}) => {
+ *first = first.clone().with_reasoning_effort(effort);
+ },
+ _ => messages.insert(0, ChatMessage::system().with_reasoning_effort(effort)),
+ }
}
}
backfill_tool_result_names(&mut messages);
@@ -480,13 +535,21 @@ fn map_finish_reason(finish_reason: &ChatReplyFinishReason) -> String {
.to_string()
}
-fn usage_from_stats(stats: &ChatReplyStats) -> ChatCompletionUsage {
- let prompt_tokens = stats.tokens_count_input.unwrap_or(0);
+fn usage_from_stats(
+ stats: &ChatReplyStats,
+ prefix_cache: bool,
+) -> ChatCompletionUsage {
+ let cached_tokens = stats.tokens_count_input_cached.unwrap_or(0);
+ let prefilled_tokens = stats.tokens_count_input.unwrap_or(0);
+ let prompt_tokens = prefilled_tokens + cached_tokens;
let completion_tokens = stats.tokens_count_output.unwrap_or(0);
ChatCompletionUsage {
prompt_tokens,
completion_tokens,
total_tokens: prompt_tokens + completion_tokens,
+ prompt_tokens_details: prefix_cache.then_some(ChatCompletionPromptTokensDetails {
+ cached_tokens,
+ }),
spec_verify_ct: stats
.speculator_stats
.as_ref()
@@ -543,6 +606,57 @@ fn chunk_json(
serde_json::to_string(&chunk).unwrap_or_default()
}
+/// Tool call identifiers are assigned per turn and never rendered into tokens, so they are
+/// excluded from the comparison; everything else about the messages must match.
+fn messages_have_prefix(
+ messages: &[ChatMessage],
+ prefix: &[ChatMessage],
+) -> bool {
+ prefix.len() <= messages.len()
+ && prefix.iter().zip(messages).all(|(prefix_message, message)| {
+ prefix_message.role == message.role
+ && prefix_message.metadata == message.metadata
+ && prefix_message.content.len() == message.content.len()
+ && prefix_message.content.iter().zip(&message.content).all(|(prefix_block, block)| {
+ match (prefix_block, block) {
+ (
+ ChatContentBlock::ToolCall {
+ value: prefix_call,
+ },
+ ChatContentBlock::ToolCall {
+ value: call,
+ },
+ ) => {
+ let (mut prefix_call, mut call) = (prefix_call.clone(), call.clone());
+ prefix_call.identifier = None;
+ call.identifier = None;
+ prefix_call == call
+ },
+ _ => prefix_block == block,
+ }
+ })
+ })
+}
+
+/// Returns only the messages the session has not seen yet when the request
+/// extends the session's current history; otherwise resets the session and
+/// returns the full list. Token-level reuse itself is decided inside the
+/// session, which falls back to a full prefill if the rendered tokens diverge.
+async fn prepare_input(
+ session: &ChatSession,
+ mut messages: Vec<ChatMessage>,
+ prefix_cache: bool,
+) -> Result<Vec<ChatMessage>, uzu::session::chat::ChatSessionError> {
+ if prefix_cache {
+ let current = session.messages().await;
+ if !current.is_empty() && messages.len() > current.len() && messages_have_prefix(&messages, &current) {
+ return Ok(messages.split_off(current.len()));
+ }
+ }
+ session.reset().await?;
+ Ok(messages)
+}
+
async fn run_blocking(
session: Arc<Mutex<ChatSession>>,
messages: Vec<ChatMessage>,
@@ -550,13 +664,19 @@ async fn run_blocking(
id: String,
model: String,
created: i64,
+ prefix_cache: bool,
+ log: RequestLog,
) -> ChatCompletionResponse {
let session = session.lock().await;
- if let Err(error) = session.reset().await {
- return error_response(id, model, created, &error.to_string());
- }
+ let input = match prepare_input(&session, messages, prefix_cache).await {
+ Ok(input) => input,
+ Err(error) => {
+ log.fail(&error.to_string());
+ return error_response(id, model, created, &error.to_string());
+ },
+ };
- match session.reply(messages, config).await {
+ match session.reply(input, config).await {
Ok(replies) => match replies.last() {
Some(reply) => {
let tool_calls = reply_tool_calls(&reply.message);
@@ -564,11 +684,32 @@ async fn run_blocking(
let content = reply.message.text().or_else(|| tool_calls.is_none().then(String::new));
let mut finish_reason =
reply.finish_reason.as_ref().map(map_finish_reason).unwrap_or_else(|| "stop".to_string());
- // A call the parser could not finalize yields ToolCalls with nothing to execute.
- // Clients would stall on an absent tool_calls array, so report stop.
- if tool_calls.is_none() && finish_reason == "tool_calls" {
- finish_reason = "stop".to_string();
+ // Calls the parser could not finalize must not execute as a partial batch.
+ // OpenAI has no finish reason for this; providers signal terminal errors with
+ // unrecognized reasons (pi maps them to a thrown provider error), so the turn
+ // fails loudly instead of stalling on an absent or incomplete tool_calls array.
+ let has_unfinished_candidates = reply
+ .message
+ .content
+ .iter()
+ .any(|block| matches!(block, ChatContentBlock::ToolCallCandidate { .. }));
+ if finish_reason == "tool_calls" && (tool_calls.is_none() || has_unfinished_candidates) {
+ finish_reason = "malformed_tool_call".to_string();
}
+ let wrapped_arguments = reply
+ .message
+ .tool_calls()
+ .iter()
+ .filter(|call| call.arguments.json.contains(UNPARSED_ARGUMENTS_KEY))
+ .count();
+ let mut notes = Vec::new();
+ if has_unfinished_candidates {
+ notes.push("model generated unfinished tool call(s)".to_string());
+ }
+ if wrapped_arguments > 0 {
+ notes.push(format!("{wrapped_arguments} tool call(s) with unparseable arguments"));
+ }
+ log.finish(&finish_reason, Some(&reply.stats), notes);
ChatCompletionResponse {
id,
object: "chat.completion".to_string(),
@@ -585,12 +726,18 @@ async fn run_blocking(
},
finish_reason,
}],
- usage: usage_from_stats(&reply.stats),
+ usage: usage_from_stats(&reply.stats, prefix_cache),
}
},
- None => error_response(id, model, created, "No response generated"),
+ None => {
+ log.fail("no response generated");
+ error_response(id, model, created, "No response generated")
+ },
+ },
+ Err(error) => {
+ log.fail(&error.to_string());
+ error_response(id, model, created, &error.to_string())
},
- Err(error) => error_response(id, model, created, &error.to_string()),
}
}
@@ -601,24 +748,31 @@ async fn run_stream(
id: String,
model: String,
created: i64,
+ prefix_cache: bool,
sender: mpsc::UnboundedSender<Event>,
+ log: RequestLog,
) {
let session = session.lock().await;
- if let Err(error) = session.reset().await {
- let _ = sender.send(Event::data(chunk_json(
- &id,
- &model,
- created,
- StreamDelta {
- content: Some(format!("Error: {error}")),
- ..StreamDelta::default()
- },
- Some("stop".to_string()),
- None,
- )));
- let _ = sender.send(Event::data("[DONE]"));
- return;
- }
+ let has_tools = messages.iter().any(|message| !message.tool_namespaces().is_empty());
+ let input = match prepare_input(&session, messages, prefix_cache).await {
+ Ok(input) => input,
+ Err(error) => {
+ log.fail(&error.to_string());
+ let _ = sender.send(Event::data(chunk_json(
+ &id,
+ &model,
+ created,
+ StreamDelta {
+ content: Some(format!("Error: {error}")),
+ ..StreamDelta::default()
+ },
+ Some("stop".to_string()),
+ None,
+ )));
+ let _ = sender.send(Event::data("[DONE]"));
+ return;
+ },
+ };
let _ = sender.send(Event::data(chunk_json(
&id,
@@ -632,16 +786,36 @@ async fn run_stream(
None,
)));
- let has_tools = messages.iter().any(|message| !message.tool_namespaces().is_empty());
- let stream = session.reply_with_stream(messages, config).await;
+ let stream = session.reply_with_stream(input, config).await;
+ let cancel_token = stream.cancel_token();
let mut emitted = 0usize;
+ let mut emitted_reasoning = 0usize;
let mut emitted_tool_calls = 0usize;
+ let mut tool_call_streamers: HashMap<usize, ToolCallStreamer> = HashMap::new();
let mut final_text = String::new();
let mut finish_reason = "stop".to_string();
- let mut usage = ChatCompletionUsage::default();
+ let mut final_stats: Option<ChatReplyStats> = None;
+ let mut final_had_candidates = false;
+ let mut wrapped_arguments = 0usize;
let mut errored = false;
- while let Some(chunk) = stream.next().await {
+ loop {
+ // Rocket drops the SSE receiver as soon as the connection dies, so racing each chunk
+ // against the channel closing notices a disconnect even while generation is still in
+ // prefill and no chunk is on its way. Cancelling stops the backend at the next token
+ // boundary and lets the session run its cancelled-turn history cleanup.
+ let chunk = tokio::select! {
+ chunk = stream.next() => chunk,
+ () = sender.closed() => {
+ cancel_token.cancel();
+ cancel_token.cancel();
+ log.fail("client disconnected");
+ return;
+ },
+ };
+ let Some(chunk) = chunk else {
+ break;
+ };
match chunk {
ChatSessionStreamChunk::Replies {
replies,
@@ -649,6 +823,30 @@ async fn run_stream(
let Some(reply) = replies.last() else {
continue;
};
+ let reasoning = reply.message.reasoning().unwrap_or_default();
+ let reasoning_start = (emitted_reasoning..=reasoning.len())
+ .find(|&index| reasoning.is_char_boundary(index))
+ .unwrap_or(reasoning.len());
+ if reasoning.len() > reasoning_start {
+ let delta = reasoning[reasoning_start..].to_string();
+ emitted_reasoning = reasoning.len();
+ let sent = sender.send(Event::data(chunk_json(
+ &id,
+ &model,
+ created,
+ StreamDelta {
+ reasoning_content: Some(delta),
+ ..StreamDelta::default()
+ },
+ None,
+ None,
+ )));
+ if sent.is_err() {
+ cancel_token.cancel();
+ log.fail("client disconnected");
+ return;
+ }
+ }
let text = reply.message.text().unwrap_or_default();
let start = (emitted..=text.len()).find(|&index| text.is_char_boundary(index)).unwrap_or(text.len());
if !withhold_stream_text(has_tools, &text) && text.len() > start {
@@ -666,26 +864,72 @@ async fn run_stream(
None,
)));
if sent.is_err() {
+ cancel_token.cancel();
+ log.fail("client disconnected");
return;
}
}
+ let finished_count = reply.message.tool_calls().len();
+ let candidates = reply.message.content.iter().filter_map(|block| match block {
+ ChatContentBlock::ToolCallCandidate {
+ value,
+ } => Some(value),
+ _ => None,
+ });
+ for (offset, value) in candidates.enumerate() {
+ let index = finished_count + offset;
+ // candidates carry their partial text as a JSON string document, except
+ // object-valued ones (e.g. muse-glimmer's name-only progress objects)
+ let raw = serde_json::from_str::<String>(&value.json).unwrap_or_else(|_| value.json.clone());
+ let deltas =
+ tool_call_streamers.entry(index).or_insert_with(ToolCallStreamer::new).update(index, &raw);
+ for delta in deltas {
+ let sent = sender.send(Event::data(chunk_json(
+ &id,
+ &model,
+ created,
+ StreamDelta {
+ tool_calls: Some(vec![delta]),
+ ..StreamDelta::default()
+ },
+ None,
+ None,
+ )));
+ if sent.is_err() {
+ cancel_token.cancel();
+ log.fail("client disconnected");
+ return;
+ }
+ }
+ }
+
let tool_calls = reply.message.tool_calls();
- if tool_calls.len() > emitted_tool_calls {
- let delta = tool_call_deltas(&tool_calls, emitted_tool_calls);
- emitted_tool_calls = tool_calls.len();
+ while emitted_tool_calls < tool_calls.len() {
+ let index = emitted_tool_calls;
+ emitted_tool_calls += 1;
+ let call = &tool_calls[index];
+ if call.arguments.json.contains(UNPARSED_ARGUMENTS_KEY) {
+ wrapped_arguments += 1;
+ }
+ let delta = match tool_call_streamers.remove(&index) {
+ Some(mut streamer) => streamer.finish(index, call),
+ None => oai_tool_call(Some(index), call),
+ };
let sent = sender.send(Event::data(chunk_json(
&id,
&model,
created,
StreamDelta {
- tool_calls: Some(delta),
+ tool_calls: Some(vec![delta]),
..StreamDelta::default()
},
None,
None,
)));
if sent.is_err() {
+ cancel_token.cancel();
+ log.fail("client disconnected");
return;
}
}
@@ -693,13 +937,19 @@ async fn run_stream(
if let Some(reason) = &reply.finish_reason {
finish_reason = map_finish_reason(reason);
}
- usage = usage_from_stats(&reply.stats);
+ final_stats = Some(reply.stats.clone());
final_text = text;
+ final_had_candidates = reply
+ .message
+ .content
+ .iter()
+ .any(|block| matches!(block, ChatContentBlock::ToolCallCandidate { .. }));
},
ChatSessionStreamChunk::Error {
error,
} => {
errored = true;
+ log.fail(&error.to_string());
let _ = sender.send(Event::data(chunk_json(
&id,
&model,
@@ -734,14 +984,28 @@ async fn run_stream(
None,
)));
if sent.is_err() {
+ cancel_token.cancel();
+ log.fail("client disconnected");
return;
}
}
- // Same guard as the blocking path: candidates that never finalized produce a ToolCalls finish with
- // no emitted tool call deltas, which reads as stop.
- if emitted_tool_calls == 0 && finish_reason == "tool_calls" {
- finish_reason = "stop".to_string();
+ // Same guard as the blocking path: calls the parser could not finalize must not
+ // execute as a partial batch, so the turn reports a malformed tool call instead of
+ // stalling on an absent or incomplete tool_calls array.
+ if finish_reason == "tool_calls" && (emitted_tool_calls == 0 || final_had_candidates) {
+ finish_reason = "malformed_tool_call".to_string();
+ }
+ let mut notes = Vec::new();
+ if final_had_candidates {
+ notes.push("model generated unfinished tool call(s)".to_string());
}
+ if wrapped_arguments > 0 {
+ notes.push(format!("{wrapped_arguments} tool call(s) with unparseable arguments"));
+ }
+ log.finish(&finish_reason, final_stats.as_ref(), notes);
+ let usage = final_stats
+ .as_ref()
+ .map_or_else(ChatCompletionUsage::default, |stats| usage_from_stats(stats, prefix_cache));
let _ = sender.send(Event::data(chunk_json(
&id,
&model,
@@ -755,35 +1019,80 @@ async fn run_stream(
}
#[allow(private_interfaces)]
-#[post("/chat/completions", format = "json", data = "<request>")]
+#[post("/chat/completions", format = "json", data = "<body>")]
pub async fn handle_chat_completions(
- request: Json<ChatCompletionRequest>,
+ body: Data<'_>,
state: &State<ServerState>,
) -> ChatCompletionResult {
- let request = request.into_inner();
+ let body = match body.open(ByteUnit::Mebibyte(64)).into_string().await {
+ Ok(body) if body.is_complete() => body.into_inner(),
+ Ok(_) => {
+ return invalid_request_response("body", "request_too_large", "request body exceeds 64 MiB".to_string());
+ },
+ Err(error) => {
+ return invalid_request_response("body", "invalid_body", format!("failed to read request body: {error}"));
+ },
+ };
+ let request = match serde_json::from_str::<ChatCompletionRequest>(&body) {
+ Ok(request) => request,
+ Err(error) => {
+ RequestLog::rejected(&format!("failed to parse chat completion request: {error}"));
+ return invalid_request_response(
+ "body",
+ "invalid_request",
+ format!("failed to parse chat completion request: {error}"),
+ );
+ },
+ };
let id = format!("chatcmpl-{}", Uuid::new_v4().simple());
let created = now_unix();
let model = state.model_name.clone();
let is_stream = request.stream.unwrap_or(false);
+ let log = RequestLog::start(
+ &id,
+ is_stream,
+ request.messages.len(),
+ request.tools.as_ref().map_or(0, Vec::len),
+ request.reasoning_effort.as_ref().and_then(serde_json::Value::as_str).or(Some("unspecified")),
+ );
let config = match build_reply_config(&request) {
Ok(config) => config,
- Err(error) => return invalid_request_response("response_format", error.code(), error.message()),
+ Err(error) => {
+ log.fail(&error.message());
+ return invalid_request_response("response_format", error.code(), error.message());
+ },
};
let messages = match build_messages(&request, state.thinking_support) {
Ok(messages) => messages,
- Err(error) => return invalid_request_response(error.param(), error.code(), error.into_detail()),
+ Err(error) => {
+ let param = error.param();
+ let code = error.code();
+ let detail = error.into_detail();
+ log.fail(&detail);
+ return invalid_request_response(param, code, detail);
+ },
};
if is_stream {
let session = Arc::clone(&state.session);
let (sender, receiver) = mpsc::unbounded_channel::<Event>();
- rocket::tokio::spawn(run_stream(session, messages, config, id, model, created, sender));
+ rocket::tokio::spawn(run_stream(
+ session,
+ messages,
+ config,
+ id,
+ model,
+ created,
+ state.prefix_cache,
+ sender,
+ log,
+ ));
let body: Pin<Box<dyn Stream<Item = Event> + Send>> = Box::pin(UnboundedReceiverStream::new(receiver));
ChatCompletionResult::Stream(EventStream::from(body))
} else {
let session = Arc::clone(&state.session);
- let response = run_blocking(session, messages, config, id, model, created).await;
+ let response = run_blocking(session, messages, config, id, model, created, state.prefix_cache, log).await;
ChatCompletionResult::Json(Json(response))
}
}
diff --git a/crates/legacy/cli/src/server/chat_tool_calls.rs b/crates/legacy/cli/src/server/chat_tool_calls.rs
index 4b6339353..5ac1e0851 100644
--- a/crates/legacy/cli/src/server/chat_tool_calls.rs
+++ b/crates/legacy/cli/src/server/chat_tool_calls.rs
@@ -1,9 +1,13 @@
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
-use uzu::types::{
- basic::{ToolCall, ToolDescription, ToolFunction, ToolNamespace, Value},
- session::chat::{ChatContentBlock, ChatMessage, ChatRole},
+use uuid::Uuid;
+use uzu::{
+ session::chat::normalize_tool_call_arguments,
+ types::{
+ basic::{ToolCall, ToolDescription, ToolFunction, ToolNamespace, Value, parse_lenient_json},
+ session::chat::{ChatContentBlock, ChatMessage, ChatRole},
+ },
};
#[derive(Serialize, Deserialize, Clone)]
@@ -11,6 +15,8 @@ pub struct OaiToolCall {
// Present only in streaming deltas, per the OpenAI wire format.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub index: Option<usize>,
+ // Empty in fragments: only the announcement and the final delta carry the id.
+ #[serde(default, skip_serializing_if = "String::is_empty")]
pub id: String,
#[serde(rename = "type", default)]
pub kind: String,
@@ -86,13 +92,16 @@ pub fn choose_tools<'t>(
}
pub fn to_tool_call(tool_call: &OaiToolCall) -> ToolCall {
- // An invalid Value fails serialization inside template rendering and errors the whole request,
- // so arguments that are not valid JSON are re-wrapped instead of passed through.
+ // Templates render arguments as key/value pairs; normalization guarantees
+ // an object no matter what the client echoes back.
let arguments = &tool_call.function.arguments;
- let json = match serde_json::from_str::<serde_json::Value>(arguments) {
- Ok(_) => arguments.clone(),
- Err(_) if arguments.trim().is_empty() => "{}".to_string(),
- Err(_) => serde_json::Value::String(arguments.clone()).to_string(),
+ let json = if arguments.trim().is_empty() {
+ "{}".to_string()
+ } else {
+ normalize_tool_call_arguments(Value {
+ json: arguments.clone(),
+ })
+ .json
};
ToolCall {
identifier: Some(tool_call.id.clone()),
@@ -199,16 +208,265 @@ pub fn withhold_stream_text(
has_tools && (trimmed.is_empty() || trimmed.starts_with('{'))
}
-pub fn tool_call_deltas(
- tool_calls: &[ToolCall],
- emitted: usize,
-) -> Vec<OaiToolCall> {
- tool_calls
- .iter()
- .enumerate()
- .skip(emitted)
- .map(|(index, tool_call)| oai_tool_call(Some(index), tool_call))
- .collect()
+// Streaming state for one in-progress tool call.
+//
+// OpenAI clients assemble `function.arguments` by concatenating delta fragments and parsing the
+// result when the call ends, so every fragment must extend one string. Candidate content comes in
+// two shapes:
+// - framed (qwen3.5-style): the candidate value is the model's raw `<function=name>` /
+// `<parameter=key>` markup, growing token by token. Arguments are synthesized into compact JSON;
+// the currently open parameter's value streams as escaped string content.
+// - bare JSON (llama-3-style): the candidate value is partial JSON text; only the name is
+// announced, since the raw text is not a prefix of the canonical arguments string.
+pub struct ToolCallStreamer {
+ id: String,
+ name_announced: bool,
+ key_order: Vec<String>,
+ open_text: String,
+}
+
+enum FramedParam {
+ Complete(String),
+ Open(String),
+}
+
+struct FramedCall {
+ name: Option<String>,
+ params: Vec<(String, FramedParam)>,
+}
+
+fn parse_framed_call(raw: &str) -> FramedCall {
+ let name = raw.find("<function=").and_then(|start| {
+ let rest = &raw[start + "<function=".len()..];
+ rest.find('>').map(|end| rest[..end].to_string())
+ });
+ let mut params = Vec::new();
+ let mut rest = raw;
+ while let Some(start) = rest.find("<parameter=") {
+ rest = &rest[start + "<parameter=".len()..];
+ let Some(tag_end) = rest.find('>') else {
+ break;
+ };
+ let key = rest[..tag_end].to_string();
+ let value_part = &rest[tag_end + 1..];
+ let value_part = value_part.strip_prefix('\n').unwrap_or(value_part);
+ match value_part.find("\n</parameter>") {
+ Some(end) => {
+ params.push((key, FramedParam::Complete(value_part[..end].to_string())));
+ rest = &value_part[end + "\n</parameter>".len()..];
+ },
+ None => {
+ params.push((key, FramedParam::Open(value_part.to_string())));
+ break;
+ },
+ }
+ }
+ FramedCall {
+ name,
+ params,
+ }
+}
+
+// Mirrors the parser's synthesis rule: values starting with `{` or `[` are typed JSON,
+// everything else is a string.
+fn serialize_param_value(value: &str) -> String {
+ let trimmed = value.trim_start();
+ if (trimmed.starts_with('{') || trimmed.starts_with('['))
+ && let Some(parsed) = parse_lenient_json(value)
+ {
+ return parsed.to_string();
+ }
+ serde_json::Value::String(value.to_string()).to_string()
+}
+
+// The longest suffix of `content` that could be the start of the parameter close tag is
+// ambiguous until more tokens arrive, so it is withheld from streaming.
+const PARAMETER_CLOSE_SEQUENCE: &str = "\n</parameter>";
+
+fn withhold_ambiguous_tail(content: &str) -> &str {
+ // the leftmost char boundary whose suffix is a prefix of the close sequence
+ // gives the longest ambiguous tail
+ for (index, _) in content.char_indices() {
+ let suffix = &content[index..];
+ if suffix.len() <= PARAMETER_CLOSE_SEQUENCE.len() && PARAMETER_CLOSE_SEQUENCE.starts_with(suffix) {
+ return &content[..index];
+ }
+ }
+ content
+}
+
+// The arguments text with no closing brace: completed parameters are frozen, the open
+// parameter's value streams as it grows.
+fn open_arguments_text(params: &[(String, FramedParam)]) -> Option<String> {
+ let mut entries: Vec<(String, String)> = Vec::new();
+ for (key, param) in params {
+ let serialized = match param {
+ FramedParam::Complete(value) => serialize_param_value(value),
+ FramedParam::Open(content) => {
+ // an empty value has no known type yet, and typed values (JSON starting
+ // with `{`/`[`, matching the parser's synthesis rule) can only be serialized
+ // once complete: either way nothing about the parameter may be emitted
+ let streamable = withhold_ambiguous_tail(content);
+ if streamable.is_empty() || streamable.starts_with('{') || streamable.starts_with('[') {
+ break;
+ }
+ let quoted = serde_json::Value::String(streamable.to_string()).to_string();
+ quoted[..quoted.len() - 1].to_string()
+ },
+ };
+ entries.push((key.clone(), serialized));
+ }
+ if entries.is_empty() {
+ return None;
+ }
+ let mut out = String::from("{");
+ for (index, (key, serialized)) in entries.iter().enumerate() {
+ if index > 0 {
+ out.push(',');
+ }
+ out.push_str(&serde_json::to_string(key).expect("key serializes"));
+ out.push(':');
+ out.push_str(serialized);
+ }
+ Some(out)
+}
+
+fn json_candidate_name(raw: &str) -> Option<String> {
+ let rest = &raw[raw.find("\"name\"")? + 6..];
+ let rest = rest[rest.find(':')? + 1..].trim_start();
+ let rest = rest.strip_prefix('"')?;
+ let mut name = String::new();
+ let mut chars = rest.chars();
+ while let Some(c) = chars.next() {
+ match c {
+ '"' => return Some(name),
+ '\\' => match chars.next() {
+ Some('n') => name.push('\n'),
+ Some(other) => name.push(other),
+ None => return None,
+ },
+ c => name.push(c),
+ }
+ }
+ None
+}
+
+impl ToolCallStreamer {
+ pub fn new() -> Self {
+ Self {
+ // The id is assigned by the server at announcement time: clients key
+ // in-progress tool call UI by id, so it must never change mid-call.
+ id: Uuid::new_v4().to_string(),
+ name_announced: false,
+ key_order: Vec::new(),
+ open_text: String::new(),
+ }
+ }
+
+ pub fn update(
+ &mut self,
+ index: usize,
+ raw: &str,
+ ) -> Vec<OaiToolCall> {
+ let mut deltas = Vec::new();
+ let name;
+ if raw.trim_start().starts_with('<') {
+ let call = parse_framed_call(raw);
+ name = call.name.filter(|name| !name.is_empty());
+ self.key_order = call.params.iter().map(|(key, _)| key.clone()).collect();
+ if let Some(open_text) = open_arguments_text(&call.params)
+ && open_text.len() > self.open_text.len()
+ && open_text.starts_with(&self.open_text)
+ {
+ deltas.push(OaiToolCall {
+ index: Some(index),
+ id: String::new(),
+ kind: "function".to_string(),
+ function: OaiFunctionCall {
+ name: String::new(),
+ arguments: open_text[self.open_text.len()..].to_string(),
+ },
+ });
+ self.open_text = open_text;
+ }
+ } else {
+ name = json_candidate_name(raw);
+ }
+ if let Some(name) = name
+ && !name.is_empty()
+ && !self.name_announced
+ {
+ self.name_announced = true;
+ deltas.insert(
+ 0,
+ OaiToolCall {
+ index: Some(index),
+ id: self.id.clone(),
+ kind: "function".to_string(),
+ function: OaiFunctionCall {
+ name,
+ arguments: String::new(),
+ },
+ },
+ );
+ }
+ deltas
+ }
+
+ pub fn finish(
+ &mut self,
+ index: usize,
+ call: &ToolCall,
+ ) -> OaiToolCall {
+ let arguments = if self.open_text.is_empty() {
+ call.arguments.json.clone()
+ } else {
+ self.closing_fragment(call)
+ };
+ OaiToolCall {
+ index: Some(index),
+ id: self.id.clone(),
+ kind: "function".to_string(),
+ function: OaiFunctionCall {
+ name: call.name.clone(),
+ arguments,
+ },
+ }
+ }
+
+ fn closing_fragment(
+ &self,
+ call: &ToolCall,
+ ) -> String {
+ // Rebuild the final arguments with keys in streamed order so the closing fragment
+ // completes the exact string the earlier fragments started.
+ let final_text = match serde_json::from_str::<serde_json::Value>(&call.arguments.json) {
+ Ok(serde_json::Value::Object(map)) => {
+ let mut keys: Vec<&str> =
+ self.key_order.iter().map(String::as_str).filter(|key| map.contains_key(*key)).collect();
+ for key in map.keys() {
+ if !keys.contains(&key.as_str()) {
+ keys.push(key);
+ }
+ }
+ let entries = keys
+ .iter()
+ .map(|key| format!("{}:{}", serde_json::to_string(key).expect("key serializes"), map[*key]))
+ .collect::<Vec<_>>();
+ format!("{{{}}}", entries.join(","))
+ },
+ _ => call.arguments.json.clone(),
+ };
+ match final_text.strip_prefix(&self.open_text) {
+ Some(suffix) => suffix.to_string(),
+ None => {
+ eprintln!(
+ "[server] tool call stream diverged from the final call; the client may reject the assembled arguments"
+ );
+ final_text
+ },
+ }
+ }
}
#[cfg(test)]
diff --git a/crates/legacy/cli/src/server/mod.rs b/crates/legacy/cli/src/server/mod.rs
index 69b688e3c..8a09adf69 100644
--- a/crates/legacy/cli/src/server/mod.rs
+++ b/crates/legacy/cli/src/server/mod.rs
@@ -1,6 +1,7 @@
pub mod chat_completions;
pub mod chat_tool_calls;
pub mod models;
+pub mod request_log;
pub mod runner;
pub mod state;
diff --git a/crates/legacy/cli/src/server/request_log.rs b/crates/legacy/cli/src/server/request_log.rs
new file mode 100644
index 000000000..e1aa7fac0
--- /dev/null
+++ b/crates/legacy/cli/src/server/request_log.rs
@@ -0,0 +1,93 @@
+use std::time::Instant;
+
+use uuid::Uuid;
+use uzu::types::session::chat::ChatReplyStats;
+
+/// Per-request console logging: one line when the request arrives, one when it
+/// ends, correlated by a short tag derived from the request id. Each line is a
+/// single `println!`, which holds the stdout lock for the whole write, so lines
+/// from concurrent requests never interleave.
+pub struct RequestLog {
+ tag: String,
+ started: Instant,
+}
+
+fn short_tag(id: &str) -> String {
+ let id = id.strip_prefix("chatcmpl-").unwrap_or(id);
+ id.chars().take(8).collect()
+}
+
+impl RequestLog {
+ pub fn start(
+ id: &str,
+ stream: bool,
+ messages: usize,
+ tools: usize,
+ reasoning_effort: Option<&str>,
+ ) -> Self {
+ let tag = short_tag(id);
+ println!(
+ "[req {tag}] received: {messages} messages, {}, {tools} tools, reasoning_effort={}",
+ if stream {
+ "stream"
+ } else {
+ "blocking"
+ },
+ reasoning_effort.unwrap_or("default"),
+ );
+ Self {
+ tag,
+ started: Instant::now(),
+ }
+ }
+
+ /// For requests rejected before an id could be assigned to them.
+ pub fn rejected(error: &str) {
+ println!("[req {}] rejected: {error}", short_tag(&Uuid::new_v4().simple().to_string()));
+ }
+
+ pub fn finish(
+ &self,
+ finish_reason: &str,
+ stats: Option<&ChatReplyStats>,
+ notes: Vec<String>,
+ ) {
+ let mut parts =
+ vec![format!("completed in {:.2}s, finish={finish_reason}", self.started.elapsed().as_secs_f64())];
+ if let Some(stats) = stats {
+ let cached = stats.tokens_count_input_cached.unwrap_or(0);
+ let prefilled = stats.tokens_count_input.unwrap_or(0);
+ parts.push(format!(
+ "prompt {} tok ({} cached, {} prefilled @ {} tok/s)",
+ cached + prefilled,
+ cached,
+ prefilled,
+ stats.prefill_tokens_per_second.map_or("-".to_string(), |tps| format!("{tps:.1}")),
+ ));
+ if let Some(ttft) = stats.time_to_first_token {
+ parts.push(format!("ttft {ttft:.2}s"));
+ }
+ if let Some(generated) = stats.tokens_count_output {
+ parts.push(format!(
+ "decode {generated} tok @ {} tok/s",
+ stats.generate_tokens_per_second.map_or("-".to_string(), |tps| format!("{tps:.1}")),
+ ));
+ }
+ if let Some(speculator) = &stats.speculator_stats {
+ parts.push(format!(
+ "spec {:.2} tok/pass ({} passes)",
+ speculator.tokens_per_forward_pass, speculator.num_decode_forward_passes
+ ));
+ }
+ }
+ parts.extend(notes);
+ println!("[req {}] {}", self.tag, parts.join(", "));
+ }
+
+ pub fn fail(
+ &self,
+ error: &str,
+ ) {
+ println!("[req {}] failed in {:.2}s: {error}", self.tag, self.started.elapsed().as_secs_f64());
+ }
+}
diff --git a/crates/legacy/cli/src/server/runner.rs b/crates/legacy/cli/src/server/runner.rs
index 0727019dc..f4b40466a 100644
--- a/crates/legacy/cli/src/server/runner.rs
+++ b/crates/legacy/cli/src/server/runner.rs
@@ -18,6 +18,7 @@ pub async fn run_server(
model: String,
host: String,
port: u16,
+ prefix_cache: bool,
) -> Result<()> {
let engine_config = EngineConfig::default().with_application_identifier("com.trymirai.cli".to_string());
let engine = Engine::new(engine_config).await.context("Failed to create engine")?;
@@ -45,6 +46,7 @@ pub async fn run_server(
model_name: model_name.clone(),
session: Arc::new(Mutex::new(session)),
thinking_support: capabilities.thinking,
+ prefix_cache,
};
let address: IpAddr = host.parse().with_context(|| format!("Invalid host: {host}"))?;
@@ -57,6 +59,14 @@ pub async fn run_server(
println!("πŸš€ OpenAI-compatible server for model: {model_name}");
println!("🌐 Available at: http://{host}:{port}");
+ println!(
+ "πŸ—„οΈ Prefix cache: {}",
+ if prefix_cache {
+ "enabled"
+ } else {
+ "disabled"
+ }
+ );
println!("πŸ“ Endpoints:");
println!(" POST /v1/chat/completions (or /chat/completions)");
println!(" GET /v1/models (or /models)");
diff --git a/crates/legacy/cli/src/server/state.rs b/crates/legacy/cli/src/server/state.rs
index 0038310fa..4cc96eaa7 100644
--- a/crates/legacy/cli/src/server/state.rs
+++ b/crates/legacy/cli/src/server/state.rs
@@ -9,4 +9,5 @@ pub struct ServerState {
pub model_name: String,
pub session: Arc<Mutex<ChatSession>>,
pub thinking_support: ThinkingSupport,
+ pub prefix_cache: bool,
}
diff --git a/crates/legacy/cli/unit/server/chat_completions_test.rs b/crates/legacy/cli/unit/server/chat_completions_test.rs
index c375a27d0..eea23674b 100644
--- a/crates/legacy/cli/unit/server/chat_completions_test.rs
+++ b/crates/legacy/cli/unit/server/chat_completions_test.rs
@@ -118,6 +118,18 @@ fn reasoning_effort_prepends_system_message_for_levels_model() {
assert_eq!(messages.reasoning_effort(), Some(ReasoningEffort::High));
}
+#[test]
+fn reasoning_effort_merges_into_leading_system_message() {
+ let messages = messages_with_support(
+ r#"{"messages":[{"role":"system","content":"be brief"},{"role":"user","content":"hi"}],"reasoning_effort":"high"}"#,
+ ThinkingSupport::Levels(ReasoningEffort::Default),
+ );
+ assert_eq!(messages.len(), 2);
+ assert_eq!(messages[0].role, ChatRole::System {});
+ assert_eq!(messages[0].reasoning_effort(), Some(ReasoningEffort::High));
+ assert_eq!(messages[0].text().as_deref(), Some("be brief"));
+}
+
#[test]
fn reasoning_effort_default_emits_no_system_message() {
for body in [
@@ -330,3 +342,90 @@ fn response_format_composes_with_sampling_options() {
}
);
}
+
+#[test]
+fn message_content_accepts_string_null_and_text_parts() {
+ let messages = request(
+ r#"{"messages":[
+ {"role":"user","content":"plain"},
+ {"role":"assistant","content":null},
+ {"role":"user","content":[{"type":"text","text":"Hello"},{"type":"text","text":" world"}]}
+ ]}"#,
+ )
+ .messages;
+ assert_eq!(messages[0].content.as_deref(), Some("plain"));
+ assert_eq!(messages[1].content, None);
+ assert_eq!(messages[2].content.as_deref(), Some("Hello world"));
+
+ let missing = request(r#"{"messages":[{"role":"user"}]}"#);
+ assert_eq!(missing.messages[0].content, None);
+
+ let image = serde_json::from_str::<ChatCompletionRequest>(
+ r#"{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,AA=="}}]}]}"#,
+ );
+ assert!(image.is_err());
+}
+
+#[test]
+fn assistant_reasoning_content_round_trips_in_session_block_order() {
+ let messages = messages_with_support(
+ r#"{"messages":[
+ {"role":"user","content":"hi"},
+ {"role":"assistant","reasoning_content":"thoughts","content":"answer","tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]},
+ {"role":"assistant","reasoning_content":"","content":"no thinking"},
+ {"role":"user","content":"again"}
+ ]}"#,
+ ThinkingSupport::Levels(ReasoningEffort::Default),
+ );
+ let assistant = &messages[1];
+ let kinds = assistant
+ .content
+ .iter()
+ .map(|block| match block {
+ ChatContentBlock::Reasoning {
+ ..
+ } => "reasoning",
+ ChatContentBlock::Text {
+ ..
+ } => "text",
+ ChatContentBlock::ToolCall {
+ ..
+ } => "tool_call",
+ _ => "other",
+ })
+ .collect::<Vec<_>>();
+ assert_eq!(kinds, ["reasoning", "text", "tool_call"]);
+ assert_eq!(assistant.reasoning().as_deref(), Some("thoughts"));
+
+ // empty reasoning_content must not create a block, matching how replies are stored
+ assert_eq!(messages[2].reasoning(), None);
+ assert_eq!(messages[2].text().as_deref(), Some("no thinking"));
+}
+
+#[test]
+fn prefix_match_ignores_tool_call_identifiers() {
+ let assistant_call = |identifier: &str| {
+ ChatMessage::assistant().with_tool_call(uzu::types::basic::ToolCall {
+ identifier: Some(identifier.to_string()),
+ name: "write".to_string(),
+ arguments: uzu::types::basic::Value {
+ json: r#"{"path":"/tmp/a"}"#.to_string(),
+ },
+ })
+ };
+
+ let current = vec![ChatMessage::user().with_text("hi".to_string()), assistant_call("nagare-uuid")];
+ let extending = vec![
+ ChatMessage::user().with_text("hi".to_string()),
+ assistant_call("server-uuid"),
+ ChatMessage::user().with_text("again".to_string()),
+ ];
+ assert!(messages_have_prefix(&extending, &current));
+
+ let mut different = assistant_call("nagare-uuid");
+ different.content.push(ChatContentBlock::Text {
+ value: "extra".to_string(),
+ });
+ let not_matching = vec![ChatMessage::user().with_text("hi".to_string()), different];
+ assert!(!messages_have_prefix(&not_matching, &current));
+}
diff --git a/crates/legacy/cli/unit/server/chat_tool_calls_test.rs b/crates/legacy/cli/unit/server/chat_tool_calls_test.rs
index 329c249e5..e11889952 100644
--- a/crates/legacy/cli/unit/server/chat_tool_calls_test.rs
+++ b/crates/legacy/cli/unit/server/chat_tool_calls_test.rs
@@ -73,7 +73,7 @@ fn invalid_tool_call_arguments_stay_serializable() {
assert_eq!(call(r#"{"a":1}"#).arguments.json, r#"{"a":1}"#);
assert_eq!(call("").arguments.json, "{}");
- assert_eq!(call(r#"{"a":"#).arguments.json, r#""{\"a\":""#);
+ assert_eq!(call(r#"{"a":"#).arguments.json, r#"{"__uzu_unparsed_arguments":"{\"a\":"}"#);
for arguments in ["", r#"{"a":"#] {
serde_json::to_value(&call(arguments).arguments).expect("arguments should stay serializable");
}
@@ -149,3 +149,142 @@ fn reply_tool_calls_serialize_in_openai_format() {
let delta = serde_json::to_value(oai_tool_call(Some(0), &tool_call)).expect("serializable tool call delta");
assert_eq!(delta["index"], 0);
}
+
+#[test]
+fn json_candidate_announces_name_once_it_is_complete() {
+ let mut streamer = ToolCallStreamer::new();
+ assert!(streamer.update(0, r#"{"name": "wr"#).is_empty());
+ assert!(streamer.update(0, r#"{"arguments": {}}"#).is_empty());
+
+ let deltas = streamer.update(1, r#"{"name": "write", "arguments": {"path":"#);
+ assert_eq!(deltas.len(), 1);
+ assert_eq!(deltas[0].index, Some(1));
+ assert_eq!(deltas[0].function.name, "write");
+ assert_eq!(deltas[0].function.arguments, "");
+
+ // the id is assigned at announcement time and must never change mid-call
+ assert!(!deltas[0].id.is_empty());
+ let call = ToolCall {
+ identifier: Some("server-side-id-must-not-leak".to_string()),
+ name: "write".to_string(),
+ arguments: Value {
+ json: r#"{"path":"/tmp/a"}"#.to_string(),
+ },
+ };
+ assert_eq!(streamer.finish(1, &call).id, deltas[0].id);
+
+ assert!(streamer.update(1, r#"{"name": "write", "arguments": {"path": "/"#).is_empty());
+}
+
+#[test]
+fn framed_tool_call_streams_arguments_incrementally() {
+ let final_markup = "\n<function=write>\n<parameter=path>\n/tmp/a.txt\n</parameter>\n<parameter=content>\nhello world\n</parameter>\n</function>\n";
+ let final_arguments = serde_json::json!({"path": "/tmp/a.txt", "content": "hello world"});
+ let call = ToolCall {
+ identifier: Some("c1".to_string()),
+ name: "write".to_string(),
+ arguments: Value {
+ json: final_arguments.to_string(),
+ },
+ };
+
+ let mut streamer = ToolCallStreamer::new();
+ let mut fragments = String::new();
+ let mut announcements = 0;
+ let chars = final_markup.chars().collect::<Vec<_>>();
+ for i in 0..=chars.len() {
+ let partial: String = chars[..i].iter().collect();
+ for delta in streamer.update(0, &partial) {
+ if !delta.function.name.is_empty() {
+ announcements += 1;
+ assert_eq!(delta.function.name, "write");
+ }
+ fragments.push_str(&delta.function.arguments);
+ }
+ }
+ assert_eq!(announcements, 1);
+ fragments.push_str(&streamer.finish(0, &call).function.arguments);
+
+ let parsed: serde_json::Value = serde_json::from_str(&fragments).expect("assembled arguments parse");
+ assert_eq!(parsed, final_arguments);
+}
+
+#[test]
+fn framed_tool_call_withholds_typed_parameter_until_complete() {
+ let mut streamer = ToolCallStreamer::new();
+ let deltas = streamer.update(0, "\n<function=read>\n<parameter=options>\n{\"a\"");
+ assert!(deltas.iter().all(|delta| delta.function.arguments.is_empty()));
+
+ let deltas = streamer.update(0, "\n<function=read>\n<parameter=options>\n{\"a\": 1}\n</parameter>\n</function>\n");
+ let fragment: String = deltas.iter().map(|delta| delta.function.arguments.as_str()).collect();
+ assert_eq!(fragment, r#"{"options":{"a":1}"#);
+
+ let call = ToolCall {
+ identifier: None,
+ name: "read".to_string(),
+ arguments: Value {
+ json: r#"{"options":{"a":1}}"#.to_string(),
+ },
+ };
+ assert_eq!(streamer.finish(0, &call).function.arguments, "}");
+}
+
+#[test]
+fn framed_tool_call_streams_multibyte_content_without_panicking() {
+ let final_markup = "\n<function=write>\n<parameter=content>\nLondon β€” a city\n</parameter>\n</function>\n";
+ let call = ToolCall {
+ identifier: None,
+ name: "write".to_string(),
+ arguments: Value {
+ json: serde_json::json!({ "content": "London β€” a city" }).to_string(),
+ },
+ };
+
+ let mut streamer = ToolCallStreamer::new();
+ let mut fragments = String::new();
+ let chars = final_markup.chars().collect::<Vec<_>>();
+ for i in 0..=chars.len() {
+ let partial: String = chars[..i].iter().collect();
+ for delta in streamer.update(0, &partial) {
+ fragments.push_str(&delta.function.arguments);
+ }
+ }
+ fragments.push_str(&streamer.finish(0, &call).function.arguments);
+ let parsed: serde_json::Value = serde_json::from_str(&fragments).expect("assembled arguments parse");
+ assert_eq!(parsed, serde_json::json!({ "content": "London β€” a city" }));
+}
+
+#[test]
+fn framed_tool_call_with_array_parameter_assembles_exactly() {
+ // the edit-call shape: a parameter whose value is a JSON array must never be
+ // string-streamed before its type is known, and the finish must not double it
+ let final_markup = "\n<function=edit>\n<parameter=path>\nlonodn.md\n</parameter>\n<parameter=edits>\n[{\"oldText\": \"London\", \"newText\": \"Londinium\"}]\n</parameter>\n</function>\n";
+ let final_arguments = serde_json::json!({
+ "path": "lonodn.md",
+ "edits": [{"oldText": "London", "newText": "Londinium"}],
+ });
+ let call = ToolCall {
+ identifier: Some("c1".to_string()),
+ name: "edit".to_string(),
+ arguments: Value {
+ json: final_arguments.to_string(),
+ },
+ };
+
+ let mut streamer = ToolCallStreamer::new();
+ let mut fragments = String::new();
+ let mut fragment_count = 0;
+ let chars = final_markup.chars().collect::<Vec<_>>();
+ for i in 0..=chars.len() {
+ let partial: String = chars[..i].iter().collect();
+ for delta in streamer.update(0, &partial) {
+ fragment_count += 1;
+ fragments.push_str(&delta.function.arguments);
+ }
+ }
+ fragments.push_str(&streamer.finish(0, &call).function.arguments);
+
+ assert!(fragment_count > 2, "fragments: {fragments:?}");
+ let parsed: serde_json::Value = serde_json::from_str(&fragments).expect("assembled arguments parse");
+ assert_eq!(parsed, final_arguments);
+}
diff --git a/crates/legacy/hanashi/configs/ordering/default.json b/crates/legacy/hanashi/configs/ordering/default.json
index 06a9495bd..d69a7deb4 100644
--- a/crates/legacy/hanashi/configs/ordering/default.json
+++ b/crates/legacy/hanashi/configs/ordering/default.json
@@ -13,7 +13,8 @@
"user"
],
"user": [
- "assistant"
+ "assistant",
+ "user"
],
"assistant": [
"tool",
@@ -21,7 +22,8 @@
],
"tool": [
"tool",
- "assistant"
+ "assistant",
+ "user"
]
}
}
\ No newline at end of file
diff --git a/crates/legacy/hanashi/configs/parsing/muse-glimmer.json b/crates/legacy/hanashi/configs/parsing/muse-glimmer.json
index 57532a676..3afbc093b 100644
--- a/crates/legacy/hanashi/configs/parsing/muse-glimmer.json
+++ b/crates/legacy/hanashi/configs/parsing/muse-glimmer.json
@@ -456,15 +456,6 @@
{
"type": "first"
}
- ],
- "arguments": [
- {
- "type": "get",
- "path": [
- "$value",
- "content"
- ]
- }
]
}
}
diff --git a/crates/legacy/nagare/src/chat/mod.rs b/crates/legacy/nagare/src/chat/mod.rs
index 965b9cd87..ae912d6e9 100644
--- a/crates/legacy/nagare/src/chat/mod.rs
+++ b/crates/legacy/nagare/src/chat/mod.rs
@@ -20,7 +20,7 @@ use shoji::{
backend::chat_message::{Output as BackendOutput, ToolCallState},
},
types::{
- basic::{CancelToken, ToolCall, ToolDescription, ToolNamespace, Value},
+ basic::{CancelToken, ToolCall, ToolDescription, ToolNamespace, Value, parse_lenient_json},
model::Model,
session::chat::{
ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyFinishReason,
@@ -316,6 +316,9 @@ impl ChatSession {
interrupted = true;
break;
}
+ // Token backends produce output from a blocking poll, so this loop never
+ // pends between tokens and consumers would only see one batch at the end.
+ tokio::task::yield_now().await;
if finish_reason.is_some() {
break;
}
@@ -665,7 +668,7 @@ fn build_message(
message.with_tool_call(ToolCall {
identifier,
name: tool_call.name.clone(),
- arguments: tool_call.arguments.clone(),
+ arguments: normalize_tool_call_arguments(tool_call.arguments.clone()),
})
},
};
@@ -673,6 +676,32 @@ fn build_message(
message
}
+/// Key wrapping tool call arguments that survived no parsing attempt, so
+/// templates always see an object and clients can recognize the failure.
+pub const UNPARSED_ARGUMENTS_KEY: &str = "__uzu_unparsed_arguments";
+
+/// Canonicalizes tool call arguments to a compact JSON object. Models sometimes
+/// emit arguments as a string containing the object (with unescaped control
+/// characters or quotes inside), which clients reject and templates cannot
+/// iterate; unparseable remnants are wrapped so templates always see an object.
+pub fn normalize_tool_call_arguments(arguments: Value) -> Value {
+ let mut parsed = match parse_lenient_json(&arguments.json) {
+ Some(parsed) => parsed,
+ None => serde_json::Value::String(arguments.json.clone()),
+ };
+ if let serde_json::Value::String(inner) = &parsed
+ && let Some(unwrapped) = parse_lenient_json(inner)
+ {
+ parsed = unwrapped;
+ }
+ if !parsed.is_object() {
+ parsed = serde_json::json!({ UNPARSED_ARGUMENTS_KEY: parsed });
+ }
+ Value {
+ json: parsed.to_string(),
+ }
+}
+
fn aggregate_stats(
completed: &[ChatReplyStats],
current: &ChatReplyStats,
@@ -686,6 +715,7 @@ fn aggregate_stats(
prefill_tokens_per_second: stats.iter().find_map(|stats| stats.prefill_tokens_per_second),
generate_tokens_per_second: aggregate_generate_rate(&stats),
tokens_count_input: sum_optional_u32(&stats, |stats| stats.tokens_count_input),
+ tokens_count_input_cached: sum_optional_u32(&stats, |stats| stats.tokens_count_input_cached),
tokens_count_output: sum_optional_u32(&stats, |stats| stats.tokens_count_output),
memory_used_bytes: stats.iter().filter_map(|stats| stats.memory_used_bytes).max(),
speculator_stats,
@@ -870,3 +900,61 @@ fn merge_tool_namespaces(
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn normalize_tool_call_arguments_canonicalizes_and_unwraps_string_encoding() {
+ let object = normalize_tool_call_arguments(Value {
+ json: "{ \"path\": \"/tmp/a\" }".to_string(),
+ });
+ assert_eq!(object.json, r#"{"path":"/tmp/a"}"#);
+
+ let double_encoded = normalize_tool_call_arguments(Value {
+ json: serde_json::Value::String("{ \"path\": \"/tmp/a\" }".to_string()).to_string(),
+ });
+ assert_eq!(double_encoded.json, r#"{"path":"/tmp/a"}"#);
+
+ let plain_string = normalize_tool_call_arguments(Value {
+ json: serde_json::Value::String("not json at all".to_string()).to_string(),
+ });
+ assert_eq!(plain_string.json, r#"{"__uzu_unparsed_arguments":"not json at all"}"#);
+
+ let invalid = normalize_tool_call_arguments(Value {
+ json: "{oops".to_string(),
+ });
+ assert_eq!(invalid.json, r#"{"__uzu_unparsed_arguments":"{oops"}"#);
+
+ // string-encoded object with bare quotes inside its string values
+ let bare_quotes = normalize_tool_call_arguments(Value {
+ json: concat!("\"{ \\\"content\\\": \\\"called ", "\"The Square Mile\" home\n", "\\\" }\"").to_string(),
+ });
+ assert_eq!(bare_quotes.json, r#"{"content":"called \"The Square Mile\" home\n"}"#);
+
+ // string-encoded object with an invalid JSON escape (the model meant a literal backslash)
+ let invalid_escape = normalize_tool_call_arguments(Value {
+ json: r#""{ \"command\": \"grep -c 'a\|b'\" }""#.to_string(),
+ });
+ assert_eq!(invalid_escape.json, r#"{"command":"grep -c 'a\\|b'"}"#);
+
+ // string-encoded object with raw newlines inside its string values
+ let with_newlines = normalize_tool_call_arguments(Value {
+ json: serde_json::Value::String("{ \"content\": \"line one\nline two\" }".to_string()).to_string(),
+ });
+ assert_eq!(with_newlines.json, r#"{"content":"line one\nline two"}"#);
+
+ // the outer document itself may carry raw control characters inside the string literal
+ let raw_outer = normalize_tool_call_arguments(Value {
+ json: "{ \"content\": \"line one\nline two\" }".to_string(),
+ });
+ assert_eq!(raw_outer.json, r#"{"content":"line one\nline two"}"#);
+
+ // ... including when string-encoded on top of that
+ let raw_outer_string_encoded = normalize_tool_call_arguments(Value {
+ json: "\"{ \\\"content\\\": \\\"a\nb\\\" }\"".to_string(),
+ });
+ assert_eq!(raw_outer_string_encoded.json, r#"{"content":"a\nb"}"#);
+ }
+}
diff --git a/crates/legacy/nagare/src/chat/token.rs b/crates/legacy/nagare/src/chat/token.rs
index e07143e19..a0f380a6a 100644
--- a/crates/legacy/nagare/src/chat/token.rs
+++ b/crates/legacy/nagare/src/chat/token.rs
@@ -165,6 +165,11 @@ impl Session {
}
}
}
+ let cached_tokens_input = if reset {
+ 0
+ } else {
+ curr_all_tokens.len()
+ };
self.input_tokens = if reset {
if let Err(err) = self.state_reset().await {
return error_stream(err);
@@ -190,6 +195,7 @@ impl Session {
time_prefill_start: Instant::now(),
time_first_token: None,
total_tokens_input: self.input_tokens.len(),
+ cached_tokens_input,
total_tokens_output: 0,
memory_usage: None,
metrics: None,
@@ -301,7 +307,10 @@ impl Session {
} => Some(ToolCallState::Finished(value.clone())),
ChatContentBlock::ToolCallCandidate {
value,
- } => Some(ToolCallState::Candidate(value.json.clone())),
+ } => Some(ToolCallState::Candidate(
+ // the block stores the candidate text as a JSON string document
+ serde_json::from_str::<String>(&value.json).unwrap_or_default(),
+ )),
_ => None,
})
.collect::<Vec<_>>();
@@ -345,6 +354,7 @@ struct StreamingState<'a> {
time_prefill_start: Instant,
time_first_token: Option<Instant>,
total_tokens_input: usize,
+ cached_tokens_input: usize,
total_tokens_output: usize,
memory_usage: Option<usize>,
metrics: Option<TokenStreamMetrics>,
@@ -411,6 +421,7 @@ impl StreamingState<'_> {
prefill_tokens_per_second: prefill_tps,
generate_tokens_per_second: generate_tps,
tokens_count_input: Some(self.total_tokens_input as u32),
+ tokens_count_input_cached: Some(self.cached_tokens_input as u32),
tokens_count_output: Some(self.total_tokens_output as u32),
memory_used_bytes: last_stat.then(|| self.memory_usage.map(|bytes| bytes as i64)).flatten(),
speculator_stats,
diff --git a/crates/legacy/shoji/src/types/basic/mod.rs b/crates/legacy/shoji/src/types/basic/mod.rs
index 09e536325..36f5eeda2 100644
--- a/crates/legacy/shoji/src/types/basic/mod.rs
+++ b/crates/legacy/shoji/src/types/basic/mod.rs
@@ -40,4 +40,4 @@ pub use tool_description::ToolDescription;
pub use tool_function::ToolFunction;
pub use tool_namespace::ToolNamespace;
pub use translation_payload::TranslationPayload;
-pub use value::Value;
+pub use value::{Value, parse_lenient_json};
diff --git a/crates/legacy/shoji/src/types/basic/value.rs b/crates/legacy/shoji/src/types/basic/value.rs
index 6f17c2756..de57e6a0f 100644
--- a/crates/legacy/shoji/src/types/basic/value.rs
+++ b/crates/legacy/shoji/src/types/basic/value.rs
@@ -53,3 +53,103 @@ impl TryFrom<Value> for serde_json::Value {
serde_json::from_str(&value.json)
}
}
+
+/// Parses JSON text produced by a model: strict first, then one repair pass that
+/// escapes literal control characters and bare quotes inside string values.
+pub fn parse_lenient_json(text: &str) -> Option<serde_json::Value> {
+ serde_json::from_str(text).ok().or_else(|| serde_json::from_str(&repair_json_text(text)).ok())
+}
+
+/// Repairs model-generated JSON text inside string values: escapes literal
+/// control characters, doubles backslashes that do not form a valid JSON
+/// escape (the model meant a literal backslash), and escapes quotes that cannot
+/// terminate a string (a closing quote must be followed by a JSON delimiter).
+/// Text outside strings passes through untouched.
+pub fn repair_json_text(text: &str) -> String {
+ let mut out = String::with_capacity(text.len());
+ let mut chars = text.chars().peekable();
+ let mut in_string = false;
+ while let Some(c) = chars.next() {
+ if in_string {
+ match c {
+ '\n' => {
+ out.push_str("\\n");
+ continue;
+ },
+ '\r' => {
+ out.push_str("\\r");
+ continue;
+ },
+ '\t' => {
+ out.push_str("\\t");
+ continue;
+ },
+ c if (c as u32) < 0x20 => {
+ out.push_str(&format!("\\u{:04x}", c as u32));
+ continue;
+ },
+ '\\' => {
+ if matches!(chars.peek(), Some('"' | '\\' | '/' | 'b' | 'f' | 'n' | 'r' | 't' | 'u')) {
+ out.push('\\');
+ out.push(chars.next().unwrap());
+ } else {
+ out.push_str("\\\\");
+ }
+ continue;
+ },
+ '"' => {
+ let mut lookahead = chars.clone();
+ let terminates = loop {
+ match lookahead.next() {
+ Some(c) if c.is_whitespace() => {},
+ Some(',' | '}' | ']' | ':') | None => break true,
+ Some(_) => break false,
+ }
+ };
+ if !terminates {
+ out.push('\\');
+ } else {
+ in_string = false;
+ }
+ out.push('"');
+ continue;
+ },
+ _ => {},
+ }
+ } else if c == '"' {
+ in_string = true;
+ }
+ out.push(c);
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn repair_json_text_only_touches_string_values() {
+ assert_eq!(repair_json_text("\"esc\\ned\\n\""), "\"esc\\ned\\n\"");
+ assert_eq!(repair_json_text("\"a\nb\""), "\"a\\nb\"");
+ assert_eq!(repair_json_text("[1,\n2]"), "[1,\n2]");
+ assert_eq!(
+ repair_json_text(r#"{"a": "called "The Square Mile" β€” here"}"#),
+ r#"{"a": "called \"The Square Mile\" β€” here"}"#
+ );
+ assert_eq!(repair_json_text(r#"{"a": "x", "b": "y"}"#), r#"{"a": "x", "b": "y"}"#);
+ assert_eq!(repair_json_text(r#"{"cmd": "grep 'a\|b'}""#), r#"{"cmd": "grep 'a\\|b'}""#);
+ assert_eq!(repair_json_text(r#"{"a": "x\ny\"z"}"#), r#"{"a": "x\ny\"z"}"#);
+ assert_eq!(repair_json_text(r#"{"a": "C:\\temp"}"#), r#"{"a": "C:\\temp"}"#);
+ }
+
+ #[test]
+ fn parse_lenient_json_repairs_control_characters_and_bare_quotes() {
+ assert_eq!(parse_lenient_json("{ \"content\": \"a\nb\" }"), Some(serde_json::json!({ "content": "a\nb" })));
+ assert_eq!(
+ parse_lenient_json(r#"{ "content": "called "The Square Mile" home" }"#),
+ Some(serde_json::json!({ "content": "called \"The Square Mile\" home" }))
+ );
+ assert_eq!(parse_lenient_json("{oops"), None);
+ }
+}
diff --git a/crates/legacy/shoji/src/types/session/chat/reply/stats.rs b/crates/legacy/shoji/src/types/session/chat/reply/stats.rs
index 136716dd7..19d94097d 100644
--- a/crates/legacy/shoji/src/types/session/chat/reply/stats.rs
+++ b/crates/legacy/shoji/src/types/session/chat/reply/stats.rs
@@ -27,6 +27,7 @@ pub struct ChatReplyStats {
pub prefill_tokens_per_second: Option<f64>,
pub generate_tokens_per_second: Option<f64>,
pub tokens_count_input: Option<u32>,
+ pub tokens_count_input_cached: Option<u32>,
pub tokens_count_output: Option<u32>,
pub memory_used_bytes: Option<i64>,
pub speculator_stats: Option<ChatReplySpeculatorStats>,
diff --git a/crates/legacy/token-stream-parser/tests/test_token_stream_parser_muse_glimmer.rs b/crates/legacy/token-stream-parser/tests/test_token_stream_parser_muse_glimmer.rs
index 4c39928a6..6cd689b3a 100644
--- a/crates/legacy/token-stream-parser/tests/test_token_stream_parser_muse_glimmer.rs
+++ b/crates/legacy/token-stream-parser/tests/test_token_stream_parser_muse_glimmer.rs
@@ -157,3 +157,114 @@ fn test_token_stream_parser_muse_glimmer_tool_output_and_reply() {
},
);
}
+
+mod hermetic {
+ use std::path::PathBuf;
+
+ use token_stream_parser::{
+ Parser,
+ token_stream::{TokenStreamParser, TokenStreamParserConfig},
+ types::Token,
+ };
+
+ // Fabricated tokens: the parser matches framing tokens by value, so no tokenizer is needed.
+ fn feed(
+ parser: &mut TokenStreamParser,
+ text: &str,
+ next_id: &mut u32,
+ ) {
+ const SPECIALS: [&str; 6] =
+ ["<|begin_of_text|>", "<|start|>", "<|message|>", "<|eom|>", "<|eot|>", "<|end_of_text|>"];
+ let mut rest = text;
+ while !rest.is_empty() {
+ let next = SPECIALS.iter().filter_map(|special| rest.find(special).map(|i| (i, special))).min();
+ match next {
+ Some((0, special)) => {
+ parser
+ .push(&Token {
+ id: *next_id,
+ value: special.to_string(),
+ is_special: true,
+ })
+ .unwrap();
+ *next_id += 1;
+ rest = &rest[special.len()..];
+ },
+ Some((index, _)) => {
+ push_text(parser, &rest[..index], next_id);
+ rest = &rest[index..];
+ },
+ None => {
+ push_text(parser, rest, next_id);
+ rest = "";
+ },
+ }
+ }
+ }
+
+ fn push_text(
+ parser: &mut TokenStreamParser,
+ text: &str,
+ next_id: &mut u32,
+ ) {
+ if text.is_empty() {
+ return;
+ }
+ parser
+ .push(&Token {
+ id: *next_id,
+ value: text.to_string(),
+ is_special: false,
+ })
+ .unwrap();
+ *next_id += 1;
+ }
+
+ fn parser() -> TokenStreamParser {
+ let config_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+ .join("..")
+ .join("hanashi")
+ .join("configs")
+ .join("parsing")
+ .join("muse-glimmer.json");
+ let config: TokenStreamParserConfig =
+ serde_json::from_str(&std::fs::read_to_string(config_path).unwrap()).unwrap();
+ TokenStreamParser::new(config).unwrap()
+ }
+
+ #[test]
+ fn unfinished_tool_call_does_not_parse_as_finished() {
+ let mut parser = parser();
+ let mut next_id = 0;
+ feed(&mut parser, "<|start|>user<|message|>hi<|eot|><|start|>assistant", &mut next_id);
+ // stream stops mid-call, right after the markup's first char
+ feed(&mut parser, " to=bash<|message|><", &mut next_id);
+
+ let assistant = &parser.state().value[1];
+ let section = &assistant["content"][0];
+ assert_eq!(section["type"], "tool_call");
+ assert!(section["value"].get("arguments").is_none(), "an unfinished call must not expose arguments: {section}");
+ }
+
+ #[test]
+ fn closed_tool_call_parses_arguments() {
+ let mut parser = parser();
+ let mut next_id = 0;
+ feed(&mut parser, "<|start|>user<|message|>hi<|eot|><|start|>assistant", &mut next_id);
+ feed(
+ &mut parser,
+ concat!(
+ " to=bash<|message|>",
+ "<atem:function_calls>\n<atem:invoke name=\"bash\">\n",
+ "<atem:parameter name=\"command\">printenv</atem:parameter>\n",
+ "</atem:invoke>\n</atem:function_calls><|eot|>"
+ ),
+ &mut next_id,
+ );
+
+ let assistant = &parser.state().value[1];
+ let section = &assistant["content"][0];
+ assert_eq!(section["value"]["name"], "bash");
+ assert_eq!(section["value"]["arguments"]["command"], "printenv");
+ }
+}