A high-performance inference engine for AI models
Gigaslop oai, message processing and tool calling fixes (#756)
uuuvnb3c021cc52ff
diff --git a/bindings/python/uzu/__init__.pyi b/bindings/python/uzu/__init__.pyiindex 08609d816..5441554bd 100644--- a/bindings/python/uzu/__init__.pyi+++ b/bindings/python/uzu/__init__.pyi@@ -381,6 +381,8 @@ class ChatReplyStats:@propertydef tokens_count_input(self) -> typing.Optional[builtins.int]: ...@property+ def tokens_count_input_cached(self) -> typing.Optional[builtins.int]: ...+ @propertydef tokens_count_output(self) -> typing.Optional[builtins.int]: ...@propertydef memory_used_bytes(self) -> typing.Optional[builtins.int]: ...@@ -392,7 +394,7 @@ class ChatReplyStats:def tokens_count(self) -> typing.Optional[builtins.int]: ...@propertydef joules_per_token(self) -> typing.Optional[builtins.float]: ...- def __new__(cls, duration: builtins.float, time_to_first_token: typing.Optional[builtins.float], prefill_tokens_per_second: typing.Optional[builtins.float], generate_tokens_per_second: typing.Optional[builtins.float], tokens_count_input: typing.Optional[builtins.int], tokens_count_output: typing.Optional[builtins.int], memory_used_bytes: typing.Optional[builtins.int], speculator_stats: typing.Optional[ChatReplySpeculatorStats], power_stats: typing.Optional[ChatReplyPowerStats]) -> ChatReplyStats: ...+ def __new__(cls, duration: builtins.float, time_to_first_token: typing.Optional[builtins.float], prefill_tokens_per_second: typing.Optional[builtins.float], generate_tokens_per_second: typing.Optional[builtins.float], tokens_count_input: typing.Optional[builtins.int], tokens_count_input_cached: typing.Optional[builtins.int], tokens_count_output: typing.Optional[builtins.int], memory_used_bytes: typing.Optional[builtins.int], speculator_stats: typing.Optional[ChatReplySpeculatorStats], power_stats: typing.Optional[ChatReplyPowerStats]) -> ChatReplyStats: ...class ChatRole:@typing.finaldiff --git a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swiftindex 7c51f5ee9..c9f7e5f2e 100644--- a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift+++ b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift@@ -1311,6 +1311,7 @@ public struct ChatReplyStats: Equatable, Hashable, Codable {public var prefillTokensPerSecond: Double?public var generateTokensPerSecond: Double?public var tokensCountInput: UInt32?+ public var tokensCountInputCached: UInt32?public var tokensCountOutput: UInt32?public var memoryUsedBytes: Int64?public var speculatorStats: ChatReplySpeculatorStats?@@ -1318,12 +1319,13 @@ public struct ChatReplyStats: Equatable, Hashable, Codable {// Default memberwise initializers are never public by default, so we// declare one manually.- public init(duration: Double, timeToFirstToken: Double?, prefillTokensPerSecond: Double?, generateTokensPerSecond: Double?, tokensCountInput: UInt32?, tokensCountOutput: UInt32?, memoryUsedBytes: Int64?, speculatorStats: ChatReplySpeculatorStats?, powerStats: ChatReplyPowerStats?) {+ public init(duration: Double, timeToFirstToken: Double?, prefillTokensPerSecond: Double?, generateTokensPerSecond: Double?, tokensCountInput: UInt32?, tokensCountInputCached: UInt32?, tokensCountOutput: UInt32?, memoryUsedBytes: Int64?, speculatorStats: ChatReplySpeculatorStats?, powerStats: ChatReplyPowerStats?) {self.duration = durationself.timeToFirstToken = timeToFirstTokenself.prefillTokensPerSecond = prefillTokensPerSecondself.generateTokensPerSecond = generateTokensPerSecondself.tokensCountInput = tokensCountInput+ self.tokensCountInputCached = tokensCountInputCachedself.tokensCountOutput = tokensCountOutputself.memoryUsedBytes = memoryUsedBytesself.speculatorStats = speculatorStats@@ -1370,6 +1372,7 @@ public struct FfiConverterTypeChatReplyStats: FfiConverterRustBuffer {prefillTokensPerSecond: FfiConverterOptionDouble.read(from: &buf),generateTokensPerSecond: FfiConverterOptionDouble.read(from: &buf),tokensCountInput: FfiConverterOptionUInt32.read(from: &buf),+ tokensCountInputCached: FfiConverterOptionUInt32.read(from: &buf),tokensCountOutput: FfiConverterOptionUInt32.read(from: &buf),memoryUsedBytes: FfiConverterOptionInt64.read(from: &buf),speculatorStats: FfiConverterOptionTypeChatReplySpeculatorStats.read(from: &buf),@@ -1383,6 +1386,7 @@ public struct FfiConverterTypeChatReplyStats: FfiConverterRustBuffer {FfiConverterOptionDouble.write(value.prefillTokensPerSecond, into: &buf)FfiConverterOptionDouble.write(value.generateTokensPerSecond, into: &buf)FfiConverterOptionUInt32.write(value.tokensCountInput, into: &buf)+ FfiConverterOptionUInt32.write(value.tokensCountInputCached, into: &buf)FfiConverterOptionUInt32.write(value.tokensCountOutput, into: &buf)FfiConverterOptionInt64.write(value.memoryUsedBytes, into: &buf)FfiConverterOptionTypeChatReplySpeculatorStats.write(value.speculatorStats, into: &buf)diff --git a/bindings/typescript/src/napi/index.d.mts b/bindings/typescript/src/napi/index.d.mtsindex 801692db9..92f9c1556 100644--- a/bindings/typescript/src/napi/index.d.mts+++ b/bindings/typescript/src/napi/index.d.mts@@ -310,11 +310,12 @@ export declare class ChatReplyStats {prefillTokensPerSecond?: numbergenerateTokensPerSecond?: numbertokensCountInput?: number+ tokensCountInputCached?: numbertokensCountOutput?: numbermemoryUsedBytes?: numberspeculatorStats?: ChatReplySpeculatorStatspowerStats?: ChatReplyPowerStats- constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, powerStats?: ChatReplyPowerStats)+ constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountInputCached?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, powerStats?: ChatReplyPowerStats)get tokensCount(): number | null/** Energy spent per processed token, counting both input and output tokens. */get joulesPerToken(): number | nulldiff --git a/bindings/typescript/src/napi/index.d.ts b/bindings/typescript/src/napi/index.d.tsindex 801692db9..92f9c1556 100644--- a/bindings/typescript/src/napi/index.d.ts+++ b/bindings/typescript/src/napi/index.d.ts@@ -310,11 +310,12 @@ export declare class ChatReplyStats {prefillTokensPerSecond?: numbergenerateTokensPerSecond?: numbertokensCountInput?: number+ tokensCountInputCached?: numbertokensCountOutput?: numbermemoryUsedBytes?: numberspeculatorStats?: ChatReplySpeculatorStatspowerStats?: ChatReplyPowerStats- constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, powerStats?: ChatReplyPowerStats)+ constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountInputCached?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, powerStats?: ChatReplyPowerStats)get tokensCount(): number | null/** Energy spent per processed token, counting both input and output tokens. */get joulesPerToken(): number | nulldiff --git a/crates/legacy/backend-remote/src/openai/bridging/chat/stream_chunk.rs b/crates/legacy/backend-remote/src/openai/bridging/chat/stream_chunk.rsindex 21304056d..222ac9c6f 100644--- a/crates/legacy/backend-remote/src/openai/bridging/chat/stream_chunk.rs+++ b/crates/legacy/backend-remote/src/openai/bridging/chat/stream_chunk.rs@@ -53,12 +53,25 @@ struct Usage {prompt_tokens: Option<u32>,#[serde(default)]completion_tokens: Option<u32>,+ #[serde(default)]+ prompt_tokens_details: Option<PromptTokensDetails>,+}++#[derive(Debug, Deserialize)]+struct PromptTokensDetails {+ #[serde(default)]+ cached_tokens: Option<u32>,}pub fn build(response: Response) -> Option<StreamChunk> {let usage = response.usage;let choice = response.choices.into_iter().next();- let tokens_input = usage.as_ref().and_then(|usage| usage.prompt_tokens);+ let tokens_input_cached =+ usage.as_ref().and_then(|usage| usage.prompt_tokens_details.as_ref()).and_then(|details| details.cached_tokens);+ let tokens_input = usage+ .as_ref()+ .and_then(|usage| usage.prompt_tokens)+ .map(|tokens| tokens.saturating_sub(tokens_input_cached.unwrap_or(0)));let tokens_output = usage.as_ref().and_then(|usage| usage.completion_tokens);let (content, tool_calls, finish_reason) = match choice {@@ -88,6 +101,7 @@ pub fn build(response: Response) -> Option<StreamChunk> {&& tool_calls.is_empty()&& finish_reason.is_none()&& tokens_input.is_none()+ && tokens_input_cached.is_none()&& tokens_output.is_none(){return None;@@ -99,6 +113,7 @@ pub fn build(response: Response) -> Option<StreamChunk> {tool_calls,finish_reason,tokens_input,+ tokens_input_cached,tokens_output,})}diff --git a/crates/legacy/backend-remote/src/openai/bridging/responses/stream_chunk.rs b/crates/legacy/backend-remote/src/openai/bridging/responses/stream_chunk.rsindex d7357fbd7..f1a9fba28 100644--- a/crates/legacy/backend-remote/src/openai/bridging/responses/stream_chunk.rs+++ b/crates/legacy/backend-remote/src/openai/bridging/responses/stream_chunk.rs@@ -1,10 +1,15 @@-use async_openai::types::responses::{OutputItem, ResponseStreamEvent};+use async_openai::types::responses::{OutputItem, ResponseStreamEvent, ResponseUsage};use crate::openai::{bridging::responses::finish_reason,stream_state::{StreamChunk, ToolCallChunk},};+fn usage_parts(usage: &ResponseUsage) -> (Option<u32>, Option<u32>, Option<u32>) {+ let cached = usage.input_tokens_details.cached_tokens;+ (Some(usage.input_tokens.saturating_sub(cached)), Some(cached), Some(usage.output_tokens))+}+pub fn build(event: ResponseStreamEvent) -> Option<StreamChunk> {match event {ResponseStreamEvent::ResponseOutputTextDelta(event) => Some(StreamChunk {@@ -40,24 +45,39 @@ pub fn build(event: ResponseStreamEvent) -> Option<StreamChunk> {}],..StreamChunk::default()}),- ResponseStreamEvent::ResponseCompleted(event) => Some(StreamChunk {- finish_reason: finish_reason::build(&event.response.status),- tokens_input: event.response.usage.as_ref().map(|usage| usage.input_tokens),- tokens_output: event.response.usage.as_ref().map(|usage| usage.output_tokens),- ..StreamChunk::default()- }),- ResponseStreamEvent::ResponseFailed(event) => Some(StreamChunk {- finish_reason: finish_reason::build(&event.response.status),- tokens_input: event.response.usage.as_ref().map(|usage| usage.input_tokens),- tokens_output: event.response.usage.as_ref().map(|usage| usage.output_tokens),- ..StreamChunk::default()- }),- ResponseStreamEvent::ResponseIncomplete(event) => Some(StreamChunk {- finish_reason: finish_reason::build(&event.response.status),- tokens_input: event.response.usage.as_ref().map(|usage| usage.input_tokens),- tokens_output: event.response.usage.as_ref().map(|usage| usage.output_tokens),- ..StreamChunk::default()- }),+ ResponseStreamEvent::ResponseCompleted(event) => {+ let (tokens_input, tokens_input_cached, tokens_output) =+ event.response.usage.as_ref().map(usage_parts).unwrap_or_default();+ Some(StreamChunk {+ finish_reason: finish_reason::build(&event.response.status),+ tokens_input,+ tokens_input_cached,+ tokens_output,+ ..StreamChunk::default()+ })+ },+ ResponseStreamEvent::ResponseFailed(event) => {+ let (tokens_input, tokens_input_cached, tokens_output) =+ event.response.usage.as_ref().map(usage_parts).unwrap_or_default();+ Some(StreamChunk {+ finish_reason: finish_reason::build(&event.response.status),+ tokens_input,+ tokens_input_cached,+ tokens_output,+ ..StreamChunk::default()+ })+ },+ ResponseStreamEvent::ResponseIncomplete(event) => {+ let (tokens_input, tokens_input_cached, tokens_output) =+ event.response.usage.as_ref().map(usage_parts).unwrap_or_default();+ Some(StreamChunk {+ finish_reason: finish_reason::build(&event.response.status),+ tokens_input,+ tokens_input_cached,+ tokens_output,+ ..StreamChunk::default()+ })+ },_ => None,}}diff --git a/crates/legacy/backend-remote/src/openai/stream_state.rs b/crates/legacy/backend-remote/src/openai/stream_state.rsindex f7c125af5..6f69cd090 100644--- a/crates/legacy/backend-remote/src/openai/stream_state.rs+++ b/crates/legacy/backend-remote/src/openai/stream_state.rs@@ -23,6 +23,7 @@ pub struct StreamChunk {pub tool_calls: Vec<ToolCallChunk>,pub finish_reason: Option<ChatReplyFinishReason>,pub tokens_input: Option<u32>,+ pub tokens_input_cached: Option<u32>,pub tokens_output: Option<u32>,}@@ -35,6 +36,7 @@ pub struct StreamState {start_moment: Instant,first_token_moment: Option<Instant>,tokens_input: Option<u32>,+ tokens_input_cached: Option<u32>,tokens_output: Option<u32>,}@@ -48,6 +50,7 @@ impl StreamState {start_moment: Instant::now(),first_token_moment: None,tokens_input: None,+ tokens_input_cached: None,tokens_output: None,}}@@ -85,6 +88,11 @@ impl StreamState {processed = true;}+ if let Some(tokens) = chunk.tokens_input_cached {+ self.tokens_input_cached = Some(tokens);+ processed = true;+ }+if let Some(tokens) = chunk.tokens_output {self.tokens_output = Some(tokens);processed = true;@@ -141,6 +149,7 @@ impl StreamState {prefill_tokens_per_second,generate_tokens_per_second,tokens_count_input: self.tokens_input,+ tokens_count_input_cached: self.tokens_input_cached,tokens_count_output: self.tokens_output,memory_used_bytes: None,speculator_stats: None,diff --git a/crates/legacy/cli/src/main.rs b/crates/legacy/cli/src/main.rsindex 4864a58c3..ac57345c1 100644--- a/crates/legacy/cli/src/main.rs+++ b/crates/legacy/cli/src/main.rs@@ -42,6 +42,12 @@ enum Commands {port: u16,#[arg(long, default_value = "127.0.0.1")]host: String,+ /// Reuse the previous request's state when the new prompt extends it (enabled by default).+ #[arg(long = "prefix-cache", action = clap::ArgAction::SetTrue, conflicts_with = "no_prefix_cache")]+ prefix_cache: bool,+ /// Reset the session on every request, disabling prefix cache reuse.+ #[arg(long = "no-prefix-cache", action = clap::ArgAction::SetTrue)]+ no_prefix_cache: bool,},Storage {#[arg(long, value_enum, default_value_t = storage::DownloadManagerCliType::default())]@@ -67,7 +73,9 @@ async fn main() -> Result<()> {model,port,host,- }) => server::run_server(model, host, port).await?,+ prefix_cache,+ no_prefix_cache,+ }) => server::run_server(model, host, port, prefix_cache || !no_prefix_cache).await?,Some(Commands::Storage {download_manager,}) => storage::run(download_manager).await?,diff --git a/crates/legacy/cli/src/server/chat_completions.rs b/crates/legacy/cli/src/server/chat_completions.rsindex 50755dc2b..614b42055 100644--- a/crates/legacy/cli/src/server/chat_completions.rs+++ b/crates/legacy/cli/src/server/chat_completions.rs@@ -1,4 +1,5 @@use std::{+ collections::HashMap,pin::Pin,str::FromStr,sync::Arc,@@ -7,6 +8,7 @@ use std::{use rocket::{Request, State,+ data::{ByteUnit, Data},futures::Stream,http::Status,post,@@ -21,10 +23,12 @@ use tokio::sync::{Mutex, mpsc};use tokio_stream::wrappers::UnboundedReceiverStream;use uuid::Uuid;use uzu::{- session::chat::{ChatSession, ChatSessionStreamChunk},+ session::chat::{ChatSession, ChatSessionStreamChunk, UNPARSED_ARGUMENTS_KEY},types::{basic::{Grammar, ReasoningEffort, SamplingMethod},- session::chat::{ChatMessage, ChatReplyConfig, ChatReplyFinishReason, ChatReplyStats, ChatRole},+ session::chat::{+ ChatContentBlock, ChatMessage, ChatReplyConfig, ChatReplyFinishReason, ChatReplyStats, ChatRole,+ },},};@@ -33,16 +37,45 @@ use crate::{server::{ServerState,chat_tool_calls::{- OaiTool, OaiToolCall, backfill_tool_result_names, choose_tools, insert_tools_message, reply_tool_calls,- to_tool_call, tool_call_deltas, tool_call_result_block, withhold_stream_text,+ OaiTool, OaiToolCall, ToolCallStreamer, backfill_tool_result_names, choose_tools, insert_tools_message,+ oai_tool_call, reply_tool_calls, to_tool_call, tool_call_result_block, withhold_stream_text,},+ request_log::RequestLog,},};+fn deserialize_message_content<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>+where+ D: serde::Deserializer<'de>,+{+ match Option::<serde_json::Value>::deserialize(deserializer)? {+ None => Ok(None),+ Some(serde_json::Value::String(text)) => Ok(Some(text)),+ Some(serde_json::Value::Array(parts)) => {+ let mut text = String::new();+ for part in &parts {+ match part.get("type").and_then(serde_json::Value::as_str) {+ Some("text") => {+ text.push_str(part.get("text").and_then(serde_json::Value::as_str).unwrap_or_default());+ },+ Some(other) => {+ return Err(serde::de::Error::custom(format!(+ "unsupported message content part type: {other}"+ )));+ },+ None => return Err(serde::de::Error::custom("message content part is missing a type")),+ }+ }+ Ok(Some(text))+ },+ Some(_) => Err(serde::de::Error::custom("message content must be a string or an array of content parts")),+ }+}+#[derive(Serialize, Deserialize, Clone)]pub struct OaiMessage {pub role: String,- #[serde(default)]+ #[serde(default, deserialize_with = "deserialize_message_content")]pub content: Option<String>,// vLLM-style reasoning channel.#[serde(default, skip_serializing_if = "Option::is_none")]@@ -112,12 +145,19 @@ pub struct ChatCompletionChoice {pub finish_reason: String,}+#[derive(Serialize, Clone)]+pub struct ChatCompletionPromptTokensDetails {+ pub cached_tokens: u32,+}+#[derive(Serialize, Clone, Default)]pub struct ChatCompletionUsage {pub prompt_tokens: u32,pub completion_tokens: u32,pub total_tokens: u32,#[serde(skip_serializing_if = "Option::is_none")]+ pub prompt_tokens_details: Option<ChatCompletionPromptTokensDetails>,+ #[serde(skip_serializing_if = "Option::is_none")]pub spec_verify_ct: Option<u32>,}@@ -138,6 +178,8 @@ struct StreamDelta {#[serde(skip_serializing_if = "Option::is_none")]content: Option<String>,#[serde(skip_serializing_if = "Option::is_none")]+ reasoning_content: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]tool_calls: Option<Vec<OaiToolCall>>,}@@ -202,6 +244,12 @@ fn to_chat_messages(messages: &[OaiMessage]) -> Vec<ChatMessage> {.map(|message| {let role = ChatRole::from_str(&message.role).unwrap_or(ChatRole::User {});let mut chat_message = ChatMessage::for_role(role);+ // Block order matches how the session stores generated replies+ // (reasoning, then text, then tool calls) so a client that echoes+ // reasoning back keeps the message prefix intact.+ if let Some(reasoning) = message.reasoning_content.as_ref().filter(|reasoning| !reasoning.is_empty()) {+ chat_message = chat_message.with_reasoning(reasoning.clone());+ }if let Some(identifier) = &message.tool_call_id {let result = tool_call_result_block(identifier, message.content.clone().unwrap_or_default());chat_message = chat_message.with_block(result);@@ -354,8 +402,15 @@ pub(crate) fn build_messages(if let Some((effort, source)) = requested_reasoning_effort(request)? {let fulfilled = thinking_support.fulfill_requested_effort(effort).map_err(|detail| source.error(detail))?;if let Some(effort) = fulfilled {- // The engine reads the effort from a reasoning_effort block carried on a system message.- messages.insert(0, ChatMessage::system().with_reasoning_effort(effort));+ // The engine reads the effort from a reasoning_effort block carried on a system+ // message. Merge into a leading system message when the client sent one:+ // templates reject two system messages in a row.+ match messages.first_mut() {+ Some(first) if first.role == (ChatRole::System {}) => {+ *first = first.clone().with_reasoning_effort(effort);+ },+ _ => messages.insert(0, ChatMessage::system().with_reasoning_effort(effort)),+ }}}backfill_tool_result_names(&mut messages);@@ -480,13 +535,21 @@ fn map_finish_reason(finish_reason: &ChatReplyFinishReason) -> String {.to_string()}-fn usage_from_stats(stats: &ChatReplyStats) -> ChatCompletionUsage {- let prompt_tokens = stats.tokens_count_input.unwrap_or(0);+fn usage_from_stats(+ stats: &ChatReplyStats,+ prefix_cache: bool,+) -> ChatCompletionUsage {+ let cached_tokens = stats.tokens_count_input_cached.unwrap_or(0);+ let prefilled_tokens = stats.tokens_count_input.unwrap_or(0);+ let prompt_tokens = prefilled_tokens + cached_tokens;let completion_tokens = stats.tokens_count_output.unwrap_or(0);ChatCompletionUsage {prompt_tokens,completion_tokens,total_tokens: prompt_tokens + completion_tokens,+ prompt_tokens_details: prefix_cache.then_some(ChatCompletionPromptTokensDetails {+ cached_tokens,+ }),spec_verify_ct: stats.speculator_stats.as_ref()@@ -543,6 +606,57 @@ fn chunk_json(serde_json::to_string(&chunk).unwrap_or_default()}+/// Tool call identifiers are assigned per turn and never rendered into tokens, so they are+/// excluded from the comparison; everything else about the messages must match.+fn messages_have_prefix(+ messages: &[ChatMessage],+ prefix: &[ChatMessage],+) -> bool {+ prefix.len() <= messages.len()+ && prefix.iter().zip(messages).all(|(prefix_message, message)| {+ prefix_message.role == message.role+ && prefix_message.metadata == message.metadata+ && prefix_message.content.len() == message.content.len()+ && prefix_message.content.iter().zip(&message.content).all(|(prefix_block, block)| {+ match (prefix_block, block) {+ (+ ChatContentBlock::ToolCall {+ value: prefix_call,+ },+ ChatContentBlock::ToolCall {+ value: call,+ },+ ) => {+ let (mut prefix_call, mut call) = (prefix_call.clone(), call.clone());+ prefix_call.identifier = None;+ call.identifier = None;+ prefix_call == call+ },+ _ => prefix_block == block,+ }+ })+ })+}++/// Returns only the messages the session has not seen yet when the request+/// extends the session's current history; otherwise resets the session and+/// returns the full list. Token-level reuse itself is decided inside the+/// session, which falls back to a full prefill if the rendered tokens diverge.+async fn prepare_input(+ session: &ChatSession,+ mut messages: Vec<ChatMessage>,+ prefix_cache: bool,+) -> Result<Vec<ChatMessage>, uzu::session::chat::ChatSessionError> {+ if prefix_cache {+ let current = session.messages().await;+ if !current.is_empty() && messages.len() > current.len() && messages_have_prefix(&messages, ¤t) {+ return Ok(messages.split_off(current.len()));+ }+ }+ session.reset().await?;+ Ok(messages)+}+async fn run_blocking(session: Arc<Mutex<ChatSession>>,messages: Vec<ChatMessage>,@@ -550,13 +664,19 @@ async fn run_blocking(id: String,model: String,created: i64,+ prefix_cache: bool,+ log: RequestLog,) -> ChatCompletionResponse {let session = session.lock().await;- if let Err(error) = session.reset().await {- return error_response(id, model, created, &error.to_string());- }+ let input = match prepare_input(&session, messages, prefix_cache).await {+ Ok(input) => input,+ Err(error) => {+ log.fail(&error.to_string());+ return error_response(id, model, created, &error.to_string());+ },+ };- match session.reply(messages, config).await {+ match session.reply(input, config).await {Ok(replies) => match replies.last() {Some(reply) => {let tool_calls = reply_tool_calls(&reply.message);@@ -564,11 +684,32 @@ async fn run_blocking(let content = reply.message.text().or_else(|| tool_calls.is_none().then(String::new));let mut finish_reason =reply.finish_reason.as_ref().map(map_finish_reason).unwrap_or_else(|| "stop".to_string());- // A call the parser could not finalize yields ToolCalls with nothing to execute.- // Clients would stall on an absent tool_calls array, so report stop.- if tool_calls.is_none() && finish_reason == "tool_calls" {- finish_reason = "stop".to_string();+ // Calls the parser could not finalize must not execute as a partial batch.+ // OpenAI has no finish reason for this; providers signal terminal errors with+ // unrecognized reasons (pi maps them to a thrown provider error), so the turn+ // fails loudly instead of stalling on an absent or incomplete tool_calls array.+ let has_unfinished_candidates = reply+ .message+ .content+ .iter()+ .any(|block| matches!(block, ChatContentBlock::ToolCallCandidate { .. }));+ if finish_reason == "tool_calls" && (tool_calls.is_none() || has_unfinished_candidates) {+ finish_reason = "malformed_tool_call".to_string();}+ let wrapped_arguments = reply+ .message+ .tool_calls()+ .iter()+ .filter(|call| call.arguments.json.contains(UNPARSED_ARGUMENTS_KEY))+ .count();+ let mut notes = Vec::new();+ if has_unfinished_candidates {+ notes.push("model generated unfinished tool call(s)".to_string());+ }+ if wrapped_arguments > 0 {+ notes.push(format!("{wrapped_arguments} tool call(s) with unparseable arguments"));+ }+ log.finish(&finish_reason, Some(&reply.stats), notes);ChatCompletionResponse {id,object: "chat.completion".to_string(),@@ -585,12 +726,18 @@ async fn run_blocking(},finish_reason,}],- usage: usage_from_stats(&reply.stats),+ usage: usage_from_stats(&reply.stats, prefix_cache),}},- None => error_response(id, model, created, "No response generated"),+ None => {+ log.fail("no response generated");+ error_response(id, model, created, "No response generated")+ },+ },+ Err(error) => {+ log.fail(&error.to_string());+ error_response(id, model, created, &error.to_string())},- Err(error) => error_response(id, model, created, &error.to_string()),}}@@ -601,24 +748,31 @@ async fn run_stream(id: String,model: String,created: i64,+ prefix_cache: bool,sender: mpsc::UnboundedSender<Event>,+ log: RequestLog,) {let session = session.lock().await;- if let Err(error) = session.reset().await {- let _ = sender.send(Event::data(chunk_json(- &id,- &model,- created,- StreamDelta {- content: Some(format!("Error: {error}")),- ..StreamDelta::default()- },- Some("stop".to_string()),- None,- )));- let _ = sender.send(Event::data("[DONE]"));- return;- }+ let has_tools = messages.iter().any(|message| !message.tool_namespaces().is_empty());+ let input = match prepare_input(&session, messages, prefix_cache).await {+ Ok(input) => input,+ Err(error) => {+ log.fail(&error.to_string());+ let _ = sender.send(Event::data(chunk_json(+ &id,+ &model,+ created,+ StreamDelta {+ content: Some(format!("Error: {error}")),+ ..StreamDelta::default()+ },+ Some("stop".to_string()),+ None,+ )));+ let _ = sender.send(Event::data("[DONE]"));+ return;+ },+ };let _ = sender.send(Event::data(chunk_json(&id,@@ -632,16 +786,36 @@ async fn run_stream(None,)));- let has_tools = messages.iter().any(|message| !message.tool_namespaces().is_empty());- let stream = session.reply_with_stream(messages, config).await;+ let stream = session.reply_with_stream(input, config).await;+ let cancel_token = stream.cancel_token();let mut emitted = 0usize;+ let mut emitted_reasoning = 0usize;let mut emitted_tool_calls = 0usize;+ let mut tool_call_streamers: HashMap<usize, ToolCallStreamer> = HashMap::new();let mut final_text = String::new();let mut finish_reason = "stop".to_string();- let mut usage = ChatCompletionUsage::default();+ let mut final_stats: Option<ChatReplyStats> = None;+ let mut final_had_candidates = false;+ let mut wrapped_arguments = 0usize;let mut errored = false;- while let Some(chunk) = stream.next().await {+ loop {+ // Rocket drops the SSE receiver as soon as the connection dies, so racing each chunk+ // against the channel closing notices a disconnect even while generation is still in+ // prefill and no chunk is on its way. Cancelling stops the backend at the next token+ // boundary and lets the session run its cancelled-turn history cleanup.+ let chunk = tokio::select! {+ chunk = stream.next() => chunk,+ () = sender.closed() => {+ cancel_token.cancel();+ cancel_token.cancel();+ log.fail("client disconnected");+ return;+ },+ };+ let Some(chunk) = chunk else {+ break;+ };match chunk {ChatSessionStreamChunk::Replies {replies,@@ -649,6 +823,30 @@ async fn run_stream(let Some(reply) = replies.last() else {continue;};+ let reasoning = reply.message.reasoning().unwrap_or_default();+ let reasoning_start = (emitted_reasoning..=reasoning.len())+ .find(|&index| reasoning.is_char_boundary(index))+ .unwrap_or(reasoning.len());+ if reasoning.len() > reasoning_start {+ let delta = reasoning[reasoning_start..].to_string();+ emitted_reasoning = reasoning.len();+ let sent = sender.send(Event::data(chunk_json(+ &id,+ &model,+ created,+ StreamDelta {+ reasoning_content: Some(delta),+ ..StreamDelta::default()+ },+ None,+ None,+ )));+ if sent.is_err() {+ cancel_token.cancel();+ log.fail("client disconnected");+ return;+ }+ }let text = reply.message.text().unwrap_or_default();let start = (emitted..=text.len()).find(|&index| text.is_char_boundary(index)).unwrap_or(text.len());if !withhold_stream_text(has_tools, &text) && text.len() > start {@@ -666,26 +864,72 @@ async fn run_stream(None,)));if sent.is_err() {+ cancel_token.cancel();+ log.fail("client disconnected");return;}}+ let finished_count = reply.message.tool_calls().len();+ let candidates = reply.message.content.iter().filter_map(|block| match block {+ ChatContentBlock::ToolCallCandidate {+ value,+ } => Some(value),+ _ => None,+ });+ for (offset, value) in candidates.enumerate() {+ let index = finished_count + offset;+ // candidates carry their partial text as a JSON string document, except+ // object-valued ones (e.g. muse-glimmer's name-only progress objects)+ let raw = serde_json::from_str::<String>(&value.json).unwrap_or_else(|_| value.json.clone());+ let deltas =+ tool_call_streamers.entry(index).or_insert_with(ToolCallStreamer::new).update(index, &raw);+ for delta in deltas {+ let sent = sender.send(Event::data(chunk_json(+ &id,+ &model,+ created,+ StreamDelta {+ tool_calls: Some(vec![delta]),+ ..StreamDelta::default()+ },+ None,+ None,+ )));+ if sent.is_err() {+ cancel_token.cancel();+ log.fail("client disconnected");+ return;+ }+ }+ }+let tool_calls = reply.message.tool_calls();- if tool_calls.len() > emitted_tool_calls {- let delta = tool_call_deltas(&tool_calls, emitted_tool_calls);- emitted_tool_calls = tool_calls.len();+ while emitted_tool_calls < tool_calls.len() {+ let index = emitted_tool_calls;+ emitted_tool_calls += 1;+ let call = &tool_calls[index];+ if call.arguments.json.contains(UNPARSED_ARGUMENTS_KEY) {+ wrapped_arguments += 1;+ }+ let delta = match tool_call_streamers.remove(&index) {+ Some(mut streamer) => streamer.finish(index, call),+ None => oai_tool_call(Some(index), call),+ };let sent = sender.send(Event::data(chunk_json(&id,&model,created,StreamDelta {- tool_calls: Some(delta),+ tool_calls: Some(vec![delta]),..StreamDelta::default()},None,None,)));if sent.is_err() {+ cancel_token.cancel();+ log.fail("client disconnected");return;}}@@ -693,13 +937,19 @@ async fn run_stream(if let Some(reason) = &reply.finish_reason {finish_reason = map_finish_reason(reason);}- usage = usage_from_stats(&reply.stats);+ final_stats = Some(reply.stats.clone());final_text = text;+ final_had_candidates = reply+ .message+ .content+ .iter()+ .any(|block| matches!(block, ChatContentBlock::ToolCallCandidate { .. }));},ChatSessionStreamChunk::Error {error,} => {errored = true;+ log.fail(&error.to_string());let _ = sender.send(Event::data(chunk_json(&id,&model,@@ -734,14 +984,28 @@ async fn run_stream(None,)));if sent.is_err() {+ cancel_token.cancel();+ log.fail("client disconnected");return;}}- // Same guard as the blocking path: candidates that never finalized produce a ToolCalls finish with- // no emitted tool call deltas, which reads as stop.- if emitted_tool_calls == 0 && finish_reason == "tool_calls" {- finish_reason = "stop".to_string();+ // Same guard as the blocking path: calls the parser could not finalize must not+ // execute as a partial batch, so the turn reports a malformed tool call instead of+ // stalling on an absent or incomplete tool_calls array.+ if finish_reason == "tool_calls" && (emitted_tool_calls == 0 || final_had_candidates) {+ finish_reason = "malformed_tool_call".to_string();+ }+ let mut notes = Vec::new();+ if final_had_candidates {+ notes.push("model generated unfinished tool call(s)".to_string());}+ if wrapped_arguments > 0 {+ notes.push(format!("{wrapped_arguments} tool call(s) with unparseable arguments"));+ }+ log.finish(&finish_reason, final_stats.as_ref(), notes);+ let usage = final_stats+ .as_ref()+ .map_or_else(ChatCompletionUsage::default, |stats| usage_from_stats(stats, prefix_cache));let _ = sender.send(Event::data(chunk_json(&id,&model,@@ -755,35 +1019,80 @@ async fn run_stream(}#[allow(private_interfaces)]-#[post("/chat/completions", format = "json", data = "<request>")]+#[post("/chat/completions", format = "json", data = "<body>")]pub async fn handle_chat_completions(- request: Json<ChatCompletionRequest>,+ body: Data<'_>,state: &State<ServerState>,) -> ChatCompletionResult {- let request = request.into_inner();+ let body = match body.open(ByteUnit::Mebibyte(64)).into_string().await {+ Ok(body) if body.is_complete() => body.into_inner(),+ Ok(_) => {+ return invalid_request_response("body", "request_too_large", "request body exceeds 64 MiB".to_string());+ },+ Err(error) => {+ return invalid_request_response("body", "invalid_body", format!("failed to read request body: {error}"));+ },+ };+ let request = match serde_json::from_str::<ChatCompletionRequest>(&body) {+ Ok(request) => request,+ Err(error) => {+ RequestLog::rejected(&format!("failed to parse chat completion request: {error}"));+ return invalid_request_response(+ "body",+ "invalid_request",+ format!("failed to parse chat completion request: {error}"),+ );+ },+ };let id = format!("chatcmpl-{}", Uuid::new_v4().simple());let created = now_unix();let model = state.model_name.clone();let is_stream = request.stream.unwrap_or(false);+ let log = RequestLog::start(+ &id,+ is_stream,+ request.messages.len(),+ request.tools.as_ref().map_or(0, Vec::len),+ request.reasoning_effort.as_ref().and_then(serde_json::Value::as_str).or(Some("unspecified")),+ );let config = match build_reply_config(&request) {Ok(config) => config,- Err(error) => return invalid_request_response("response_format", error.code(), error.message()),+ Err(error) => {+ log.fail(&error.message());+ return invalid_request_response("response_format", error.code(), error.message());+ },};let messages = match build_messages(&request, state.thinking_support) {Ok(messages) => messages,- Err(error) => return invalid_request_response(error.param(), error.code(), error.into_detail()),+ Err(error) => {+ let param = error.param();+ let code = error.code();+ let detail = error.into_detail();+ log.fail(&detail);+ return invalid_request_response(param, code, detail);+ },};if is_stream {let session = Arc::clone(&state.session);let (sender, receiver) = mpsc::unbounded_channel::<Event>();- rocket::tokio::spawn(run_stream(session, messages, config, id, model, created, sender));+ rocket::tokio::spawn(run_stream(+ session,+ messages,+ config,+ id,+ model,+ created,+ state.prefix_cache,+ sender,+ log,+ ));let body: Pin<Box<dyn Stream<Item = Event> + Send>> = Box::pin(UnboundedReceiverStream::new(receiver));ChatCompletionResult::Stream(EventStream::from(body))} else {let session = Arc::clone(&state.session);- let response = run_blocking(session, messages, config, id, model, created).await;+ let response = run_blocking(session, messages, config, id, model, created, state.prefix_cache, log).await;ChatCompletionResult::Json(Json(response))}}diff --git a/crates/legacy/cli/src/server/chat_tool_calls.rs b/crates/legacy/cli/src/server/chat_tool_calls.rsindex 4b6339353..5ac1e0851 100644--- a/crates/legacy/cli/src/server/chat_tool_calls.rs+++ b/crates/legacy/cli/src/server/chat_tool_calls.rs@@ -1,9 +1,13 @@use std::collections::HashMap;use serde::{Deserialize, Serialize};-use uzu::types::{- basic::{ToolCall, ToolDescription, ToolFunction, ToolNamespace, Value},- session::chat::{ChatContentBlock, ChatMessage, ChatRole},+use uuid::Uuid;+use uzu::{+ session::chat::normalize_tool_call_arguments,+ types::{+ basic::{ToolCall, ToolDescription, ToolFunction, ToolNamespace, Value, parse_lenient_json},+ session::chat::{ChatContentBlock, ChatMessage, ChatRole},+ },};#[derive(Serialize, Deserialize, Clone)]@@ -11,6 +15,8 @@ pub struct OaiToolCall {// Present only in streaming deltas, per the OpenAI wire format.#[serde(default, skip_serializing_if = "Option::is_none")]pub index: Option<usize>,+ // Empty in fragments: only the announcement and the final delta carry the id.+ #[serde(default, skip_serializing_if = "String::is_empty")]pub id: String,#[serde(rename = "type", default)]pub kind: String,@@ -86,13 +92,16 @@ pub fn choose_tools<'t>(}pub fn to_tool_call(tool_call: &OaiToolCall) -> ToolCall {- // An invalid Value fails serialization inside template rendering and errors the whole request,- // so arguments that are not valid JSON are re-wrapped instead of passed through.+ // Templates render arguments as key/value pairs; normalization guarantees+ // an object no matter what the client echoes back.let arguments = &tool_call.function.arguments;- let json = match serde_json::from_str::<serde_json::Value>(arguments) {- Ok(_) => arguments.clone(),- Err(_) if arguments.trim().is_empty() => "{}".to_string(),- Err(_) => serde_json::Value::String(arguments.clone()).to_string(),+ let json = if arguments.trim().is_empty() {+ "{}".to_string()+ } else {+ normalize_tool_call_arguments(Value {+ json: arguments.clone(),+ })+ .json};ToolCall {identifier: Some(tool_call.id.clone()),@@ -199,16 +208,265 @@ pub fn withhold_stream_text(has_tools && (trimmed.is_empty() || trimmed.starts_with('{'))}-pub fn tool_call_deltas(- tool_calls: &[ToolCall],- emitted: usize,-) -> Vec<OaiToolCall> {- tool_calls- .iter()- .enumerate()- .skip(emitted)- .map(|(index, tool_call)| oai_tool_call(Some(index), tool_call))- .collect()+// Streaming state for one in-progress tool call.+//+// OpenAI clients assemble `function.arguments` by concatenating delta fragments and parsing the+// result when the call ends, so every fragment must extend one string. Candidate content comes in+// two shapes:+// - framed (qwen3.5-style): the candidate value is the model's raw `<function=name>` /+// `<parameter=key>` markup, growing token by token. Arguments are synthesized into compact JSON;+// the currently open parameter's value streams as escaped string content.+// - bare JSON (llama-3-style): the candidate value is partial JSON text; only the name is+// announced, since the raw text is not a prefix of the canonical arguments string.+pub struct ToolCallStreamer {+ id: String,+ name_announced: bool,+ key_order: Vec<String>,+ open_text: String,+}++enum FramedParam {+ Complete(String),+ Open(String),+}++struct FramedCall {+ name: Option<String>,+ params: Vec<(String, FramedParam)>,+}++fn parse_framed_call(raw: &str) -> FramedCall {+ let name = raw.find("<function=").and_then(|start| {+ let rest = &raw[start + "<function=".len()..];+ rest.find('>').map(|end| rest[..end].to_string())+ });+ let mut params = Vec::new();+ let mut rest = raw;+ while let Some(start) = rest.find("<parameter=") {+ rest = &rest[start + "<parameter=".len()..];+ let Some(tag_end) = rest.find('>') else {+ break;+ };+ let key = rest[..tag_end].to_string();+ let value_part = &rest[tag_end + 1..];+ let value_part = value_part.strip_prefix('\n').unwrap_or(value_part);+ match value_part.find("\n</parameter>") {+ Some(end) => {+ params.push((key, FramedParam::Complete(value_part[..end].to_string())));+ rest = &value_part[end + "\n</parameter>".len()..];+ },+ None => {+ params.push((key, FramedParam::Open(value_part.to_string())));+ break;+ },+ }+ }+ FramedCall {+ name,+ params,+ }+}++// Mirrors the parser's synthesis rule: values starting with `{` or `[` are typed JSON,+// everything else is a string.+fn serialize_param_value(value: &str) -> String {+ let trimmed = value.trim_start();+ if (trimmed.starts_with('{') || trimmed.starts_with('['))+ && let Some(parsed) = parse_lenient_json(value)+ {+ return parsed.to_string();+ }+ serde_json::Value::String(value.to_string()).to_string()+}++// The longest suffix of `content` that could be the start of the parameter close tag is+// ambiguous until more tokens arrive, so it is withheld from streaming.+const PARAMETER_CLOSE_SEQUENCE: &str = "\n</parameter>";++fn withhold_ambiguous_tail(content: &str) -> &str {+ // the leftmost char boundary whose suffix is a prefix of the close sequence+ // gives the longest ambiguous tail+ for (index, _) in content.char_indices() {+ let suffix = &content[index..];+ if suffix.len() <= PARAMETER_CLOSE_SEQUENCE.len() && PARAMETER_CLOSE_SEQUENCE.starts_with(suffix) {+ return &content[..index];+ }+ }+ content+}++// The arguments text with no closing brace: completed parameters are frozen, the open+// parameter's value streams as it grows.+fn open_arguments_text(params: &[(String, FramedParam)]) -> Option<String> {+ let mut entries: Vec<(String, String)> = Vec::new();+ for (key, param) in params {+ let serialized = match param {+ FramedParam::Complete(value) => serialize_param_value(value),+ FramedParam::Open(content) => {+ // an empty value has no known type yet, and typed values (JSON starting+ // with `{`/`[`, matching the parser's synthesis rule) can only be serialized+ // once complete: either way nothing about the parameter may be emitted+ let streamable = withhold_ambiguous_tail(content);+ if streamable.is_empty() || streamable.starts_with('{') || streamable.starts_with('[') {+ break;+ }+ let quoted = serde_json::Value::String(streamable.to_string()).to_string();+ quoted[..quoted.len() - 1].to_string()+ },+ };+ entries.push((key.clone(), serialized));+ }+ if entries.is_empty() {+ return None;+ }+ let mut out = String::from("{");+ for (index, (key, serialized)) in entries.iter().enumerate() {+ if index > 0 {+ out.push(',');+ }+ out.push_str(&serde_json::to_string(key).expect("key serializes"));+ out.push(':');+ out.push_str(serialized);+ }+ Some(out)+}++fn json_candidate_name(raw: &str) -> Option<String> {+ let rest = &raw[raw.find("\"name\"")? + 6..];+ let rest = rest[rest.find(':')? + 1..].trim_start();+ let rest = rest.strip_prefix('"')?;+ let mut name = String::new();+ let mut chars = rest.chars();+ while let Some(c) = chars.next() {+ match c {+ '"' => return Some(name),+ '\\' => match chars.next() {+ Some('n') => name.push('\n'),+ Some(other) => name.push(other),+ None => return None,+ },+ c => name.push(c),+ }+ }+ None+}++impl ToolCallStreamer {+ pub fn new() -> Self {+ Self {+ // The id is assigned by the server at announcement time: clients key+ // in-progress tool call UI by id, so it must never change mid-call.+ id: Uuid::new_v4().to_string(),+ name_announced: false,+ key_order: Vec::new(),+ open_text: String::new(),+ }+ }++ pub fn update(+ &mut self,+ index: usize,+ raw: &str,+ ) -> Vec<OaiToolCall> {+ let mut deltas = Vec::new();+ let name;+ if raw.trim_start().starts_with('<') {+ let call = parse_framed_call(raw);+ name = call.name.filter(|name| !name.is_empty());+ self.key_order = call.params.iter().map(|(key, _)| key.clone()).collect();+ if let Some(open_text) = open_arguments_text(&call.params)+ && open_text.len() > self.open_text.len()+ && open_text.starts_with(&self.open_text)+ {+ deltas.push(OaiToolCall {+ index: Some(index),+ id: String::new(),+ kind: "function".to_string(),+ function: OaiFunctionCall {+ name: String::new(),+ arguments: open_text[self.open_text.len()..].to_string(),+ },+ });+ self.open_text = open_text;+ }+ } else {+ name = json_candidate_name(raw);+ }+ if let Some(name) = name+ && !name.is_empty()+ && !self.name_announced+ {+ self.name_announced = true;+ deltas.insert(+ 0,+ OaiToolCall {+ index: Some(index),+ id: self.id.clone(),+ kind: "function".to_string(),+ function: OaiFunctionCall {+ name,+ arguments: String::new(),+ },+ },+ );+ }+ deltas+ }++ pub fn finish(+ &mut self,+ index: usize,+ call: &ToolCall,+ ) -> OaiToolCall {+ let arguments = if self.open_text.is_empty() {+ call.arguments.json.clone()+ } else {+ self.closing_fragment(call)+ };+ OaiToolCall {+ index: Some(index),+ id: self.id.clone(),+ kind: "function".to_string(),+ function: OaiFunctionCall {+ name: call.name.clone(),+ arguments,+ },+ }+ }++ fn closing_fragment(+ &self,+ call: &ToolCall,+ ) -> String {+ // Rebuild the final arguments with keys in streamed order so the closing fragment+ // completes the exact string the earlier fragments started.+ let final_text = match serde_json::from_str::<serde_json::Value>(&call.arguments.json) {+ Ok(serde_json::Value::Object(map)) => {+ let mut keys: Vec<&str> =+ self.key_order.iter().map(String::as_str).filter(|key| map.contains_key(*key)).collect();+ for key in map.keys() {+ if !keys.contains(&key.as_str()) {+ keys.push(key);+ }+ }+ let entries = keys+ .iter()+ .map(|key| format!("{}:{}", serde_json::to_string(key).expect("key serializes"), map[*key]))+ .collect::<Vec<_>>();+ format!("{{{}}}", entries.join(","))+ },+ _ => call.arguments.json.clone(),+ };+ match final_text.strip_prefix(&self.open_text) {+ Some(suffix) => suffix.to_string(),+ None => {+ eprintln!(+ "[server] tool call stream diverged from the final call; the client may reject the assembled arguments"+ );+ final_text+ },+ }+ }}#[cfg(test)]diff --git a/crates/legacy/cli/src/server/mod.rs b/crates/legacy/cli/src/server/mod.rsindex 69b688e3c..8a09adf69 100644--- a/crates/legacy/cli/src/server/mod.rs+++ b/crates/legacy/cli/src/server/mod.rs@@ -1,6 +1,7 @@pub mod chat_completions;pub mod chat_tool_calls;pub mod models;+pub mod request_log;pub mod runner;pub mod state;diff --git a/crates/legacy/cli/src/server/request_log.rs b/crates/legacy/cli/src/server/request_log.rsnew file mode 100644index 000000000..e1aa7fac0--- /dev/null+++ b/crates/legacy/cli/src/server/request_log.rs@@ -0,0 +1,93 @@+use std::time::Instant;++use uuid::Uuid;+use uzu::types::session::chat::ChatReplyStats;++/// Per-request console logging: one line when the request arrives, one when it+/// ends, correlated by a short tag derived from the request id. Each line is a+/// single `println!`, which holds the stdout lock for the whole write, so lines+/// from concurrent requests never interleave.+pub struct RequestLog {+ tag: String,+ started: Instant,+}++fn short_tag(id: &str) -> String {+ let id = id.strip_prefix("chatcmpl-").unwrap_or(id);+ id.chars().take(8).collect()+}++impl RequestLog {+ pub fn start(+ id: &str,+ stream: bool,+ messages: usize,+ tools: usize,+ reasoning_effort: Option<&str>,+ ) -> Self {+ let tag = short_tag(id);+ println!(+ "[req {tag}] received: {messages} messages, {}, {tools} tools, reasoning_effort={}",+ if stream {+ "stream"+ } else {+ "blocking"+ },+ reasoning_effort.unwrap_or("default"),+ );+ Self {+ tag,+ started: Instant::now(),+ }+ }++ /// For requests rejected before an id could be assigned to them.+ pub fn rejected(error: &str) {+ println!("[req {}] rejected: {error}", short_tag(&Uuid::new_v4().simple().to_string()));+ }++ pub fn finish(+ &self,+ finish_reason: &str,+ stats: Option<&ChatReplyStats>,+ notes: Vec<String>,+ ) {+ let mut parts =+ vec![format!("completed in {:.2}s, finish={finish_reason}", self.started.elapsed().as_secs_f64())];+ if let Some(stats) = stats {+ let cached = stats.tokens_count_input_cached.unwrap_or(0);+ let prefilled = stats.tokens_count_input.unwrap_or(0);+ parts.push(format!(+ "prompt {} tok ({} cached, {} prefilled @ {} tok/s)",+ cached + prefilled,+ cached,+ prefilled,+ stats.prefill_tokens_per_second.map_or("-".to_string(), |tps| format!("{tps:.1}")),+ ));+ if let Some(ttft) = stats.time_to_first_token {+ parts.push(format!("ttft {ttft:.2}s"));+ }+ if let Some(generated) = stats.tokens_count_output {+ parts.push(format!(+ "decode {generated} tok @ {} tok/s",+ stats.generate_tokens_per_second.map_or("-".to_string(), |tps| format!("{tps:.1}")),+ ));+ }+ if let Some(speculator) = &stats.speculator_stats {+ parts.push(format!(+ "spec {:.2} tok/pass ({} passes)",+ speculator.tokens_per_forward_pass, speculator.num_decode_forward_passes+ ));+ }+ }+ parts.extend(notes);+ println!("[req {}] {}", self.tag, parts.join(", "));+ }++ pub fn fail(+ &self,+ error: &str,+ ) {+ println!("[req {}] failed in {:.2}s: {error}", self.tag, self.started.elapsed().as_secs_f64());+ }+}diff --git a/crates/legacy/cli/src/server/runner.rs b/crates/legacy/cli/src/server/runner.rsindex 0727019dc..f4b40466a 100644--- a/crates/legacy/cli/src/server/runner.rs+++ b/crates/legacy/cli/src/server/runner.rs@@ -18,6 +18,7 @@ pub async fn run_server(model: String,host: String,port: u16,+ prefix_cache: bool,) -> Result<()> {let engine_config = EngineConfig::default().with_application_identifier("com.trymirai.cli".to_string());let engine = Engine::new(engine_config).await.context("Failed to create engine")?;@@ -45,6 +46,7 @@ pub async fn run_server(model_name: model_name.clone(),session: Arc::new(Mutex::new(session)),thinking_support: capabilities.thinking,+ prefix_cache,};let address: IpAddr = host.parse().with_context(|| format!("Invalid host: {host}"))?;@@ -57,6 +59,14 @@ pub async fn run_server(println!("π OpenAI-compatible server for model: {model_name}");println!("π Available at: http://{host}:{port}");+ println!(+ "ποΈ Prefix cache: {}",+ if prefix_cache {+ "enabled"+ } else {+ "disabled"+ }+ );println!("π Endpoints:");println!(" POST /v1/chat/completions (or /chat/completions)");println!(" GET /v1/models (or /models)");diff --git a/crates/legacy/cli/src/server/state.rs b/crates/legacy/cli/src/server/state.rsindex 0038310fa..4cc96eaa7 100644--- a/crates/legacy/cli/src/server/state.rs+++ b/crates/legacy/cli/src/server/state.rs@@ -9,4 +9,5 @@ pub struct ServerState {pub model_name: String,pub session: Arc<Mutex<ChatSession>>,pub thinking_support: ThinkingSupport,+ pub prefix_cache: bool,}diff --git a/crates/legacy/cli/unit/server/chat_completions_test.rs b/crates/legacy/cli/unit/server/chat_completions_test.rsindex c375a27d0..eea23674b 100644--- a/crates/legacy/cli/unit/server/chat_completions_test.rs+++ b/crates/legacy/cli/unit/server/chat_completions_test.rs@@ -118,6 +118,18 @@ fn reasoning_effort_prepends_system_message_for_levels_model() {assert_eq!(messages.reasoning_effort(), Some(ReasoningEffort::High));}+#[test]+fn reasoning_effort_merges_into_leading_system_message() {+ let messages = messages_with_support(+ r#"{"messages":[{"role":"system","content":"be brief"},{"role":"user","content":"hi"}],"reasoning_effort":"high"}"#,+ ThinkingSupport::Levels(ReasoningEffort::Default),+ );+ assert_eq!(messages.len(), 2);+ assert_eq!(messages[0].role, ChatRole::System {});+ assert_eq!(messages[0].reasoning_effort(), Some(ReasoningEffort::High));+ assert_eq!(messages[0].text().as_deref(), Some("be brief"));+}+#[test]fn reasoning_effort_default_emits_no_system_message() {for body in [@@ -330,3 +342,90 @@ fn response_format_composes_with_sampling_options() {});}++#[test]+fn message_content_accepts_string_null_and_text_parts() {+ let messages = request(+ r#"{"messages":[+ {"role":"user","content":"plain"},+ {"role":"assistant","content":null},+ {"role":"user","content":[{"type":"text","text":"Hello"},{"type":"text","text":" world"}]}+ ]}"#,+ )+ .messages;+ assert_eq!(messages[0].content.as_deref(), Some("plain"));+ assert_eq!(messages[1].content, None);+ assert_eq!(messages[2].content.as_deref(), Some("Hello world"));++ let missing = request(r#"{"messages":[{"role":"user"}]}"#);+ assert_eq!(missing.messages[0].content, None);++ let image = serde_json::from_str::<ChatCompletionRequest>(+ r#"{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,AA=="}}]}]}"#,+ );+ assert!(image.is_err());+}++#[test]+fn assistant_reasoning_content_round_trips_in_session_block_order() {+ let messages = messages_with_support(+ r#"{"messages":[+ {"role":"user","content":"hi"},+ {"role":"assistant","reasoning_content":"thoughts","content":"answer","tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]},+ {"role":"assistant","reasoning_content":"","content":"no thinking"},+ {"role":"user","content":"again"}+ ]}"#,+ ThinkingSupport::Levels(ReasoningEffort::Default),+ );+ let assistant = &messages[1];+ let kinds = assistant+ .content+ .iter()+ .map(|block| match block {+ ChatContentBlock::Reasoning {+ ..+ } => "reasoning",+ ChatContentBlock::Text {+ ..+ } => "text",+ ChatContentBlock::ToolCall {+ ..+ } => "tool_call",+ _ => "other",+ })+ .collect::<Vec<_>>();+ assert_eq!(kinds, ["reasoning", "text", "tool_call"]);+ assert_eq!(assistant.reasoning().as_deref(), Some("thoughts"));++ // empty reasoning_content must not create a block, matching how replies are stored+ assert_eq!(messages[2].reasoning(), None);+ assert_eq!(messages[2].text().as_deref(), Some("no thinking"));+}++#[test]+fn prefix_match_ignores_tool_call_identifiers() {+ let assistant_call = |identifier: &str| {+ ChatMessage::assistant().with_tool_call(uzu::types::basic::ToolCall {+ identifier: Some(identifier.to_string()),+ name: "write".to_string(),+ arguments: uzu::types::basic::Value {+ json: r#"{"path":"/tmp/a"}"#.to_string(),+ },+ })+ };++ let current = vec![ChatMessage::user().with_text("hi".to_string()), assistant_call("nagare-uuid")];+ let extending = vec![+ ChatMessage::user().with_text("hi".to_string()),+ assistant_call("server-uuid"),+ ChatMessage::user().with_text("again".to_string()),+ ];+ assert!(messages_have_prefix(&extending, ¤t));++ let mut different = assistant_call("nagare-uuid");+ different.content.push(ChatContentBlock::Text {+ value: "extra".to_string(),+ });+ let not_matching = vec![ChatMessage::user().with_text("hi".to_string()), different];+ assert!(!messages_have_prefix(¬_matching, ¤t));+}diff --git a/crates/legacy/cli/unit/server/chat_tool_calls_test.rs b/crates/legacy/cli/unit/server/chat_tool_calls_test.rsindex 329c249e5..e11889952 100644--- a/crates/legacy/cli/unit/server/chat_tool_calls_test.rs+++ b/crates/legacy/cli/unit/server/chat_tool_calls_test.rs@@ -73,7 +73,7 @@ fn invalid_tool_call_arguments_stay_serializable() {assert_eq!(call(r#"{"a":1}"#).arguments.json, r#"{"a":1}"#);assert_eq!(call("").arguments.json, "{}");- assert_eq!(call(r#"{"a":"#).arguments.json, r#""{\"a\":""#);+ assert_eq!(call(r#"{"a":"#).arguments.json, r#"{"__uzu_unparsed_arguments":"{\"a\":"}"#);for arguments in ["", r#"{"a":"#] {serde_json::to_value(&call(arguments).arguments).expect("arguments should stay serializable");}@@ -149,3 +149,142 @@ fn reply_tool_calls_serialize_in_openai_format() {let delta = serde_json::to_value(oai_tool_call(Some(0), &tool_call)).expect("serializable tool call delta");assert_eq!(delta["index"], 0);}++#[test]+fn json_candidate_announces_name_once_it_is_complete() {+ let mut streamer = ToolCallStreamer::new();+ assert!(streamer.update(0, r#"{"name": "wr"#).is_empty());+ assert!(streamer.update(0, r#"{"arguments": {}}"#).is_empty());++ let deltas = streamer.update(1, r#"{"name": "write", "arguments": {"path":"#);+ assert_eq!(deltas.len(), 1);+ assert_eq!(deltas[0].index, Some(1));+ assert_eq!(deltas[0].function.name, "write");+ assert_eq!(deltas[0].function.arguments, "");++ // the id is assigned at announcement time and must never change mid-call+ assert!(!deltas[0].id.is_empty());+ let call = ToolCall {+ identifier: Some("server-side-id-must-not-leak".to_string()),+ name: "write".to_string(),+ arguments: Value {+ json: r#"{"path":"/tmp/a"}"#.to_string(),+ },+ };+ assert_eq!(streamer.finish(1, &call).id, deltas[0].id);++ assert!(streamer.update(1, r#"{"name": "write", "arguments": {"path": "/"#).is_empty());+}++#[test]+fn framed_tool_call_streams_arguments_incrementally() {+ let final_markup = "\n<function=write>\n<parameter=path>\n/tmp/a.txt\n</parameter>\n<parameter=content>\nhello world\n</parameter>\n</function>\n";+ let final_arguments = serde_json::json!({"path": "/tmp/a.txt", "content": "hello world"});+ let call = ToolCall {+ identifier: Some("c1".to_string()),+ name: "write".to_string(),+ arguments: Value {+ json: final_arguments.to_string(),+ },+ };++ let mut streamer = ToolCallStreamer::new();+ let mut fragments = String::new();+ let mut announcements = 0;+ let chars = final_markup.chars().collect::<Vec<_>>();+ for i in 0..=chars.len() {+ let partial: String = chars[..i].iter().collect();+ for delta in streamer.update(0, &partial) {+ if !delta.function.name.is_empty() {+ announcements += 1;+ assert_eq!(delta.function.name, "write");+ }+ fragments.push_str(&delta.function.arguments);+ }+ }+ assert_eq!(announcements, 1);+ fragments.push_str(&streamer.finish(0, &call).function.arguments);++ let parsed: serde_json::Value = serde_json::from_str(&fragments).expect("assembled arguments parse");+ assert_eq!(parsed, final_arguments);+}++#[test]+fn framed_tool_call_withholds_typed_parameter_until_complete() {+ let mut streamer = ToolCallStreamer::new();+ let deltas = streamer.update(0, "\n<function=read>\n<parameter=options>\n{\"a\"");+ assert!(deltas.iter().all(|delta| delta.function.arguments.is_empty()));++ let deltas = streamer.update(0, "\n<function=read>\n<parameter=options>\n{\"a\": 1}\n</parameter>\n</function>\n");+ let fragment: String = deltas.iter().map(|delta| delta.function.arguments.as_str()).collect();+ assert_eq!(fragment, r#"{"options":{"a":1}"#);++ let call = ToolCall {+ identifier: None,+ name: "read".to_string(),+ arguments: Value {+ json: r#"{"options":{"a":1}}"#.to_string(),+ },+ };+ assert_eq!(streamer.finish(0, &call).function.arguments, "}");+}++#[test]+fn framed_tool_call_streams_multibyte_content_without_panicking() {+ let final_markup = "\n<function=write>\n<parameter=content>\nLondon β a city\n</parameter>\n</function>\n";+ let call = ToolCall {+ identifier: None,+ name: "write".to_string(),+ arguments: Value {+ json: serde_json::json!({ "content": "London β a city" }).to_string(),+ },+ };++ let mut streamer = ToolCallStreamer::new();+ let mut fragments = String::new();+ let chars = final_markup.chars().collect::<Vec<_>>();+ for i in 0..=chars.len() {+ let partial: String = chars[..i].iter().collect();+ for delta in streamer.update(0, &partial) {+ fragments.push_str(&delta.function.arguments);+ }+ }+ fragments.push_str(&streamer.finish(0, &call).function.arguments);+ let parsed: serde_json::Value = serde_json::from_str(&fragments).expect("assembled arguments parse");+ assert_eq!(parsed, serde_json::json!({ "content": "London β a city" }));+}++#[test]+fn framed_tool_call_with_array_parameter_assembles_exactly() {+ // the edit-call shape: a parameter whose value is a JSON array must never be+ // string-streamed before its type is known, and the finish must not double it+ let final_markup = "\n<function=edit>\n<parameter=path>\nlonodn.md\n</parameter>\n<parameter=edits>\n[{\"oldText\": \"London\", \"newText\": \"Londinium\"}]\n</parameter>\n</function>\n";+ let final_arguments = serde_json::json!({+ "path": "lonodn.md",+ "edits": [{"oldText": "London", "newText": "Londinium"}],+ });+ let call = ToolCall {+ identifier: Some("c1".to_string()),+ name: "edit".to_string(),+ arguments: Value {+ json: final_arguments.to_string(),+ },+ };++ let mut streamer = ToolCallStreamer::new();+ let mut fragments = String::new();+ let mut fragment_count = 0;+ let chars = final_markup.chars().collect::<Vec<_>>();+ for i in 0..=chars.len() {+ let partial: String = chars[..i].iter().collect();+ for delta in streamer.update(0, &partial) {+ fragment_count += 1;+ fragments.push_str(&delta.function.arguments);+ }+ }+ fragments.push_str(&streamer.finish(0, &call).function.arguments);++ assert!(fragment_count > 2, "fragments: {fragments:?}");+ let parsed: serde_json::Value = serde_json::from_str(&fragments).expect("assembled arguments parse");+ assert_eq!(parsed, final_arguments);+}diff --git a/crates/legacy/hanashi/configs/ordering/default.json b/crates/legacy/hanashi/configs/ordering/default.jsonindex 06a9495bd..d69a7deb4 100644--- a/crates/legacy/hanashi/configs/ordering/default.json+++ b/crates/legacy/hanashi/configs/ordering/default.json@@ -13,7 +13,8 @@"user"],"user": [- "assistant"+ "assistant",+ "user"],"assistant": ["tool",@@ -21,7 +22,8 @@],"tool": ["tool",- "assistant"+ "assistant",+ "user"]}}\ No newline at end of filediff --git a/crates/legacy/hanashi/configs/parsing/muse-glimmer.json b/crates/legacy/hanashi/configs/parsing/muse-glimmer.jsonindex 57532a676..3afbc093b 100644--- a/crates/legacy/hanashi/configs/parsing/muse-glimmer.json+++ b/crates/legacy/hanashi/configs/parsing/muse-glimmer.json@@ -456,15 +456,6 @@{"type": "first"}- ],- "arguments": [- {- "type": "get",- "path": [- "$value",- "content"- ]- }]}}diff --git a/crates/legacy/nagare/src/chat/mod.rs b/crates/legacy/nagare/src/chat/mod.rsindex 965b9cd87..ae912d6e9 100644--- a/crates/legacy/nagare/src/chat/mod.rs+++ b/crates/legacy/nagare/src/chat/mod.rs@@ -20,7 +20,7 @@ use shoji::{backend::chat_message::{Output as BackendOutput, ToolCallState},},types::{- basic::{CancelToken, ToolCall, ToolDescription, ToolNamespace, Value},+ basic::{CancelToken, ToolCall, ToolDescription, ToolNamespace, Value, parse_lenient_json},model::Model,session::chat::{ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyFinishReason,@@ -316,6 +316,9 @@ impl ChatSession {interrupted = true;break;}+ // Token backends produce output from a blocking poll, so this loop never+ // pends between tokens and consumers would only see one batch at the end.+ tokio::task::yield_now().await;if finish_reason.is_some() {break;}@@ -665,7 +668,7 @@ fn build_message(message.with_tool_call(ToolCall {identifier,name: tool_call.name.clone(),- arguments: tool_call.arguments.clone(),+ arguments: normalize_tool_call_arguments(tool_call.arguments.clone()),})},};@@ -673,6 +676,32 @@ fn build_message(message}+/// Key wrapping tool call arguments that survived no parsing attempt, so+/// templates always see an object and clients can recognize the failure.+pub const UNPARSED_ARGUMENTS_KEY: &str = "__uzu_unparsed_arguments";++/// Canonicalizes tool call arguments to a compact JSON object. Models sometimes+/// emit arguments as a string containing the object (with unescaped control+/// characters or quotes inside), which clients reject and templates cannot+/// iterate; unparseable remnants are wrapped so templates always see an object.+pub fn normalize_tool_call_arguments(arguments: Value) -> Value {+ let mut parsed = match parse_lenient_json(&arguments.json) {+ Some(parsed) => parsed,+ None => serde_json::Value::String(arguments.json.clone()),+ };+ if let serde_json::Value::String(inner) = &parsed+ && let Some(unwrapped) = parse_lenient_json(inner)+ {+ parsed = unwrapped;+ }+ if !parsed.is_object() {+ parsed = serde_json::json!({ UNPARSED_ARGUMENTS_KEY: parsed });+ }+ Value {+ json: parsed.to_string(),+ }+}+fn aggregate_stats(completed: &[ChatReplyStats],current: &ChatReplyStats,@@ -686,6 +715,7 @@ fn aggregate_stats(prefill_tokens_per_second: stats.iter().find_map(|stats| stats.prefill_tokens_per_second),generate_tokens_per_second: aggregate_generate_rate(&stats),tokens_count_input: sum_optional_u32(&stats, |stats| stats.tokens_count_input),+ tokens_count_input_cached: sum_optional_u32(&stats, |stats| stats.tokens_count_input_cached),tokens_count_output: sum_optional_u32(&stats, |stats| stats.tokens_count_output),memory_used_bytes: stats.iter().filter_map(|stats| stats.memory_used_bytes).max(),speculator_stats,@@ -870,3 +900,61 @@ fn merge_tool_namespaces(}}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn normalize_tool_call_arguments_canonicalizes_and_unwraps_string_encoding() {+ let object = normalize_tool_call_arguments(Value {+ json: "{ \"path\": \"/tmp/a\" }".to_string(),+ });+ assert_eq!(object.json, r#"{"path":"/tmp/a"}"#);++ let double_encoded = normalize_tool_call_arguments(Value {+ json: serde_json::Value::String("{ \"path\": \"/tmp/a\" }".to_string()).to_string(),+ });+ assert_eq!(double_encoded.json, r#"{"path":"/tmp/a"}"#);++ let plain_string = normalize_tool_call_arguments(Value {+ json: serde_json::Value::String("not json at all".to_string()).to_string(),+ });+ assert_eq!(plain_string.json, r#"{"__uzu_unparsed_arguments":"not json at all"}"#);++ let invalid = normalize_tool_call_arguments(Value {+ json: "{oops".to_string(),+ });+ assert_eq!(invalid.json, r#"{"__uzu_unparsed_arguments":"{oops"}"#);++ // string-encoded object with bare quotes inside its string values+ let bare_quotes = normalize_tool_call_arguments(Value {+ json: concat!("\"{ \\\"content\\\": \\\"called ", "\"The Square Mile\" home\n", "\\\" }\"").to_string(),+ });+ assert_eq!(bare_quotes.json, r#"{"content":"called \"The Square Mile\" home\n"}"#);++ // string-encoded object with an invalid JSON escape (the model meant a literal backslash)+ let invalid_escape = normalize_tool_call_arguments(Value {+ json: r#""{ \"command\": \"grep -c 'a\|b'\" }""#.to_string(),+ });+ assert_eq!(invalid_escape.json, r#"{"command":"grep -c 'a\\|b'"}"#);++ // string-encoded object with raw newlines inside its string values+ let with_newlines = normalize_tool_call_arguments(Value {+ json: serde_json::Value::String("{ \"content\": \"line one\nline two\" }".to_string()).to_string(),+ });+ assert_eq!(with_newlines.json, r#"{"content":"line one\nline two"}"#);++ // the outer document itself may carry raw control characters inside the string literal+ let raw_outer = normalize_tool_call_arguments(Value {+ json: "{ \"content\": \"line one\nline two\" }".to_string(),+ });+ assert_eq!(raw_outer.json, r#"{"content":"line one\nline two"}"#);++ // ... including when string-encoded on top of that+ let raw_outer_string_encoded = normalize_tool_call_arguments(Value {+ json: "\"{ \\\"content\\\": \\\"a\nb\\\" }\"".to_string(),+ });+ assert_eq!(raw_outer_string_encoded.json, r#"{"content":"a\nb"}"#);+ }+}diff --git a/crates/legacy/nagare/src/chat/token.rs b/crates/legacy/nagare/src/chat/token.rsindex e07143e19..a0f380a6a 100644--- a/crates/legacy/nagare/src/chat/token.rs+++ b/crates/legacy/nagare/src/chat/token.rs@@ -165,6 +165,11 @@ impl Session {}}}+ let cached_tokens_input = if reset {+ 0+ } else {+ curr_all_tokens.len()+ };self.input_tokens = if reset {if let Err(err) = self.state_reset().await {return error_stream(err);@@ -190,6 +195,7 @@ impl Session {time_prefill_start: Instant::now(),time_first_token: None,total_tokens_input: self.input_tokens.len(),+ cached_tokens_input,total_tokens_output: 0,memory_usage: None,metrics: None,@@ -301,7 +307,10 @@ impl Session {} => Some(ToolCallState::Finished(value.clone())),ChatContentBlock::ToolCallCandidate {value,- } => Some(ToolCallState::Candidate(value.json.clone())),+ } => Some(ToolCallState::Candidate(+ // the block stores the candidate text as a JSON string document+ serde_json::from_str::<String>(&value.json).unwrap_or_default(),+ )),_ => None,}).collect::<Vec<_>>();@@ -345,6 +354,7 @@ struct StreamingState<'a> {time_prefill_start: Instant,time_first_token: Option<Instant>,total_tokens_input: usize,+ cached_tokens_input: usize,total_tokens_output: usize,memory_usage: Option<usize>,metrics: Option<TokenStreamMetrics>,@@ -411,6 +421,7 @@ impl StreamingState<'_> {prefill_tokens_per_second: prefill_tps,generate_tokens_per_second: generate_tps,tokens_count_input: Some(self.total_tokens_input as u32),+ tokens_count_input_cached: Some(self.cached_tokens_input as u32),tokens_count_output: Some(self.total_tokens_output as u32),memory_used_bytes: last_stat.then(|| self.memory_usage.map(|bytes| bytes as i64)).flatten(),speculator_stats,diff --git a/crates/legacy/shoji/src/types/basic/mod.rs b/crates/legacy/shoji/src/types/basic/mod.rsindex 09e536325..36f5eeda2 100644--- a/crates/legacy/shoji/src/types/basic/mod.rs+++ b/crates/legacy/shoji/src/types/basic/mod.rs@@ -40,4 +40,4 @@ pub use tool_description::ToolDescription;pub use tool_function::ToolFunction;pub use tool_namespace::ToolNamespace;pub use translation_payload::TranslationPayload;-pub use value::Value;+pub use value::{Value, parse_lenient_json};diff --git a/crates/legacy/shoji/src/types/basic/value.rs b/crates/legacy/shoji/src/types/basic/value.rsindex 6f17c2756..de57e6a0f 100644--- a/crates/legacy/shoji/src/types/basic/value.rs+++ b/crates/legacy/shoji/src/types/basic/value.rs@@ -53,3 +53,103 @@ impl TryFrom<Value> for serde_json::Value {serde_json::from_str(&value.json)}}++/// Parses JSON text produced by a model: strict first, then one repair pass that+/// escapes literal control characters and bare quotes inside string values.+pub fn parse_lenient_json(text: &str) -> Option<serde_json::Value> {+ serde_json::from_str(text).ok().or_else(|| serde_json::from_str(&repair_json_text(text)).ok())+}++/// Repairs model-generated JSON text inside string values: escapes literal+/// control characters, doubles backslashes that do not form a valid JSON+/// escape (the model meant a literal backslash), and escapes quotes that cannot+/// terminate a string (a closing quote must be followed by a JSON delimiter).+/// Text outside strings passes through untouched.+pub fn repair_json_text(text: &str) -> String {+ let mut out = String::with_capacity(text.len());+ let mut chars = text.chars().peekable();+ let mut in_string = false;+ while let Some(c) = chars.next() {+ if in_string {+ match c {+ '\n' => {+ out.push_str("\\n");+ continue;+ },+ '\r' => {+ out.push_str("\\r");+ continue;+ },+ '\t' => {+ out.push_str("\\t");+ continue;+ },+ c if (c as u32) < 0x20 => {+ out.push_str(&format!("\\u{:04x}", c as u32));+ continue;+ },+ '\\' => {+ if matches!(chars.peek(), Some('"' | '\\' | '/' | 'b' | 'f' | 'n' | 'r' | 't' | 'u')) {+ out.push('\\');+ out.push(chars.next().unwrap());+ } else {+ out.push_str("\\\\");+ }+ continue;+ },+ '"' => {+ let mut lookahead = chars.clone();+ let terminates = loop {+ match lookahead.next() {+ Some(c) if c.is_whitespace() => {},+ Some(',' | '}' | ']' | ':') | None => break true,+ Some(_) => break false,+ }+ };+ if !terminates {+ out.push('\\');+ } else {+ in_string = false;+ }+ out.push('"');+ continue;+ },+ _ => {},+ }+ } else if c == '"' {+ in_string = true;+ }+ out.push(c);+ }+ out+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn repair_json_text_only_touches_string_values() {+ assert_eq!(repair_json_text("\"esc\\ned\\n\""), "\"esc\\ned\\n\"");+ assert_eq!(repair_json_text("\"a\nb\""), "\"a\\nb\"");+ assert_eq!(repair_json_text("[1,\n2]"), "[1,\n2]");+ assert_eq!(+ repair_json_text(r#"{"a": "called "The Square Mile" β here"}"#),+ r#"{"a": "called \"The Square Mile\" β here"}"#+ );+ assert_eq!(repair_json_text(r#"{"a": "x", "b": "y"}"#), r#"{"a": "x", "b": "y"}"#);+ assert_eq!(repair_json_text(r#"{"cmd": "grep 'a\|b'}""#), r#"{"cmd": "grep 'a\\|b'}""#);+ assert_eq!(repair_json_text(r#"{"a": "x\ny\"z"}"#), r#"{"a": "x\ny\"z"}"#);+ assert_eq!(repair_json_text(r#"{"a": "C:\\temp"}"#), r#"{"a": "C:\\temp"}"#);+ }++ #[test]+ fn parse_lenient_json_repairs_control_characters_and_bare_quotes() {+ assert_eq!(parse_lenient_json("{ \"content\": \"a\nb\" }"), Some(serde_json::json!({ "content": "a\nb" })));+ assert_eq!(+ parse_lenient_json(r#"{ "content": "called "The Square Mile" home" }"#),+ Some(serde_json::json!({ "content": "called \"The Square Mile\" home" }))+ );+ assert_eq!(parse_lenient_json("{oops"), None);+ }+}diff --git a/crates/legacy/shoji/src/types/session/chat/reply/stats.rs b/crates/legacy/shoji/src/types/session/chat/reply/stats.rsindex 136716dd7..19d94097d 100644--- a/crates/legacy/shoji/src/types/session/chat/reply/stats.rs+++ b/crates/legacy/shoji/src/types/session/chat/reply/stats.rs@@ -27,6 +27,7 @@ pub struct ChatReplyStats {pub prefill_tokens_per_second: Option<f64>,pub generate_tokens_per_second: Option<f64>,pub tokens_count_input: Option<u32>,+ pub tokens_count_input_cached: Option<u32>,pub tokens_count_output: Option<u32>,pub memory_used_bytes: Option<i64>,pub speculator_stats: Option<ChatReplySpeculatorStats>,diff --git a/crates/legacy/token-stream-parser/tests/test_token_stream_parser_muse_glimmer.rs b/crates/legacy/token-stream-parser/tests/test_token_stream_parser_muse_glimmer.rsindex 4c39928a6..6cd689b3a 100644--- a/crates/legacy/token-stream-parser/tests/test_token_stream_parser_muse_glimmer.rs+++ b/crates/legacy/token-stream-parser/tests/test_token_stream_parser_muse_glimmer.rs@@ -157,3 +157,114 @@ fn test_token_stream_parser_muse_glimmer_tool_output_and_reply() {},);}++mod hermetic {+ use std::path::PathBuf;++ use token_stream_parser::{+ Parser,+ token_stream::{TokenStreamParser, TokenStreamParserConfig},+ types::Token,+ };++ // Fabricated tokens: the parser matches framing tokens by value, so no tokenizer is needed.+ fn feed(+ parser: &mut TokenStreamParser,+ text: &str,+ next_id: &mut u32,+ ) {+ const SPECIALS: [&str; 6] =+ ["<|begin_of_text|>", "<|start|>", "<|message|>", "<|eom|>", "<|eot|>", "<|end_of_text|>"];+ let mut rest = text;+ while !rest.is_empty() {+ let next = SPECIALS.iter().filter_map(|special| rest.find(special).map(|i| (i, special))).min();+ match next {+ Some((0, special)) => {+ parser+ .push(&Token {+ id: *next_id,+ value: special.to_string(),+ is_special: true,+ })+ .unwrap();+ *next_id += 1;+ rest = &rest[special.len()..];+ },+ Some((index, _)) => {+ push_text(parser, &rest[..index], next_id);+ rest = &rest[index..];+ },+ None => {+ push_text(parser, rest, next_id);+ rest = "";+ },+ }+ }+ }++ fn push_text(+ parser: &mut TokenStreamParser,+ text: &str,+ next_id: &mut u32,+ ) {+ if text.is_empty() {+ return;+ }+ parser+ .push(&Token {+ id: *next_id,+ value: text.to_string(),+ is_special: false,+ })+ .unwrap();+ *next_id += 1;+ }++ fn parser() -> TokenStreamParser {+ let config_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))+ .join("..")+ .join("hanashi")+ .join("configs")+ .join("parsing")+ .join("muse-glimmer.json");+ let config: TokenStreamParserConfig =+ serde_json::from_str(&std::fs::read_to_string(config_path).unwrap()).unwrap();+ TokenStreamParser::new(config).unwrap()+ }++ #[test]+ fn unfinished_tool_call_does_not_parse_as_finished() {+ let mut parser = parser();+ let mut next_id = 0;+ feed(&mut parser, "<|start|>user<|message|>hi<|eot|><|start|>assistant", &mut next_id);+ // stream stops mid-call, right after the markup's first char+ feed(&mut parser, " to=bash<|message|><", &mut next_id);++ let assistant = &parser.state().value[1];+ let section = &assistant["content"][0];+ assert_eq!(section["type"], "tool_call");+ assert!(section["value"].get("arguments").is_none(), "an unfinished call must not expose arguments: {section}");+ }++ #[test]+ fn closed_tool_call_parses_arguments() {+ let mut parser = parser();+ let mut next_id = 0;+ feed(&mut parser, "<|start|>user<|message|>hi<|eot|><|start|>assistant", &mut next_id);+ feed(+ &mut parser,+ concat!(+ " to=bash<|message|>",+ "<atem:function_calls>\n<atem:invoke name=\"bash\">\n",+ "<atem:parameter name=\"command\">printenv</atem:parameter>\n",+ "</atem:invoke>\n</atem:function_calls><|eot|>"+ ),+ &mut next_id,+ );++ let assistant = &parser.state().value[1];+ let section = &assistant["content"][0];+ assert_eq!(section["value"]["name"], "bash");+ assert_eq!(section["value"]["arguments"]["command"], "printenv");+ }+}