A high-performance inference engine for AI models
Glimmer and oai server fixes (#771)
Aleksandr Golokoz9e4ddada3c8c
diff --git a/crates/legacy/cli/src/server/chat_completions.rs b/crates/legacy/cli/src/server/chat_completions.rsindex 614b42055..ecc64899e 100644--- a/crates/legacy/cli/src/server/chat_completions.rs+++ b/crates/legacy/cli/src/server/chat_completions.rs@@ -10,11 +10,11 @@ use rocket::{Request, State,data::{ByteUnit, Data},futures::Stream,- http::Status,+ http::{ContentType, Status},post,response::{Responder, status,- stream::{Event, EventStream},+ stream::{ByteStream, Event, EventStream},},serde::json::Json,};@@ -23,7 +23,7 @@ use tokio::sync::{Mutex, mpsc};use tokio_stream::wrappers::UnboundedReceiverStream;use uuid::Uuid;use uzu::{- session::chat::{ChatSession, ChatSessionStreamChunk, UNPARSED_ARGUMENTS_KEY},+ session::chat::{ChatSession, ChatSessionStream, ChatSessionStreamChunk, UNPARSED_ARGUMENTS_KEY},types::{basic::{Grammar, ReasoningEffort, SamplingMethod},session::chat::{@@ -37,8 +37,9 @@ use crate::{server::{ServerState,chat_tool_calls::{- OaiTool, OaiToolCall, ToolCallStreamer, backfill_tool_result_names, choose_tools, insert_tools_message,- oai_tool_call, reply_tool_calls, to_tool_call, tool_call_result_block, withhold_stream_text,+ OaiTool, OaiToolCall, ToolCallStreamer, ToolParameterTypes, backfill_tool_result_names, choose_tools,+ coerce_tool_call, insert_tools_message, oai_tool_call, parse_scalar_text, reply_tool_calls, to_tool_call,+ tool_call_result_block, withhold_stream_text,},request_log::RequestLog,},@@ -216,7 +217,10 @@ struct OaiError {}pub enum ChatCompletionResult {- Json(Json<ChatCompletionResponse>),+ // The single JSON body arrives as a stream: Rocket only surfaces a client+ // disconnect by dropping the body stream, which is the blocking path's one+ // signal to cancel a generation nobody is waiting for.+ Json(Status, ByteStream<Pin<Box<dyn Stream<Item = Vec<u8>> + Send>>>),Stream(EventStream<Pin<Box<dyn Stream<Item = Event> + Send>>>),Error(status::Custom<Json<OaiErrorResponse>>),}@@ -227,7 +231,11 @@ impl<'r> Responder<'r, 'r> for ChatCompletionResult {request: &'r Request<'_>,) -> rocket::response::Result<'r> {match self {- ChatCompletionResult::Json(json) => json.respond_to(request),+ ChatCompletionResult::Json(status, body) => {+ let mut response = (ContentType::JSON, body).respond_to(request)?;+ response.set_status(status);+ Ok(response)+ },ChatCompletionResult::Stream(stream) => stream.respond_to(request),ChatCompletionResult::Error(error) => error.respond_to(request),}@@ -489,6 +497,28 @@ fn json_schema_grammar(json_schema: &JsonSchemaFormat) -> Result<Grammar, Respon})}+/// Wrong-typed values already fail JSON extraction; out-of-range numbers would+/// otherwise reach the sampler unchecked (a huge temperature generates garbage+/// with HTTP 200).+fn validate_sampling(request: &ChatCompletionRequest) -> Result<(), (&'static str, String)> {+ if let Some(temperature) = request.temperature+ && !(0.0..=2.0).contains(&temperature)+ {+ return Err(("temperature", format!("temperature must be between 0 and 2, got {temperature}")));+ }+ if let Some(top_p) = request.top_p+ && !(top_p > 0.0 && top_p <= 1.0)+ {+ return Err(("top_p", format!("top_p must be greater than 0 and at most 1, got {top_p}")));+ }+ if let Some(top_k) = request.top_k+ && top_k < 1+ {+ return Err(("top_k", format!("top_k must be at least 1, got {top_k}")));+ }+ Ok(())+}+fn build_reply_config(request: &ChatCompletionRequest) -> Result<ChatReplyConfig, ResponseFormatError> {let token_limit = request.max_completion_tokens.or(request.max_tokens);let mut config = ChatReplyConfig::default().with_token_limit(token_limit);@@ -557,32 +587,43 @@ fn usage_from_stats(}}-fn error_response(- id: String,- model: String,- created: i64,+/// nagare re-wraps backend errors at each layer, so a message can arrive as+/// "Backend error: Backend error: ..."; the wire message keeps only the inner text.+fn strip_backend_prefixes(message: &str) -> &str {+ let mut message = message;+ while let Some(inner) = message.strip_prefix("Backend error: ") {+ message = inner;+ }+ message+}++fn oai_error_body(+ status: Status,+ code: &str,message: &str,-) -> ChatCompletionResponse {- ChatCompletionResponse {- id,- object: "chat.completion".to_string(),- created,- model,- choices: vec![ChatCompletionChoice {- index: 0,- message: OaiMessage {- role: "assistant".to_string(),- content: Some(format!("Error: {message}")),- reasoning_content: None,- tool_calls: None,- tool_call_id: None,- },- finish_reason: "stop".to_string(),- }],- usage: ChatCompletionUsage::default(),+) -> OaiErrorResponse {+ let kind = if status.code < 500 {+ "invalid_request_error"+ } else {+ "server_error"+ };+ OaiErrorResponse {+ error: OaiError {+ message: strip_backend_prefixes(message).to_string(),+ kind: kind.to_string(),+ param: None,+ code: Some(code.to_string()),+ },}}+fn send_error_body(+ sender: &mpsc::UnboundedSender<Vec<u8>>,+ body: &OaiErrorResponse,+) {+ let _ = sender.send(serde_json::to_vec(body).unwrap_or_default());+}+fn chunk_json(id: &str,model: &str,@@ -606,6 +647,41 @@ fn chunk_json(serde_json::to_string(&chunk).unwrap_or_default()}+/// Schema coercion retypes scalar arguments on the wire, so a client may echo+/// `5` where the session stored `"5"` (or a JSON string where the session+/// stored the value it spells); such pairs must still count as the same prefix.+fn tool_call_arguments_equivalent(+ stored: &str,+ echoed: &str,+) -> bool {+ if stored == echoed {+ return true;+ }+ match (serde_json::from_str::<serde_json::Value>(stored), serde_json::from_str::<serde_json::Value>(echoed)) {+ (Ok(stored), Ok(echoed)) => json_values_equivalent(&stored, &echoed),+ _ => false,+ }+}++fn json_values_equivalent(+ left: &serde_json::Value,+ right: &serde_json::Value,+) -> bool {+ use serde_json::Value;+ match (left, right) {+ (Value::Object(left), Value::Object(right)) => {+ left.len() == right.len()+ && left+ .iter()+ .all(|(key, value)| right.get(key).is_some_and(|other| json_values_equivalent(value, other)))+ },+ (Value::String(text), other) | (other, Value::String(text)) if !other.is_string() => {+ parse_scalar_text(text).is_some_and(|parsed| parsed == *other)+ },+ (left, right) => left == right,+ }+}+/// Tool call identifiers are assigned per turn and never rendered into tokens, so they are/// excluded from the comparison; everything else about the messages must match.fn messages_have_prefix(@@ -627,10 +703,8 @@ fn messages_have_prefix(value: call,},) => {- let (mut prefix_call, mut call) = (prefix_call.clone(), call.clone());- prefix_call.identifier = None;- call.identifier = None;- prefix_call == call+ prefix_call.name == call.name+ && tool_call_arguments_equivalent(&prefix_call.arguments.json, &call.arguments.json)},_ => prefix_block == block,}@@ -657,6 +731,21 @@ async fn prepare_input(Ok(messages)}+fn send_response(+ sender: &mpsc::UnboundedSender<Vec<u8>>,+ response: &ChatCompletionResponse,+) {+ let _ = sender.send(serde_json::to_vec(response).unwrap_or_default());+}++/// Cancels the turn and waits for it to wind down before the caller releases+/// the session lock: a request arriving right after a cancellation must not+/// catch the session mid-cleanup, where it rejects operations.+async fn cancel_and_drain(stream: &ChatSessionStream) {+ stream.cancel_token().cancel();+ while stream.next().await.is_some() {}+}+async fn run_blocking(session: Arc<Mutex<ChatSession>>,messages: Vec<ChatMessage>,@@ -665,80 +754,126 @@ async fn run_blocking(model: String,created: i64,prefix_cache: bool,+ parameter_types: ToolParameterTypes,+ sender: mpsc::UnboundedSender<Vec<u8>>,log: RequestLog,-) -> ChatCompletionResponse {- let session = session.lock().await;+) {+ // A blocking response writes nothing until generation finishes, and Rocket+ // only discovers a vanished client through a failed socket write, which is+ // what makes it drop the body receiver that `sender.closed()` observes. A+ // periodic newline — insignificant leading whitespace for the JSON body —+ // keeps forcing that write while the request queues and generates.+ let mut keepalive = tokio::time::interval(std::time::Duration::from_secs(3));++ // Requests serialize on the single session, so a client can disconnect+ // while still queued; generating for it would block everyone behind it.+ let mut lock = std::pin::pin!(session.lock());+ let session = loop {+ tokio::select! {+ session = &mut lock => break session,+ () = sender.closed() => {+ log.fail("client disconnected");+ return;+ },+ _ = keepalive.tick() => {+ let _ = sender.send(b"\n".to_vec());+ },+ }+ };let input = match prepare_input(&session, messages, prefix_cache).await {Ok(input) => input,Err(error) => {log.fail(&error.to_string());- return error_response(id, model, created, &error.to_string());+ send_error_body(&sender, &oai_error_body(Status::InternalServerError, "backend_error", &error.to_string()));+ return;},};- match session.reply(input, config).await {- Ok(replies) => match replies.last() {- Some(reply) => {- let tool_calls = reply_tool_calls(&reply.message);- // OpenAI sets content to null (not "") on tool call replies.- let content = reply.message.text().or_else(|| tool_calls.is_none().then(String::new));- let mut finish_reason =- reply.finish_reason.as_ref().map(map_finish_reason).unwrap_or_else(|| "stop".to_string());- // Calls the parser could not finalize must not execute as a partial batch.- // OpenAI has no finish reason for this; providers signal terminal errors with- // unrecognized reasons (pi maps them to a thrown provider error), so the turn- // fails loudly instead of stalling on an absent or incomplete tool_calls array.- let has_unfinished_candidates = reply- .message- .content- .iter()- .any(|block| matches!(block, ChatContentBlock::ToolCallCandidate { .. }));- if finish_reason == "tool_calls" && (tool_calls.is_none() || has_unfinished_candidates) {- finish_reason = "malformed_tool_call".to_string();- }- let wrapped_arguments = reply- .message- .tool_calls()- .iter()- .filter(|call| call.arguments.json.contains(UNPARSED_ARGUMENTS_KEY))- .count();- let mut notes = Vec::new();- if has_unfinished_candidates {- notes.push("model generated unfinished tool call(s)".to_string());- }- if wrapped_arguments > 0 {- notes.push(format!("{wrapped_arguments} tool call(s) with unparseable arguments"));- }- log.finish(&finish_reason, Some(&reply.stats), notes);- ChatCompletionResponse {- id,- object: "chat.completion".to_string(),- created,- model,- choices: vec![ChatCompletionChoice {- index: 0,- message: OaiMessage {- role: "assistant".to_string(),- content,- reasoning_content: reply.message.reasoning().filter(|reasoning| !reasoning.is_empty()),- tool_calls,- tool_call_id: None,- },- finish_reason,- }],- usage: usage_from_stats(&reply.stats, prefix_cache),- }+ // Drive the reply through its stream instead of ChatSession::reply, which+ // cannot be cancelled: racing each chunk against the channel closing+ // notices a disconnect mid-generation. Cancelling stops the backend at the+ // next token boundary and lets the session run its cancelled-turn cleanup.+ let stream = session.reply_with_stream(input, config).await;+ let mut final_replies = Vec::new();+ loop {+ let chunk = tokio::select! {+ chunk = stream.next() => chunk,+ () = sender.closed() => {+ cancel_and_drain(&stream).await;+ log.fail("client disconnected");+ return;},- None => {- log.fail("no response generated");- error_response(id, model, created, "No response generated")+ _ = keepalive.tick() => {+ let _ = sender.send(b"\n".to_vec());+ continue;},- },- Err(error) => {- log.fail(&error.to_string());- error_response(id, model, created, &error.to_string())- },+ };+ let Some(chunk) = chunk else {+ break;+ };+ match chunk {+ ChatSessionStreamChunk::Replies {+ replies,+ } => {+ final_replies = replies;+ },+ ChatSessionStreamChunk::Error {+ error,+ } => {+ log.fail(&error.to_string());+ send_error_body(&sender, &oai_error_body(Status::BadRequest, "backend_error", &error.to_string()));+ return;+ },+ }}++ let Some(reply) = final_replies.last() else {+ log.fail("no response generated");+ send_error_body(&sender, &oai_error_body(Status::InternalServerError, "no_response", "No response generated"));+ return;+ };+ let tool_calls = reply_tool_calls(&reply.message, ¶meter_types);+ // OpenAI sets content to null (not "") on tool call replies.+ let content = reply.message.text().or_else(|| tool_calls.is_none().then(String::new));+ let mut finish_reason = reply.finish_reason.as_ref().map(map_finish_reason).unwrap_or_else(|| "stop".to_string());+ // Calls the parser could not finalize must not execute as a partial batch.+ // OpenAI has no finish reason for this; providers signal terminal errors with+ // unrecognized reasons (pi maps them to a thrown provider error), so the turn+ // fails loudly instead of stalling on an absent or incomplete tool_calls array.+ let has_unfinished_candidates =+ reply.message.content.iter().any(|block| matches!(block, ChatContentBlock::ToolCallCandidate { .. }));+ if finish_reason == "tool_calls" && (tool_calls.is_none() || has_unfinished_candidates) {+ finish_reason = "malformed_tool_call".to_string();+ }+ let wrapped_arguments =+ reply.message.tool_calls().iter().filter(|call| call.arguments.json.contains(UNPARSED_ARGUMENTS_KEY)).count();+ let mut notes = Vec::new();+ if has_unfinished_candidates {+ notes.push("model generated unfinished tool call(s)".to_string());+ }+ if wrapped_arguments > 0 {+ notes.push(format!("{wrapped_arguments} tool call(s) with unparseable arguments"));+ }+ log.finish(&finish_reason, Some(&reply.stats), notes);+ let response = ChatCompletionResponse {+ id,+ object: "chat.completion".to_string(),+ created,+ model,+ choices: vec![ChatCompletionChoice {+ index: 0,+ message: OaiMessage {+ role: "assistant".to_string(),+ content,+ reasoning_content: reply.message.reasoning().filter(|reasoning| !reasoning.is_empty()),+ tool_calls,+ tool_call_id: None,+ },+ finish_reason,+ }],+ usage: usage_from_stats(&reply.stats, prefix_cache),+ };+ send_response(&sender, &response);}async fn run_stream(@@ -749,26 +884,28 @@ async fn run_stream(model: String,created: i64,prefix_cache: bool,+ parameter_types: ToolParameterTypes,sender: mpsc::UnboundedSender<Event>,log: RequestLog,) {- let session = session.lock().await;+ // Same as the blocking path: don't start generating for a client that+ // disconnected while queued on the session.+ let session = tokio::select! {+ session = session.lock() => session,+ () = sender.closed() => {+ log.fail("client disconnected");+ return;+ },+ };let has_tools = messages.iter().any(|message| !message.tool_namespaces().is_empty());let input = match prepare_input(&session, messages, prefix_cache).await {Ok(input) => input,Err(error) => {log.fail(&error.to_string());- let _ = sender.send(Event::data(chunk_json(- &id,- &model,- created,- StreamDelta {- content: Some(format!("Error: {error}")),- ..StreamDelta::default()- },- Some("stop".to_string()),- None,- )));+ // The SSE response is already committed as 200, so the error travels+ // as an OpenAI-style error event, the way OpenAI reports mid-stream.+ let body = oai_error_body(Status::InternalServerError, "backend_error", &error.to_string());+ let _ = sender.send(Event::data(serde_json::to_string(&body).unwrap_or_default()));let _ = sender.send(Event::data("[DONE]"));return;},@@ -787,7 +924,6 @@ async fn run_stream()));let stream = session.reply_with_stream(input, config).await;- let cancel_token = stream.cancel_token();let mut emitted = 0usize;let mut emitted_reasoning = 0usize;let mut emitted_tool_calls = 0usize;@@ -807,8 +943,7 @@ async fn run_stream(let chunk = tokio::select! {chunk = stream.next() => chunk,() = sender.closed() => {- cancel_token.cancel();- cancel_token.cancel();+ cancel_and_drain(&stream).await;log.fail("client disconnected");return;},@@ -842,7 +977,7 @@ async fn run_stream(None,)));if sent.is_err() {- cancel_token.cancel();+ cancel_and_drain(&stream).await;log.fail("client disconnected");return;}@@ -864,7 +999,7 @@ async fn run_stream(None,)));if sent.is_err() {- cancel_token.cancel();+ cancel_and_drain(&stream).await;log.fail("client disconnected");return;}@@ -882,8 +1017,11 @@ async fn run_stream(// candidates carry their partial text as a JSON string document, except// object-valued ones (e.g. muse-glimmer's name-only progress objects)let raw = serde_json::from_str::<String>(&value.json).unwrap_or_else(|_| value.json.clone());- let deltas =- tool_call_streamers.entry(index).or_insert_with(ToolCallStreamer::new).update(index, &raw);+ let deltas = tool_call_streamers.entry(index).or_insert_with(ToolCallStreamer::new).update(+ index,+ &raw,+ ¶meter_types,+ );for delta in deltas {let sent = sender.send(Event::data(chunk_json(&id,@@ -897,7 +1035,7 @@ async fn run_stream(None,)));if sent.is_err() {- cancel_token.cancel();+ cancel_and_drain(&stream).await;log.fail("client disconnected");return;}@@ -908,13 +1046,13 @@ async fn run_stream(while emitted_tool_calls < tool_calls.len() {let index = emitted_tool_calls;emitted_tool_calls += 1;- let call = &tool_calls[index];- if call.arguments.json.contains(UNPARSED_ARGUMENTS_KEY) {+ if tool_calls[index].arguments.json.contains(UNPARSED_ARGUMENTS_KEY) {wrapped_arguments += 1;}+ let call = coerce_tool_call(&tool_calls[index], ¶meter_types);let delta = match tool_call_streamers.remove(&index) {- Some(mut streamer) => streamer.finish(index, call),- None => oai_tool_call(Some(index), call),+ Some(mut streamer) => streamer.finish(index, &call),+ None => oai_tool_call(Some(index), &call),};let sent = sender.send(Event::data(chunk_json(&id,@@ -928,7 +1066,7 @@ async fn run_stream(None,)));if sent.is_err() {- cancel_token.cancel();+ cancel_and_drain(&stream).await;log.fail("client disconnected");return;}@@ -950,17 +1088,9 @@ async fn run_stream(} => {errored = true;log.fail(&error.to_string());- let _ = sender.send(Event::data(chunk_json(- &id,- &model,- created,- StreamDelta {- content: Some(format!("Error: {error}")),- ..StreamDelta::default()- },- Some("stop".to_string()),- None,- )));+ // Committed 200 stream: report as an OpenAI-style error event.+ let body = oai_error_body(Status::BadRequest, "backend_error", &error.to_string());+ let _ = sender.send(Event::data(serde_json::to_string(&body).unwrap_or_default()));break;},}@@ -984,7 +1114,7 @@ async fn run_stream(None,)));if sent.is_err() {- cancel_token.cancel();+ cancel_and_drain(&stream).await;log.fail("client disconnected");return;}@@ -1056,6 +1186,10 @@ pub async fn handle_chat_completions(request.reasoning_effort.as_ref().and_then(serde_json::Value::as_str).or(Some("unspecified")),);+ if let Err((param, message)) = validate_sampling(&request) {+ log.fail(&message);+ return invalid_request_response(param, "out_of_range", message);+ }let config = match build_reply_config(&request) {Ok(config) => config,Err(error) => {@@ -1074,6 +1208,10 @@ pub async fn handle_chat_completions(},};+ // The parser cannot type scalar arguments; the declared schemas restore+ // the types when replies cross back into the OpenAI wire format.+ let parameter_types = ToolParameterTypes::from_tools(request.tools.as_deref());+if is_stream {let session = Arc::clone(&state.session);let (sender, receiver) = mpsc::unbounded_channel::<Event>();@@ -1085,6 +1223,7 @@ pub async fn handle_chat_completions(model,created,state.prefix_cache,+ parameter_types,sender,log,));@@ -1092,8 +1231,24 @@ pub async fn handle_chat_completions(ChatCompletionResult::Stream(EventStream::from(body))} else {let session = Arc::clone(&state.session);- let response = run_blocking(session, messages, config, id, model, created, state.prefix_cache, log).await;- ChatCompletionResult::Json(Json(response))+ let (sender, receiver) = mpsc::unbounded_channel::<Vec<u8>>();+ rocket::tokio::spawn(run_blocking(+ session,+ messages,+ config,+ id,+ model,+ created,+ state.prefix_cache,+ parameter_types,+ sender,+ log,+ ));+ let body: Pin<Box<dyn Stream<Item = Vec<u8>> + Send>> = Box::pin(UnboundedReceiverStream::new(receiver));+ // Hand the body to Rocket immediately so it can write keepalives and+ // drop the receiver when the client disconnects. Any backend error+ // occurs after the 200 response is committed and is sent in the body.+ ChatCompletionResult::Json(Status::Ok, ByteStream::from(body))}}diff --git a/crates/legacy/cli/src/server/chat_tool_calls.rs b/crates/legacy/cli/src/server/chat_tool_calls.rsindex 5ac1e0851..cb8fd6fa4 100644--- a/crates/legacy/cli/src/server/chat_tool_calls.rs+++ b/crates/legacy/cli/src/server/chat_tool_calls.rs@@ -15,16 +15,17 @@ pub struct OaiToolCall {// Present only in streaming deltas, per the OpenAI wire format.#[serde(default, skip_serializing_if = "Option::is_none")]pub index: Option<usize>,- // Empty in fragments: only the announcement and the final delta carry the id.+ // Empty after announcement; finish carries metadata only if no announcement occurred.#[serde(default, skip_serializing_if = "String::is_empty")]pub id: String,- #[serde(rename = "type", default)]+ #[serde(rename = "type", default, skip_serializing_if = "String::is_empty")]pub kind: String,pub function: OaiFunctionCall,}#[derive(Serialize, Deserialize, Clone)]pub struct OaiFunctionCall {+ #[serde(default, skip_serializing_if = "String::is_empty")]pub name: String,pub arguments: String,}@@ -91,6 +92,133 @@ pub fn choose_tools<'t>(}}+/// Declared JSON-Schema types of each tool parameter, keyed by function name.+/// Tool-call markup cannot carry scalar types — the parser keeps every scalar+/// parameter a string and types JSON-shaped values by their braces — so the+/// declared schema is what restores the wire types clients validate against.+#[derive(Clone, Default)]+pub struct ToolParameterTypes(HashMap<String, HashMap<String, Vec<String>>>);++impl ToolParameterTypes {+ pub fn from_tools(tools: Option<&[OaiTool]>) -> Self {+ let mut functions = HashMap::new();+ for tool in tools.unwrap_or_default() {+ let Some(parameters) = &tool.function.parameters else {+ continue;+ };+ let Ok(schema) = serde_json::from_str::<serde_json::Value>(¶meters.json) else {+ continue;+ };+ let Some(properties) = schema.get("properties").and_then(serde_json::Value::as_object) else {+ continue;+ };+ let parameters = properties+ .iter()+ .filter_map(|(name, property)| {+ let types = match property.get("type")? {+ serde_json::Value::String(kind) => vec![kind.clone()],+ serde_json::Value::Array(kinds) => {+ kinds.iter().filter_map(|kind| kind.as_str().map(str::to_string)).collect()+ },+ _ => return None,+ };+ (!types.is_empty()).then(|| (name.clone(), types))+ })+ .collect();+ functions.insert(tool.function.name.clone(), parameters);+ }+ Self(functions)+ }++ fn declared(+ &self,+ function: &str,+ parameter: &str,+ ) -> Option<&[String]> {+ Some(self.0.get(function)?.get(parameter)?.as_slice())+ }+}++fn matches_declared_type(+ value: &serde_json::Value,+ declared: &[String],+) -> bool {+ declared.iter().any(|kind| match kind.as_str() {+ "integer" => value.as_i64().is_some() || value.as_u64().is_some(),+ "number" => value.is_number(),+ "boolean" => value.is_boolean(),+ "null" => value.is_null(),+ "object" => value.is_object(),+ "array" => value.is_array(),+ "string" => value.is_string(),+ _ => false,+ })+}++/// Bare scalar text is strict JSON, plus the Python-style booleans some models+/// emit in tool markup (qwen3.5 writes `True`/`False`).+pub fn parse_scalar_text(text: &str) -> Option<serde_json::Value> {+ if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(text) {+ return Some(parsed);+ }+ match text.trim() {+ text if text.eq_ignore_ascii_case("true") => Some(serde_json::Value::Bool(true)),+ text if text.eq_ignore_ascii_case("false") => Some(serde_json::Value::Bool(false)),+ _ => None,+ }+}++fn coerce_parameter_value(+ value: &serde_json::Value,+ declared: &[String],+) -> Option<serde_json::Value> {+ let declares_string = declared.iter().any(|kind| kind == "string");+ match value {+ // The parser kept the bare markup text as a string; restore the+ // declared type when the text reads as it. A union that includes+ // "string" stays a string: the text is already schema-valid and the+ // intended type is unknowable.+ serde_json::Value::String(text) if !declares_string => {+ let parsed = parse_scalar_text(text)?;+ matches_declared_type(&parsed, declared).then_some(parsed)+ },+ // JSON-shaped markup text was typed by its braces although the+ // parameter is a plain string.+ value if declared == ["string"] && !value.is_string() => Some(serde_json::Value::String(value.to_string())),+ _ => None,+ }+}++/// Restores the declared scalar types the markup could not carry. Applied only+/// at the OpenAI boundary: the session keeps the parser's values so its stored+/// history stays consistent with what the template renders.+pub fn coerce_tool_call(+ tool_call: &ToolCall,+ types: &ToolParameterTypes,+) -> ToolCall {+ let Ok(serde_json::Value::Object(mut object)) = serde_json::from_str(&tool_call.arguments.json) else {+ return tool_call.clone();+ };+ let mut changed = false;+ for (parameter, value) in object.iter_mut() {+ if let Some(declared) = types.declared(&tool_call.name, parameter)+ && let Some(coerced) = coerce_parameter_value(value, declared)+ {+ *value = coerced;+ changed = true;+ }+ }+ if !changed {+ return tool_call.clone();+ }+ ToolCall {+ arguments: Value {+ json: serde_json::Value::Object(object).to_string(),+ },+ ..tool_call.clone()+ }+}+pub fn to_tool_call(tool_call: &OaiToolCall) -> ToolCall {// Templates render arguments as key/value pairs; normalization guarantees// an object no matter what the client echoes back.@@ -192,9 +320,13 @@ pub fn oai_tool_call(}}-pub fn reply_tool_calls(message: &ChatMessage) -> Option<Vec<OaiToolCall>> {+pub fn reply_tool_calls(+ message: &ChatMessage,+ types: &ToolParameterTypes,+) -> Option<Vec<OaiToolCall>> {let tool_calls = message.tool_calls();- (!tool_calls.is_empty()).then(|| tool_calls.iter().map(|tool_call| oai_tool_call(None, tool_call)).collect())+ (!tool_calls.is_empty())+ .then(|| tool_calls.iter().map(|tool_call| oai_tool_call(None, &coerce_tool_call(tool_call, types))).collect())}// Bare-JSON formats (e.g. llama-3) stream a tool call as ordinary text and only rewrite@@ -267,12 +399,27 @@ fn parse_framed_call(raw: &str) -> FramedCall {}}-// Mirrors the parser's synthesis rule: values starting with `{` or `[` are typed JSON,-// everything else is a string.-fn serialize_param_value(value: &str) -> String {+// Mirrors the parser's synthesis rule composed with the schema coercion the+// final call goes through: values starting with `{` or `[` are typed JSON+// unless the parameter is declared a plain string, bare scalars take a+// declared non-string type when they parse as it, everything else is a string.+fn serialize_param_value(+ value: &str,+ declared: Option<&[String]>,+) -> String {let trimmed = value.trim_start();- if (trimmed.starts_with('{') || trimmed.starts_with('['))- && let Some(parsed) = parse_lenient_json(value)+ let json_shaped = trimmed.starts_with('{') || trimmed.starts_with('[');+ if json_shaped && let Some(parsed) = parse_lenient_json(value) {+ if declared.is_some_and(|declared| declared == ["string"]) {+ return serde_json::Value::String(parsed.to_string()).to_string();+ }+ return parsed.to_string();+ }+ if !json_shaped+ && let Some(declared) = declared+ && !declared.iter().any(|kind| kind == "string")+ && let Some(parsed) = parse_scalar_text(value)+ && matches_declared_type(&parsed, declared){return parsed.to_string();}@@ -297,17 +444,29 @@ fn withhold_ambiguous_tail(content: &str) -> &str {// The arguments text with no closing brace: completed parameters are frozen, the open// parameter's value streams as it grows.-fn open_arguments_text(params: &[(String, FramedParam)]) -> Option<String> {+fn open_arguments_text(+ params: &[(String, FramedParam)],+ function: Option<&str>,+ types: &ToolParameterTypes,+) -> Option<String> {let mut entries: Vec<(String, String)> = Vec::new();for (key, param) in params {+ let declared = function.and_then(|function| types.declared(function, key));let serialized = match param {- FramedParam::Complete(value) => serialize_param_value(value),+ FramedParam::Complete(value) => serialize_param_value(value, declared),FramedParam::Open(content) => {- // an empty value has no known type yet, and typed values (JSON starting- // with `{`/`[`, matching the parser's synthesis rule) can only be serialized- // once complete: either way nothing about the parameter may be emitted+ // an empty value has no known type yet, and typed values (JSON+ // starting with `{`/`[`, or declared non-string so the final+ // form is a bare literal) can only be serialized once+ // complete: either way nothing about the parameter may be emittedlet streamable = withhold_ambiguous_tail(content);- if streamable.is_empty() || streamable.starts_with('{') || streamable.starts_with('[') {+ let declared_non_string =+ declared.is_some_and(|declared| !declared.iter().any(|kind| kind == "string"));+ if streamable.is_empty()+ || streamable.starts_with('{')+ || streamable.starts_with('[')+ || declared_non_string+ {break;}let quoted = serde_json::Value::String(streamable.to_string()).to_string();@@ -367,6 +526,7 @@ impl ToolCallStreamer {&mut self,index: usize,raw: &str,+ types: &ToolParameterTypes,) -> Vec<OaiToolCall> {let mut deltas = Vec::new();let name;@@ -374,14 +534,14 @@ impl ToolCallStreamer {let call = parse_framed_call(raw);name = call.name.filter(|name| !name.is_empty());self.key_order = call.params.iter().map(|(key, _)| key.clone()).collect();- if let Some(open_text) = open_arguments_text(&call.params)+ if let Some(open_text) = open_arguments_text(&call.params, name.as_deref(), types)&& open_text.len() > self.open_text.len()&& open_text.starts_with(&self.open_text){deltas.push(OaiToolCall {index: Some(index),id: String::new(),- kind: "function".to_string(),+ kind: String::new(),function: OaiFunctionCall {name: String::new(),arguments: open_text[self.open_text.len()..].to_string(),@@ -423,12 +583,17 @@ impl ToolCallStreamer {} else {self.closing_fragment(call)};+ let (id, kind, name) = if self.name_announced {+ (String::new(), String::new(), String::new())+ } else {+ (self.id.clone(), "function".to_string(), call.name.clone())+ };OaiToolCall {index: Some(index),- id: self.id.clone(),- kind: "function".to_string(),+ id,+ kind,function: OaiFunctionCall {- name: call.name.clone(),+ name,arguments,},}diff --git a/crates/legacy/cli/unit/server/chat_completions_test.rs b/crates/legacy/cli/unit/server/chat_completions_test.rsindex eea23674b..be13a2c31 100644--- a/crates/legacy/cli/unit/server/chat_completions_test.rs+++ b/crates/legacy/cli/unit/server/chat_completions_test.rs@@ -81,7 +81,7 @@ fn response_format_validation_errors_are_request_errors() {let error = ResponseFormatError::GrammarUnsupported;match invalid_request_response("response_format", error.code(), error.message()) {ChatCompletionResult::Error(_) => {},- ChatCompletionResult::Json(_) | ChatCompletionResult::Stream(_) => {+ ChatCompletionResult::Json(_, _) | ChatCompletionResult::Stream(_) => {panic!("response_format validation errors should be request errors")},}@@ -429,3 +429,84 @@ fn prefix_match_ignores_tool_call_identifiers() {let not_matching = vec![ChatMessage::user().with_text("hi".to_string()), different];assert!(!messages_have_prefix(¬_matching, ¤t));}++#[test]+fn prefix_match_treats_coerced_arguments_as_equal() {+ let assistant_call = |arguments: &str| {+ ChatMessage::assistant().with_tool_call(uzu::types::basic::ToolCall {+ identifier: Some("id".to_string()),+ name: "search".to_string(),+ arguments: uzu::types::basic::Value {+ json: arguments.to_string(),+ },+ })+ };++ // the session stores the parser's stringified scalar; the client echoes the+ // coerced typed value it received+ let current = vec![assistant_call(r#"{"query":"cats","limit":"5"}"#)];+ let extending =+ vec![assistant_call(r#"{"query":"cats","limit":5}"#), ChatMessage::user().with_text("more".to_string())];+ assert!(messages_have_prefix(&extending, ¤t));++ // a genuinely different value is still a mismatch+ let different =+ vec![assistant_call(r#"{"query":"cats","limit":6}"#), ChatMessage::user().with_text("more".to_string())];+ assert!(!messages_have_prefix(&different, ¤t));++ // string-declared values that the parser typed by their braces also match their echo+ let stored_object = vec![assistant_call(r#"{"content":{"a":1}}"#)];+ let echoed_string =+ vec![assistant_call(r#"{"content":"{\"a\":1}"}"#), ChatMessage::user().with_text("more".to_string())];+ assert!(messages_have_prefix(&echoed_string, &stored_object));++ // the session stores a Python-style boolean (qwen3.5 markup); the client+ // echoes the coerced JSON boolean it received+ let stored_python = vec![assistant_call(r#"{"metric":"True"}"#)];+ let echoed_boolean = vec![assistant_call(r#"{"metric":true}"#), ChatMessage::user().with_text("more".to_string())];+ assert!(messages_have_prefix(&echoed_boolean, &stored_python));+}++#[test]+fn sampling_values_out_of_range_are_rejected() {+ for body in [+ r#"{"messages":[],"temperature":0}"#,+ r#"{"messages":[],"temperature":2}"#,+ r#"{"messages":[],"top_p":1}"#,+ r#"{"messages":[],"top_p":0.01}"#,+ r#"{"messages":[],"top_k":1}"#,+ ] {+ validate_sampling(&request(body)).unwrap_or_else(|(_, message)| panic!("expected {body} to pass: {message}"));+ }++ for (body, param) in [+ (r#"{"messages":[],"temperature":-5}"#, "temperature"),+ (r#"{"messages":[],"temperature":100}"#, "temperature"),+ (r#"{"messages":[],"top_p":0}"#, "top_p"),+ (r#"{"messages":[],"top_p":7}"#, "top_p"),+ (r#"{"messages":[],"top_k":0}"#, "top_k"),+ (r#"{"messages":[],"top_k":-3}"#, "top_k"),+ ] {+ let error = validate_sampling(&request(body)).expect_err("out-of-range value should be rejected");+ assert_eq!(error.0, param, "for {body}");+ }+}++#[test]+fn backend_error_bodies_are_openai_errors_with_collapsed_prefixes() {+ let body = serde_json::to_value(oai_error_body(+ Status::BadRequest,+ "backend_error",+ "Backend error: Backend error: Role 'wizard' is not supported",+ ))+ .expect("serializable error body");+ assert_eq!(body["error"]["type"], "invalid_request_error");+ assert_eq!(body["error"]["code"], "backend_error");+ assert_eq!(body["error"]["message"], "Role 'wizard' is not supported");++ let body =+ serde_json::to_value(oai_error_body(Status::InternalServerError, "no_response", "No response generated"))+ .expect("serializable error body");+ assert_eq!(body["error"]["type"], "server_error");+ assert_eq!(body["error"]["message"], "No response generated");+}diff --git a/crates/legacy/cli/unit/server/chat_tool_calls_test.rs b/crates/legacy/cli/unit/server/chat_tool_calls_test.rsindex e11889952..d6e05885e 100644--- a/crates/legacy/cli/unit/server/chat_tool_calls_test.rs+++ b/crates/legacy/cli/unit/server/chat_tool_calls_test.rs@@ -153,16 +153,16 @@ fn reply_tool_calls_serialize_in_openai_format() {#[test]fn json_candidate_announces_name_once_it_is_complete() {let mut streamer = ToolCallStreamer::new();- assert!(streamer.update(0, r#"{"name": "wr"#).is_empty());- assert!(streamer.update(0, r#"{"arguments": {}}"#).is_empty());+ assert!(streamer.update(0, r#"{"name": "wr"#, &ToolParameterTypes::default()).is_empty());+ assert!(streamer.update(0, r#"{"arguments": {}}"#, &ToolParameterTypes::default()).is_empty());- let deltas = streamer.update(1, r#"{"name": "write", "arguments": {"path":"#);+ let deltas = streamer.update(1, r#"{"name": "write", "arguments": {"path":"#, &ToolParameterTypes::default());assert_eq!(deltas.len(), 1);assert_eq!(deltas[0].index, Some(1));assert_eq!(deltas[0].function.name, "write");assert_eq!(deltas[0].function.arguments, "");- // the id is assigned at announcement time and must never change mid-call+ // Metadata is emitted exactly once, in the announcement delta.assert!(!deltas[0].id.is_empty());let call = ToolCall {identifier: Some("server-side-id-must-not-leak".to_string()),@@ -171,9 +171,56 @@ fn json_candidate_announces_name_once_it_is_complete() {json: r#"{"path":"/tmp/a"}"#.to_string(),},};- assert_eq!(streamer.finish(1, &call).id, deltas[0].id);- assert!(streamer.update(1, r#"{"name": "write", "arguments": {"path": "/"#).is_empty());+ let finish = serde_json::to_value(streamer.finish(1, &call)).expect("serializable final delta");+ assert_eq!(+ finish,+ serde_json::json!({+ "index": 1,+ "function": {"arguments": r#"{"path":"/tmp/a"}"#}+ })+ );++ assert!(+ streamer.update(1, r#"{"name": "write", "arguments": {"path": "/"#, &ToolParameterTypes::default()).is_empty()+ );+}++#[test]+fn framed_argument_deltas_do_not_repeat_announced_metadata() {+ let mut streamer = ToolCallStreamer::new();+ let deltas = streamer.update(0, "<function=write>\n<parameter=path>\n/tmp/a", &ToolParameterTypes::default());+ assert_eq!(deltas.len(), 2);++ let announcement = serde_json::to_value(&deltas[0]).expect("serializable announcement");+ assert_eq!(announcement["type"], "function");+ assert_eq!(announcement["function"]["name"], "write");++ let arguments = serde_json::to_value(&deltas[1]).expect("serializable argument delta");+ assert_eq!(+ arguments,+ serde_json::json!({+ "index": 0,+ "function": {"arguments": r#"{"path":"/tmp/a"#}+ })+ );+}++#[test]+fn finish_emits_metadata_when_no_announcement_was_possible() {+ let mut streamer = ToolCallStreamer::new();+ let call = ToolCall {+ identifier: None,+ name: "write".to_string(),+ arguments: Value {+ json: r#"{"path":"/tmp/a"}"#.to_string(),+ },+ };++ let finish = serde_json::to_value(streamer.finish(0, &call)).expect("serializable final delta");+ assert_eq!(finish["type"], "function");+ assert_eq!(finish["function"]["name"], "write");+ assert!(!finish["id"].as_str().expect("string id").is_empty());}#[test]@@ -194,7 +241,7 @@ fn framed_tool_call_streams_arguments_incrementally() {let chars = final_markup.chars().collect::<Vec<_>>();for i in 0..=chars.len() {let partial: String = chars[..i].iter().collect();- for delta in streamer.update(0, &partial) {+ for delta in streamer.update(0, &partial, &ToolParameterTypes::default()) {if !delta.function.name.is_empty() {announcements += 1;assert_eq!(delta.function.name, "write");@@ -212,10 +259,14 @@ fn framed_tool_call_streams_arguments_incrementally() {#[test]fn framed_tool_call_withholds_typed_parameter_until_complete() {let mut streamer = ToolCallStreamer::new();- let deltas = streamer.update(0, "\n<function=read>\n<parameter=options>\n{\"a\"");+ let deltas = streamer.update(0, "\n<function=read>\n<parameter=options>\n{\"a\"", &ToolParameterTypes::default());assert!(deltas.iter().all(|delta| delta.function.arguments.is_empty()));- let deltas = streamer.update(0, "\n<function=read>\n<parameter=options>\n{\"a\": 1}\n</parameter>\n</function>\n");+ let deltas = streamer.update(+ 0,+ "\n<function=read>\n<parameter=options>\n{\"a\": 1}\n</parameter>\n</function>\n",+ &ToolParameterTypes::default(),+ );let fragment: String = deltas.iter().map(|delta| delta.function.arguments.as_str()).collect();assert_eq!(fragment, r#"{"options":{"a":1}"#);@@ -245,7 +296,7 @@ fn framed_tool_call_streams_multibyte_content_without_panicking() {let chars = final_markup.chars().collect::<Vec<_>>();for i in 0..=chars.len() {let partial: String = chars[..i].iter().collect();- for delta in streamer.update(0, &partial) {+ for delta in streamer.update(0, &partial, &ToolParameterTypes::default()) {fragments.push_str(&delta.function.arguments);}}@@ -277,7 +328,7 @@ fn framed_tool_call_with_array_parameter_assembles_exactly() {let chars = final_markup.chars().collect::<Vec<_>>();for i in 0..=chars.len() {let partial: String = chars[..i].iter().collect();- for delta in streamer.update(0, &partial) {+ for delta in streamer.update(0, &partial, &ToolParameterTypes::default()) {fragment_count += 1;fragments.push_str(&delta.function.arguments);}@@ -288,3 +339,173 @@ fn framed_tool_call_with_array_parameter_assembles_exactly() {let parsed: serde_json::Value = serde_json::from_str(&fragments).expect("assembled arguments parse");assert_eq!(parsed, final_arguments);}++fn parameter_types(tools_json: &str) -> ToolParameterTypes {+ let tools: Vec<OaiTool> = serde_json::from_str(tools_json).expect("valid tools json");+ ToolParameterTypes::from_tools(Some(&tools))+}++const SEARCH_TOOL: &str = r#"[{"type":"function","function":{"name":"search","description":"Search",+ "parameters":{"type":"object","properties":{+ "query":{"type":"string"},+ "limit":{"type":"integer"},+ "ratio":{"type":"number"},+ "metric":{"type":"boolean"},+ "note":{"type":["string","null"]},+ "padded_id":{"type":"string"}+ },"required":["query"]}}}]"#;++#[test]+fn declared_types_restore_scalar_arguments() {+ let types = parameter_types(SEARCH_TOOL);+ let call = ToolCall {+ identifier: None,+ name: "search".to_string(),+ arguments: Value {+ json: r#"{"query":"cats","limit":"5","ratio":"-0.75","metric":"true","note":"null","padded_id":"00123","unknown":"7"}"#.to_string(),+ },+ };++ let coerced: serde_json::Value =+ serde_json::from_str(&coerce_tool_call(&call, &types).arguments.json).expect("coerced arguments parse");+ assert_eq!(+ coerced,+ serde_json::json!({+ "query": "cats",+ "limit": 5,+ "ratio": -0.75,+ "metric": true,+ // a union that includes "string" keeps the string: it is already schema-valid+ "note": "null",+ // declared strings are never retyped, so padded ids survive+ "padded_id": "00123",+ // undeclared parameters are left alone+ "unknown": "7"+ })+ );+}++#[test]+fn python_style_booleans_coerce_to_declared_boolean() {+ let types = parameter_types(SEARCH_TOOL);+ // qwen3.5 writes Python-style booleans into its tool markup+ let call = ToolCall {+ identifier: None,+ name: "search".to_string(),+ arguments: Value {+ json: r#"{"metric":"True","query":"False","limit":"True"}"#.to_string(),+ },+ };++ let coerced: serde_json::Value =+ serde_json::from_str(&coerce_tool_call(&call, &types).arguments.json).expect("coerced arguments parse");+ assert_eq!(+ coerced,+ serde_json::json!({+ "metric": true,+ // declared strings are never retyped+ "query": "False",+ // a boolean does not read as the declared integer+ "limit": "True"+ })+ );+}++#[test]+fn framed_streamer_agrees_with_coerced_boolean_finish() {+ let types = parameter_types(SEARCH_TOOL);+ let final_markup =+ "<function=search>\n<parameter=query>\ncats\n</parameter>\n<parameter=metric>\nTrue\n</parameter>\n</function>";+ // what coerce_tool_call produces from the parser's Python-style boolean+ let call = ToolCall {+ identifier: Some("c1".to_string()),+ name: "search".to_string(),+ arguments: Value {+ json: r#"{"query":"cats","metric":true}"#.to_string(),+ },+ };++ let mut streamer = ToolCallStreamer::new();+ let mut fragments = String::new();+ let chars = final_markup.chars().collect::<Vec<_>>();+ for i in 0..=chars.len() {+ let partial: String = chars[..i].iter().collect();+ for delta in streamer.update(0, &partial, &types) {+ fragments.push_str(&delta.function.arguments);+ }+ }+ fragments.push_str(&streamer.finish(0, &call).function.arguments);++ let parsed: serde_json::Value = serde_json::from_str(&fragments).expect("assembled arguments parse");+ assert_eq!(parsed, serde_json::json!({"query": "cats", "metric": true}));+}++#[test]+fn declared_string_keeps_json_shaped_text() {+ let types = parameter_types(+ r#"[{"type":"function","function":{"name":"write_file","description":"Write",+ "parameters":{"type":"object","properties":{"path":{"type":"string"},"content":{"type":"string"}},+ "required":["path","content"]}}}]"#,+ );+ // the parser typed the JSON-shaped content by its braces+ let call = ToolCall {+ identifier: None,+ name: "write_file".to_string(),+ arguments: Value {+ json: r#"{"path":"data.json","content":{"a":1}}"#.to_string(),+ },+ };++ let coerced: serde_json::Value =+ serde_json::from_str(&coerce_tool_call(&call, &types).arguments.json).expect("coerced arguments parse");+ assert_eq!(coerced["content"], serde_json::json!(r#"{"a":1}"#));+}++#[test]+fn coercion_leaves_unparseable_scalars_and_wrapped_arguments_alone() {+ let types = parameter_types(SEARCH_TOOL);+ let call = |json: &str| ToolCall {+ identifier: None,+ name: "search".to_string(),+ arguments: Value {+ json: json.to_string(),+ },+ };++ // text that does not read as the declared type stays a string+ assert_eq!(coerce_tool_call(&call(r#"{"limit":"lots"}"#), &types).arguments.json, r#"{"limit":"lots"}"#);+ // a declared integer that reads as a float stays a string+ assert_eq!(coerce_tool_call(&call(r#"{"limit":"5.5"}"#), &types).arguments.json, r#"{"limit":"5.5"}"#);+ // wrapped unparseable arguments are not an object of parameters+ let wrapped = r#"{"__uzu_unparsed_arguments":"{\"a\":"}"#;+ assert_eq!(coerce_tool_call(&call(wrapped), &types).arguments.json, wrapped);+}++#[test]+fn framed_streamer_agrees_with_coerced_finish() {+ let types = parameter_types(SEARCH_TOOL);+ let final_markup =+ "<function=search>\n<parameter=query>\ncats\n</parameter>\n<parameter=limit>\n5\n</parameter>\n</function>";+ // what coerce_tool_call produces from the parser's stringified scalars+ let call = ToolCall {+ identifier: Some("c1".to_string()),+ name: "search".to_string(),+ arguments: Value {+ json: r#"{"query":"cats","limit":5}"#.to_string(),+ },+ };++ let mut streamer = ToolCallStreamer::new();+ let mut fragments = String::new();+ let chars = final_markup.chars().collect::<Vec<_>>();+ for i in 0..=chars.len() {+ let partial: String = chars[..i].iter().collect();+ for delta in streamer.update(0, &partial, &types) {+ fragments.push_str(&delta.function.arguments);+ }+ }+ fragments.push_str(&streamer.finish(0, &call).function.arguments);++ let parsed: serde_json::Value = serde_json::from_str(&fragments).expect("assembled arguments parse");+ assert_eq!(parsed, serde_json::json!({"query": "cats", "limit": 5}));+}diff --git a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rsindex 21a324eb8..f310778e4 100644--- a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs+++ b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs@@ -35,7 +35,9 @@ use uzu_engine::{};#[cfg(feature = "capability-grammar")]-use crate::engine::bridge::helpers::get_grammar;+use crate::engine::bridge::helpers::{+ get_grammar, grammar_trigger_token_sequence, grammar_trigger_token_sequence_for_prompt,+};use crate::engine::bridge::{chat_token_state::UzuChatTokenBackendInstanceState,helpers::{error_stream, get_max_context_length, get_sampling_method},@@ -47,6 +49,8 @@ pub struct UzuChatTokenBackendInstance<B: Backend> {config: ChatConfig,stop_token_ids: Vec<i32>,max_context_length: Option<u32>,+ #[cfg(feature = "capability-grammar")]+ grammar_trigger_token_sequence: Option<Vec<u64>>,}impl<B: Backend> UzuChatTokenBackendInstance<B> {@@ -60,6 +64,8 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {let stop_token_ids = model.generation_config().stop_token_ids.iter().map(|id| *id as i32).collect();let max_context_length = get_max_context_length(&model, config.context_length.clone());+ #[cfg(feature = "capability-grammar")]+ let grammar_trigger_token_sequence = grammar_trigger_token_sequence(&model);Ok(Self {engine,@@ -67,6 +73,8 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {config,stop_token_ids,max_context_length,+ #[cfg(feature = "capability-grammar")]+ grammar_trigger_token_sequence,})}}@@ -116,7 +124,12 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {#[cfg(feature = "capability-grammar")]let grammar = if let Some(grammar_config) = config.grammar {- match get_grammar(grammar_config, self.model.tokenizer(), &self.stop_token_ids) {+ let trigger_token_sequence = grammar_trigger_token_sequence_for_prompt(+ self.grammar_trigger_token_sequence.as_deref(),+ input,+ self.model.tokenizer(),+ );+ match get_grammar(grammar_config, self.model.tokenizer(), &self.stop_token_ids, trigger_token_sequence) {Ok(grammar) => Some(grammar),Err(err) => {return Box::pin(NoMetricsStream::new(error_stream(err.to_string())));diff --git a/crates/legacy/uzu/src/engine/bridge/helpers.rs b/crates/legacy/uzu/src/engine/bridge/helpers.rsindex 1fc4e61bb..1208b2b82 100644--- a/crates/legacy/uzu/src/engine/bridge/helpers.rs+++ b/crates/legacy/uzu/src/engine/bridge/helpers.rs@@ -29,6 +29,7 @@ pub fn get_grammar(grammar: ShojiGrammar,tokenizer: &Tokenizer,stop_token_ids: &[i32],+ trigger_token_sequence: Option<Vec<u64>>,) -> Result<UzuGrammar, GrammarError> {let config = match grammar {ShojiGrammar::JsonAny {@@ -41,7 +42,44 @@ pub fn get_grammar(pattern,} => GrammarConfig::regex(pattern, false),};- UzuGrammar::new(&config, tokenizer, None, Some(stop_token_ids))+ UzuGrammar::new(&config, tokenizer, trigger_token_sequence, Some(stop_token_ids))+}++/// The token sequence of the model's end-of-thinking tag. A reasoning model+/// opens every reply with reasoning tokens that no grammar can accept, so the+/// grammar must stay disengaged until this sequence has been generated.+#[cfg(feature = "capability-grammar")]+pub fn grammar_trigger_token_sequence<B: Backend>(model: &LanguageModel<B>) -> Option<Vec<u64>> {+ let tag = model.end_of_thinking_tag()?;+ let token_ids: Vec<u64> =+ model.tokenizer().encode(tag, false).ok()?.get_ids().iter().map(|token_id| u64::from(*token_id)).collect();+ (!token_ids.is_empty()).then_some(token_ids)+}++/// Returns the end-of-thinking trigger only when it still has to be generated.+/// Toggle-capable models render the trigger into the generation prompt when+/// reasoning is disabled, in which case the grammar must engage immediately.+#[cfg(feature = "capability-grammar")]+pub fn grammar_trigger_token_sequence_for_prompt(+ trigger_token_sequence: Option<&[u64]>,+ prompt: &[u64],+ tokenizer: &Tokenizer,+) -> Option<Vec<u64>> {+ let trigger_token_sequence = trigger_token_sequence.filter(|sequence| !sequence.is_empty())?;+ let trailing_tokens = prompt+ .windows(trigger_token_sequence.len())+ .rposition(|window| window == trigger_token_sequence)+ .map(|start| &prompt[start + trigger_token_sequence.len()..]);++ let trigger_is_in_generation_prompt = trailing_tokens.is_some_and(|trailing_tokens| {+ let Ok(trailing_tokens) = trailing_tokens.iter().copied().map(u32::try_from).collect::<Result<Vec<_>, _>>()+ else {+ return false;+ };+ tokenizer.decode(&trailing_tokens, false).is_ok_and(|suffix| suffix.trim().is_empty())+ });++ (!trigger_is_in_generation_prompt).then(|| trigger_token_sequence.to_vec())}pub fn get_max_context_length<B: Backend>(@@ -93,3 +131,43 @@ pub fn get_sampling_method<B: Backend>(},}}++#[cfg(all(test, feature = "capability-grammar"))]+mod tests {+ use tokenizers::{Tokenizer, decoders::fuse::Fuse, models::wordlevel::WordLevel};++ use super::grammar_trigger_token_sequence_for_prompt;++ fn tokenizer() -> Tokenizer {+ let model = WordLevel::builder()+ .vocab(+ [+ ("<unk>".to_string(), 0),+ ("</think>".to_string(), 1),+ ("\n\n".to_string(), 2),+ ("next".to_string(), 3),+ ]+ .into_iter()+ .collect(),+ )+ .build()+ .unwrap();+ let mut tokenizer = Tokenizer::new(model);+ tokenizer.with_decoder(Some(Fuse::new()));+ tokenizer+ }++ #[test]+ fn grammar_engages_immediately_when_prompt_closes_reasoning() {+ let tokenizer = tokenizer();++ assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2], &tokenizer), None);+ }++ #[test]+ fn grammar_waits_for_trigger_when_only_history_contains_it() {+ let tokenizer = tokenizer();++ assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2, 3], &tokenizer), Some(vec![1]));+ }+}diff --git a/crates/uzu-engine/src/engine/language_model/grammar/engagement.rs b/crates/uzu-engine/src/engine/language_model/grammar/engagement.rsindex e1c7f9ae1..d530b85ea 100644--- a/crates/uzu-engine/src/engine/language_model/grammar/engagement.rs+++ b/crates/uzu-engine/src/engine/language_model/grammar/engagement.rs@@ -1,19 +1,52 @@+/// Decides when the grammar starts constraining sampling. `Always` engages from+/// the first generated token. `Triggered` stays dormant until the generated+/// stream contains the trigger token sequence (a reasoning model's+/// end-of-thinking tag), so reasoning tokens flow unconstrained and only the+/// final answer is grammar-constrained.pub enum GrammarEngagementState {Always,Triggered {- trigger_token_id: u64,- trigger_distance: Option<usize>,+ trigger_sequence: Vec<u64>,+ /// Match-automaton state after each accepted token: the length of the+ /// longest prefix of `trigger_sequence` that is a suffix of the stream,+ /// held at `trigger_sequence.len()` once the trigger has completed. One+ /// entry per accepted token so `rollback` restores any earlier state+ /// exactly.+ match_history: Vec<usize>,},}+/// The last `matched` tokens of the stream equal `trigger_sequence[..matched]`,+/// so the new match length is the longest prefix of `trigger_sequence` that is+/// a suffix of `trigger_sequence[..matched]` followed by `token_id`.+fn advance(+ trigger_sequence: &[u64],+ matched: usize,+ token_id: u64,+) -> usize {+ if matched == trigger_sequence.len() {+ return matched;+ }+ let mut length = matched + 1;+ while length > 0 {+ if trigger_sequence[length - 1] == token_id+ && trigger_sequence[matched + 1 - length..matched] == trigger_sequence[..length - 1]+ {+ return length;+ }+ length -= 1;+ }+ 0+}+impl GrammarEngagementState {pub fn is_engaged(&self) -> bool {match self {Self::Always => true,Self::Triggered {- trigger_token_id: _,- trigger_distance,- } => trigger_distance.is_some(),+ trigger_sequence,+ match_history,+ } => match_history.last() == Some(&trigger_sequence.len()),}}@@ -24,14 +57,11 @@ impl GrammarEngagementState {match self {Self::Always => (),Self::Triggered {- trigger_token_id,- trigger_distance,+ trigger_sequence,+ match_history,} => {- if let Some(trigger_distance) = trigger_distance {- *trigger_distance += 1;- } else if token_id == *trigger_token_id {- *trigger_distance = Some(0);- }+ let matched = match_history.last().copied().unwrap_or(0);+ match_history.push(advance(trigger_sequence, matched, token_id));},}}@@ -43,13 +73,28 @@ impl GrammarEngagementState {match self {Self::Always => num_tokens,Self::Triggered {- trigger_token_id: _,- trigger_distance,+ trigger_sequence,+ match_history,} => {- let num_grammar_tokens = usize::min(trigger_distance.unwrap_or(0), num_tokens);- *trigger_distance = trigger_distance.and_then(|x| x.checked_sub(num_tokens));+ let full_match = trigger_sequence.len();+ let mut num_grammar_tokens = 0;+ for _ in 0..num_tokens {+ if match_history.pop().is_none() {+ break;+ }+ // A token was fed to the matcher iff the trigger was already+ // complete when it was accepted, i.e. the state before it+ // was a full match.+ if match_history.last().copied().unwrap_or(0) == full_match {+ num_grammar_tokens += 1;+ }+ }num_grammar_tokens},}}}++#[cfg(test)]+#[path = "../../../../unit/engine/language_model/grammar/engagement_test.rs"]+mod tests;diff --git a/crates/uzu-engine/src/engine/language_model/grammar/mod.rs b/crates/uzu-engine/src/engine/language_model/grammar/mod.rsindex 0a70da241..59bcd5628 100644--- a/crates/uzu-engine/src/engine/language_model/grammar/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/grammar/mod.rs@@ -33,7 +33,7 @@ impl Grammar {pub fn new(config: &GrammarConfig,tokenizer: &Tokenizer,- trigger_token_id: Option<u64>,+ trigger_token_sequence: Option<Vec<u64>>,stop_token_ids: Option<&[i32]>,) -> Result<Self, GrammarError> {let tokenizer_info =@@ -72,13 +72,12 @@ impl Grammar {let compiled = compiler.compile_grammar(&grammar).map_err(GrammarError::XGrammar)?;let matcher = GrammarMatcher::new(&compiled, None, true, -1).map_err(GrammarError::XGrammar)?;- let engagement_state = if let Some(trigger_token_id) = trigger_token_id {- GrammarEngagementState::Triggered {- trigger_token_id,- trigger_distance: None,- }- } else {- GrammarEngagementState::Always+ let engagement_state = match trigger_token_sequence.filter(|sequence| !sequence.is_empty()) {+ Some(trigger_sequence) => GrammarEngagementState::Triggered {+ trigger_sequence,+ match_history: Vec::new(),+ },+ None => GrammarEngagementState::Always,};Ok(Self {@@ -131,7 +130,13 @@ impl Grammar {&mut self,token_id: u64,) -> Result<(), GrammarError> {- if self.engagement_state.is_engaged() && !self.matcher.accept_token(token_id as i32) {+ // A terminated matcher cannot advance and its bitmask only allows stop+ // tokens, which close generation without being part of the grammar, so+ // tokens sampled after termination must not be rejected.+ if self.engagement_state.is_engaged()+ && !self.matcher.is_terminated()+ && !self.matcher.accept_token(token_id as i32)+ {return Err(GrammarError::GrammarReject);}diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rsindex 8e4c208bd..ba3486a08 100644--- a/crates/uzu-engine/src/engine/language_model/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/mod.rs@@ -5,7 +5,10 @@ use tokenizers::Tokenizer;use crate::{backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},- config::model::{generation::GenerationConfig, language_model::LanguageModelConfig},+ config::{+ model::{generation::GenerationConfig, language_model::LanguageModelConfig},+ token_codec::AnyTokenCodecConfig,+ },data_type::DataType,encodable_block::{decoder::{Decoder, DecoderError},@@ -29,6 +32,9 @@ pub struct LanguageModel<B: Backend> {sampling: Sampling<B>,context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,generation_config: GenerationConfig,+ /// The literal text the model emits between its reasoning and its final+ /// answer (e.g. "</think>"); None when the model does not separate them.+ end_of_thinking_tag: Option<String>,tokenizer: Arc<Tokenizer>,#[cfg(grammar)]vocab_size: usize,@@ -99,6 +105,10 @@ impl<B: Backend> Engine<B> {weight_loader.tree().assert_all_tensors_validated()?;let generation_config = config.generation_config;+ let end_of_thinking_tag = match &config.token_codec_config {+ AnyTokenCodecConfig::ChatCodecConfig(token_codec_config) => token_codec_config.end_of_thinking_tag.clone(),+ AnyTokenCodecConfig::RawTextCodecConfig(_) => None,+ };#[cfg(grammar)]let vocab_size = config.decoder_config.vocab_size as usize;@@ -110,6 +120,7 @@ impl<B: Backend> Engine<B> {sampling,context_ring_update,generation_config,+ end_of_thinking_tag,tokenizer,#[cfg(grammar)]vocab_size,@@ -168,6 +179,10 @@ impl<B: Backend> LanguageModel<B> {&self.generation_config}+ pub fn end_of_thinking_tag(&self) -> Option<&str> {+ self.end_of_thinking_tag.as_deref()+ }+pub fn tokenizer(&self) -> &Arc<Tokenizer> {&self.tokenizer}diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex f8316410b..f18c31edf 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -347,9 +347,11 @@ impl<B: Backend> DFlashTfmSpeculator<B> {node.logprob,);for &child_index in &node.child_indices {+ // Grammar-illegal proposals are dropped; nothing may be proposed+ // past termination either, since only stop tokens can follow.#[cfg(grammar)]if let Some(grammar) = grammar.as_mut()- && grammar.accept_token(nodes[child_index].token_id as u64).is_err()+ && (grammar.is_terminated() || grammar.accept_token(nodes[child_index].token_id as u64).is_err()){continue;}diff --git a/crates/uzu-engine/src/trie.rs b/crates/uzu-engine/src/trie.rsindex a48b06e9a..e00c3fa5f 100644--- a/crates/uzu-engine/src/trie.rs+++ b/crates/uzu-engine/src/trie.rs@@ -291,11 +291,9 @@ impl<'a> FlatTrie<'a> {break;};- #[cfg(grammar)]- if let Some(grammar) = grammar.as_mut() {- assert!(!grammar.is_terminated(), "Grammar has terminated but llm continued generation");- }-+ // The grammar may already be terminated here: when the draft model+ // proposed the grammar-closing token the trie descends one node+ // further, where the mask only allows stop tokens.current_token = next_token;}diff --git a/crates/uzu-engine/unit/engine/language_model/grammar/engagement_test.rs b/crates/uzu-engine/unit/engine/language_model/grammar/engagement_test.rsnew file mode 100644index 000000000..4ebeb23ac--- /dev/null+++ b/crates/uzu-engine/unit/engine/language_model/grammar/engagement_test.rs@@ -0,0 +1,77 @@+use uzu_engine_macros::uzu_test;++use super::GrammarEngagementState;++// The muse-glimmer end-of-thinking tag:+// <|eom|> <|start|> assistant " to" "=user" <|message|>+const TRIGGER: [u64; 6] = [200007, 200022, 140680, 328, 76976, 200023];++fn triggered() -> GrammarEngagementState {+ GrammarEngagementState::Triggered {+ trigger_sequence: TRIGGER.to_vec(),+ match_history: Vec::new(),+ }+}++#[uzu_test]+fn always_is_engaged_and_rolls_back_everything() {+ let mut state = GrammarEngagementState::Always;+ assert!(state.is_engaged());+ state.accept_token(1);+ assert!(state.is_engaged());+ assert_eq!(state.rollback(3), 3);+}++#[uzu_test]+fn engages_only_after_the_full_trigger_sequence() {+ let mut state = triggered();+ // reasoning tokens, including a stray prefix of the trigger+ for token in [11, 200007, 200022, 42, 13] {+ state.accept_token(token);+ assert!(!state.is_engaged());+ }+ for token in TRIGGER {+ assert!(!state.is_engaged());+ state.accept_token(token);+ }+ assert!(state.is_engaged());+ // stays engaged for the rest of the stream+ state.accept_token(99);+ assert!(state.is_engaged());+}++#[uzu_test]+fn restarts_matching_on_a_mismatch_mid_sequence() {+ let mut state = triggered();+ for token in [TRIGGER[0], TRIGGER[1], TRIGGER[0], TRIGGER[1], TRIGGER[2], TRIGGER[3], TRIGGER[4], TRIGGER[5]] {+ state.accept_token(token);+ }+ // ..., eom, start, eom, start, assistant, to, =user, message: the second+ // eom restarts the match, which then runs to completion+ assert!(state.is_engaged());+}++#[uzu_test]+fn rollback_counts_only_tokens_accepted_while_engaged() {+ let mut state = triggered();+ state.accept_token(11);+ for token in TRIGGER {+ state.accept_token(token);+ }+ state.accept_token(90); // grammar token+ state.accept_token(91); // grammar token+ // 91, 90 were matcher-fed; <|message|> completed the trigger and was not+ assert_eq!(state.rollback(3), 2);+ assert!(!state.is_engaged());+ // the partial trigger state is restored: finishing it re-engages+ state.accept_token(TRIGGER[5]);+ assert!(state.is_engaged());+}++#[uzu_test]+fn rollback_past_the_start_is_safe() {+ let mut state = triggered();+ state.accept_token(11);+ assert_eq!(state.rollback(5), 0);+ assert!(!state.is_engaged());+}