Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Glimmer and oai server fixes (#771)

Aleksandr Golokoz9e4ddada3c8c
diff --git a/crates/legacy/cli/src/server/chat_completions.rs b/crates/legacy/cli/src/server/chat_completions.rs
index 614b42055..ecc64899e 100644
--- a/crates/legacy/cli/src/server/chat_completions.rs
+++ b/crates/legacy/cli/src/server/chat_completions.rs
@@ -10,11 +10,11 @@ use rocket::{
Request, State,
data::{ByteUnit, Data},
futures::Stream,
- http::Status,
+ http::{ContentType, Status},
post,
response::{
Responder, status,
- stream::{Event, EventStream},
+ stream::{ByteStream, Event, EventStream},
},
serde::json::Json,
};
@@ -23,7 +23,7 @@ use tokio::sync::{Mutex, mpsc};
use tokio_stream::wrappers::UnboundedReceiverStream;
use uuid::Uuid;
use uzu::{
- session::chat::{ChatSession, ChatSessionStreamChunk, UNPARSED_ARGUMENTS_KEY},
+ session::chat::{ChatSession, ChatSessionStream, ChatSessionStreamChunk, UNPARSED_ARGUMENTS_KEY},
types::{
basic::{Grammar, ReasoningEffort, SamplingMethod},
session::chat::{
@@ -37,8 +37,9 @@ use crate::{
server::{
ServerState,
chat_tool_calls::{
- OaiTool, OaiToolCall, ToolCallStreamer, backfill_tool_result_names, choose_tools, insert_tools_message,
- oai_tool_call, reply_tool_calls, to_tool_call, tool_call_result_block, withhold_stream_text,
+ OaiTool, OaiToolCall, ToolCallStreamer, ToolParameterTypes, backfill_tool_result_names, choose_tools,
+ coerce_tool_call, insert_tools_message, oai_tool_call, parse_scalar_text, reply_tool_calls, to_tool_call,
+ tool_call_result_block, withhold_stream_text,
},
request_log::RequestLog,
},
@@ -216,7 +217,10 @@ struct OaiError {
}
pub enum ChatCompletionResult {
- Json(Json<ChatCompletionResponse>),
+ // The single JSON body arrives as a stream: Rocket only surfaces a client
+ // disconnect by dropping the body stream, which is the blocking path's one
+ // signal to cancel a generation nobody is waiting for.
+ Json(Status, ByteStream<Pin<Box<dyn Stream<Item = Vec<u8>> + Send>>>),
Stream(EventStream<Pin<Box<dyn Stream<Item = Event> + Send>>>),
Error(status::Custom<Json<OaiErrorResponse>>),
}
@@ -227,7 +231,11 @@ impl<'r> Responder<'r, 'r> for ChatCompletionResult {
request: &'r Request<'_>,
) -> rocket::response::Result<'r> {
match self {
- ChatCompletionResult::Json(json) => json.respond_to(request),
+ ChatCompletionResult::Json(status, body) => {
+ let mut response = (ContentType::JSON, body).respond_to(request)?;
+ response.set_status(status);
+ Ok(response)
+ },
ChatCompletionResult::Stream(stream) => stream.respond_to(request),
ChatCompletionResult::Error(error) => error.respond_to(request),
}
@@ -489,6 +497,28 @@ fn json_schema_grammar(json_schema: &JsonSchemaFormat) -> Result<Grammar, Respon
})
}
+/// Wrong-typed values already fail JSON extraction; out-of-range numbers would
+/// otherwise reach the sampler unchecked (a huge temperature generates garbage
+/// with HTTP 200).
+fn validate_sampling(request: &ChatCompletionRequest) -> Result<(), (&'static str, String)> {
+ if let Some(temperature) = request.temperature
+ && !(0.0..=2.0).contains(&temperature)
+ {
+ return Err(("temperature", format!("temperature must be between 0 and 2, got {temperature}")));
+ }
+ if let Some(top_p) = request.top_p
+ && !(top_p > 0.0 && top_p <= 1.0)
+ {
+ return Err(("top_p", format!("top_p must be greater than 0 and at most 1, got {top_p}")));
+ }
+ if let Some(top_k) = request.top_k
+ && top_k < 1
+ {
+ return Err(("top_k", format!("top_k must be at least 1, got {top_k}")));
+ }
+ Ok(())
+}
+
fn build_reply_config(request: &ChatCompletionRequest) -> Result<ChatReplyConfig, ResponseFormatError> {
let token_limit = request.max_completion_tokens.or(request.max_tokens);
let mut config = ChatReplyConfig::default().with_token_limit(token_limit);
@@ -557,32 +587,43 @@ fn usage_from_stats(
}
}
-fn error_response(
- id: String,
- model: String,
- created: i64,
+/// nagare re-wraps backend errors at each layer, so a message can arrive as
+/// "Backend error: Backend error: ..."; the wire message keeps only the inner text.
+fn strip_backend_prefixes(message: &str) -> &str {
+ let mut message = message;
+ while let Some(inner) = message.strip_prefix("Backend error: ") {
+ message = inner;
+ }
+ message
+}
+
+fn oai_error_body(
+ status: Status,
+ code: &str,
message: &str,
-) -> ChatCompletionResponse {
- ChatCompletionResponse {
- id,
- object: "chat.completion".to_string(),
- created,
- model,
- choices: vec![ChatCompletionChoice {
- index: 0,
- message: OaiMessage {
- role: "assistant".to_string(),
- content: Some(format!("Error: {message}")),
- reasoning_content: None,
- tool_calls: None,
- tool_call_id: None,
- },
- finish_reason: "stop".to_string(),
- }],
- usage: ChatCompletionUsage::default(),
+) -> OaiErrorResponse {
+ let kind = if status.code < 500 {
+ "invalid_request_error"
+ } else {
+ "server_error"
+ };
+ OaiErrorResponse {
+ error: OaiError {
+ message: strip_backend_prefixes(message).to_string(),
+ kind: kind.to_string(),
+ param: None,
+ code: Some(code.to_string()),
+ },
}
}
+fn send_error_body(
+ sender: &mpsc::UnboundedSender<Vec<u8>>,
+ body: &OaiErrorResponse,
+) {
+ let _ = sender.send(serde_json::to_vec(body).unwrap_or_default());
+}
+
fn chunk_json(
id: &str,
model: &str,
@@ -606,6 +647,41 @@ fn chunk_json(
serde_json::to_string(&chunk).unwrap_or_default()
}
+/// Schema coercion retypes scalar arguments on the wire, so a client may echo
+/// `5` where the session stored `"5"` (or a JSON string where the session
+/// stored the value it spells); such pairs must still count as the same prefix.
+fn tool_call_arguments_equivalent(
+ stored: &str,
+ echoed: &str,
+) -> bool {
+ if stored == echoed {
+ return true;
+ }
+ match (serde_json::from_str::<serde_json::Value>(stored), serde_json::from_str::<serde_json::Value>(echoed)) {
+ (Ok(stored), Ok(echoed)) => json_values_equivalent(&stored, &echoed),
+ _ => false,
+ }
+}
+
+fn json_values_equivalent(
+ left: &serde_json::Value,
+ right: &serde_json::Value,
+) -> bool {
+ use serde_json::Value;
+ match (left, right) {
+ (Value::Object(left), Value::Object(right)) => {
+ left.len() == right.len()
+ && left
+ .iter()
+ .all(|(key, value)| right.get(key).is_some_and(|other| json_values_equivalent(value, other)))
+ },
+ (Value::String(text), other) | (other, Value::String(text)) if !other.is_string() => {
+ parse_scalar_text(text).is_some_and(|parsed| parsed == *other)
+ },
+ (left, right) => left == right,
+ }
+}
+
/// Tool call identifiers are assigned per turn and never rendered into tokens, so they are
/// excluded from the comparison; everything else about the messages must match.
fn messages_have_prefix(
@@ -627,10 +703,8 @@ fn messages_have_prefix(
value: call,
},
) => {
- let (mut prefix_call, mut call) = (prefix_call.clone(), call.clone());
- prefix_call.identifier = None;
- call.identifier = None;
- prefix_call == call
+ prefix_call.name == call.name
+ && tool_call_arguments_equivalent(&prefix_call.arguments.json, &call.arguments.json)
},
_ => prefix_block == block,
}
@@ -657,6 +731,21 @@ async fn prepare_input(
Ok(messages)
}
+fn send_response(
+ sender: &mpsc::UnboundedSender<Vec<u8>>,
+ response: &ChatCompletionResponse,
+) {
+ let _ = sender.send(serde_json::to_vec(response).unwrap_or_default());
+}
+
+/// Cancels the turn and waits for it to wind down before the caller releases
+/// the session lock: a request arriving right after a cancellation must not
+/// catch the session mid-cleanup, where it rejects operations.
+async fn cancel_and_drain(stream: &ChatSessionStream) {
+ stream.cancel_token().cancel();
+ while stream.next().await.is_some() {}
+}
+
async fn run_blocking(
session: Arc<Mutex<ChatSession>>,
messages: Vec<ChatMessage>,
@@ -665,80 +754,126 @@ async fn run_blocking(
model: String,
created: i64,
prefix_cache: bool,
+ parameter_types: ToolParameterTypes,
+ sender: mpsc::UnboundedSender<Vec<u8>>,
log: RequestLog,
-) -> ChatCompletionResponse {
- let session = session.lock().await;
+) {
+ // A blocking response writes nothing until generation finishes, and Rocket
+ // only discovers a vanished client through a failed socket write, which is
+ // what makes it drop the body receiver that `sender.closed()` observes. A
+ // periodic newline — insignificant leading whitespace for the JSON body —
+ // keeps forcing that write while the request queues and generates.
+ let mut keepalive = tokio::time::interval(std::time::Duration::from_secs(3));
+
+ // Requests serialize on the single session, so a client can disconnect
+ // while still queued; generating for it would block everyone behind it.
+ let mut lock = std::pin::pin!(session.lock());
+ let session = loop {
+ tokio::select! {
+ session = &mut lock => break session,
+ () = sender.closed() => {
+ log.fail("client disconnected");
+ return;
+ },
+ _ = keepalive.tick() => {
+ let _ = sender.send(b"\n".to_vec());
+ },
+ }
+ };
let input = match prepare_input(&session, messages, prefix_cache).await {
Ok(input) => input,
Err(error) => {
log.fail(&error.to_string());
- return error_response(id, model, created, &error.to_string());
+ send_error_body(&sender, &oai_error_body(Status::InternalServerError, "backend_error", &error.to_string()));
+ return;
},
};
- match session.reply(input, config).await {
- Ok(replies) => match replies.last() {
- Some(reply) => {
- let tool_calls = reply_tool_calls(&reply.message);
- // OpenAI sets content to null (not "") on tool call replies.
- let content = reply.message.text().or_else(|| tool_calls.is_none().then(String::new));
- let mut finish_reason =
- reply.finish_reason.as_ref().map(map_finish_reason).unwrap_or_else(|| "stop".to_string());
- // Calls the parser could not finalize must not execute as a partial batch.
- // OpenAI has no finish reason for this; providers signal terminal errors with
- // unrecognized reasons (pi maps them to a thrown provider error), so the turn
- // fails loudly instead of stalling on an absent or incomplete tool_calls array.
- let has_unfinished_candidates = reply
- .message
- .content
- .iter()
- .any(|block| matches!(block, ChatContentBlock::ToolCallCandidate { .. }));
- if finish_reason == "tool_calls" && (tool_calls.is_none() || has_unfinished_candidates) {
- finish_reason = "malformed_tool_call".to_string();
- }
- let wrapped_arguments = reply
- .message
- .tool_calls()
- .iter()
- .filter(|call| call.arguments.json.contains(UNPARSED_ARGUMENTS_KEY))
- .count();
- let mut notes = Vec::new();
- if has_unfinished_candidates {
- notes.push("model generated unfinished tool call(s)".to_string());
- }
- if wrapped_arguments > 0 {
- notes.push(format!("{wrapped_arguments} tool call(s) with unparseable arguments"));
- }
- log.finish(&finish_reason, Some(&reply.stats), notes);
- ChatCompletionResponse {
- id,
- object: "chat.completion".to_string(),
- created,
- model,
- choices: vec![ChatCompletionChoice {
- index: 0,
- message: OaiMessage {
- role: "assistant".to_string(),
- content,
- reasoning_content: reply.message.reasoning().filter(|reasoning| !reasoning.is_empty()),
- tool_calls,
- tool_call_id: None,
- },
- finish_reason,
- }],
- usage: usage_from_stats(&reply.stats, prefix_cache),
- }
+ // Drive the reply through its stream instead of ChatSession::reply, which
+ // cannot be cancelled: racing each chunk against the channel closing
+ // notices a disconnect mid-generation. Cancelling stops the backend at the
+ // next token boundary and lets the session run its cancelled-turn cleanup.
+ let stream = session.reply_with_stream(input, config).await;
+ let mut final_replies = Vec::new();
+ loop {
+ let chunk = tokio::select! {
+ chunk = stream.next() => chunk,
+ () = sender.closed() => {
+ cancel_and_drain(&stream).await;
+ log.fail("client disconnected");
+ return;
},
- None => {
- log.fail("no response generated");
- error_response(id, model, created, "No response generated")
+ _ = keepalive.tick() => {
+ let _ = sender.send(b"\n".to_vec());
+ continue;
},
- },
- Err(error) => {
- log.fail(&error.to_string());
- error_response(id, model, created, &error.to_string())
- },
+ };
+ let Some(chunk) = chunk else {
+ break;
+ };
+ match chunk {
+ ChatSessionStreamChunk::Replies {
+ replies,
+ } => {
+ final_replies = replies;
+ },
+ ChatSessionStreamChunk::Error {
+ error,
+ } => {
+ log.fail(&error.to_string());
+ send_error_body(&sender, &oai_error_body(Status::BadRequest, "backend_error", &error.to_string()));
+ return;
+ },
+ }
}
+
+ let Some(reply) = final_replies.last() else {
+ log.fail("no response generated");
+ send_error_body(&sender, &oai_error_body(Status::InternalServerError, "no_response", "No response generated"));
+ return;
+ };
+ let tool_calls = reply_tool_calls(&reply.message, &parameter_types);
+ // OpenAI sets content to null (not "") on tool call replies.
+ let content = reply.message.text().or_else(|| tool_calls.is_none().then(String::new));
+ let mut finish_reason = reply.finish_reason.as_ref().map(map_finish_reason).unwrap_or_else(|| "stop".to_string());
+ // Calls the parser could not finalize must not execute as a partial batch.
+ // OpenAI has no finish reason for this; providers signal terminal errors with
+ // unrecognized reasons (pi maps them to a thrown provider error), so the turn
+ // fails loudly instead of stalling on an absent or incomplete tool_calls array.
+ let has_unfinished_candidates =
+ reply.message.content.iter().any(|block| matches!(block, ChatContentBlock::ToolCallCandidate { .. }));
+ if finish_reason == "tool_calls" && (tool_calls.is_none() || has_unfinished_candidates) {
+ finish_reason = "malformed_tool_call".to_string();
+ }
+ let wrapped_arguments =
+ reply.message.tool_calls().iter().filter(|call| call.arguments.json.contains(UNPARSED_ARGUMENTS_KEY)).count();
+ let mut notes = Vec::new();
+ if has_unfinished_candidates {
+ notes.push("model generated unfinished tool call(s)".to_string());
+ }
+ if wrapped_arguments > 0 {
+ notes.push(format!("{wrapped_arguments} tool call(s) with unparseable arguments"));
+ }
+ log.finish(&finish_reason, Some(&reply.stats), notes);
+ let response = ChatCompletionResponse {
+ id,
+ object: "chat.completion".to_string(),
+ created,
+ model,
+ choices: vec![ChatCompletionChoice {
+ index: 0,
+ message: OaiMessage {
+ role: "assistant".to_string(),
+ content,
+ reasoning_content: reply.message.reasoning().filter(|reasoning| !reasoning.is_empty()),
+ tool_calls,
+ tool_call_id: None,
+ },
+ finish_reason,
+ }],
+ usage: usage_from_stats(&reply.stats, prefix_cache),
+ };
+ send_response(&sender, &response);
}
async fn run_stream(
@@ -749,26 +884,28 @@ async fn run_stream(
model: String,
created: i64,
prefix_cache: bool,
+ parameter_types: ToolParameterTypes,
sender: mpsc::UnboundedSender<Event>,
log: RequestLog,
) {
- let session = session.lock().await;
+ // Same as the blocking path: don't start generating for a client that
+ // disconnected while queued on the session.
+ let session = tokio::select! {
+ session = session.lock() => session,
+ () = sender.closed() => {
+ log.fail("client disconnected");
+ return;
+ },
+ };
let has_tools = messages.iter().any(|message| !message.tool_namespaces().is_empty());
let input = match prepare_input(&session, messages, prefix_cache).await {
Ok(input) => input,
Err(error) => {
log.fail(&error.to_string());
- let _ = sender.send(Event::data(chunk_json(
- &id,
- &model,
- created,
- StreamDelta {
- content: Some(format!("Error: {error}")),
- ..StreamDelta::default()
- },
- Some("stop".to_string()),
- None,
- )));
+ // The SSE response is already committed as 200, so the error travels
+ // as an OpenAI-style error event, the way OpenAI reports mid-stream.
+ let body = oai_error_body(Status::InternalServerError, "backend_error", &error.to_string());
+ let _ = sender.send(Event::data(serde_json::to_string(&body).unwrap_or_default()));
let _ = sender.send(Event::data("[DONE]"));
return;
},
@@ -787,7 +924,6 @@ async fn run_stream(
)));
let stream = session.reply_with_stream(input, config).await;
- let cancel_token = stream.cancel_token();
let mut emitted = 0usize;
let mut emitted_reasoning = 0usize;
let mut emitted_tool_calls = 0usize;
@@ -807,8 +943,7 @@ async fn run_stream(
let chunk = tokio::select! {
chunk = stream.next() => chunk,
() = sender.closed() => {
- cancel_token.cancel();
- cancel_token.cancel();
+ cancel_and_drain(&stream).await;
log.fail("client disconnected");
return;
},
@@ -842,7 +977,7 @@ async fn run_stream(
None,
)));
if sent.is_err() {
- cancel_token.cancel();
+ cancel_and_drain(&stream).await;
log.fail("client disconnected");
return;
}
@@ -864,7 +999,7 @@ async fn run_stream(
None,
)));
if sent.is_err() {
- cancel_token.cancel();
+ cancel_and_drain(&stream).await;
log.fail("client disconnected");
return;
}
@@ -882,8 +1017,11 @@ async fn run_stream(
// candidates carry their partial text as a JSON string document, except
// object-valued ones (e.g. muse-glimmer's name-only progress objects)
let raw = serde_json::from_str::<String>(&value.json).unwrap_or_else(|_| value.json.clone());
- let deltas =
- tool_call_streamers.entry(index).or_insert_with(ToolCallStreamer::new).update(index, &raw);
+ let deltas = tool_call_streamers.entry(index).or_insert_with(ToolCallStreamer::new).update(
+ index,
+ &raw,
+ &parameter_types,
+ );
for delta in deltas {
let sent = sender.send(Event::data(chunk_json(
&id,
@@ -897,7 +1035,7 @@ async fn run_stream(
None,
)));
if sent.is_err() {
- cancel_token.cancel();
+ cancel_and_drain(&stream).await;
log.fail("client disconnected");
return;
}
@@ -908,13 +1046,13 @@ async fn run_stream(
while emitted_tool_calls < tool_calls.len() {
let index = emitted_tool_calls;
emitted_tool_calls += 1;
- let call = &tool_calls[index];
- if call.arguments.json.contains(UNPARSED_ARGUMENTS_KEY) {
+ if tool_calls[index].arguments.json.contains(UNPARSED_ARGUMENTS_KEY) {
wrapped_arguments += 1;
}
+ let call = coerce_tool_call(&tool_calls[index], &parameter_types);
let delta = match tool_call_streamers.remove(&index) {
- Some(mut streamer) => streamer.finish(index, call),
- None => oai_tool_call(Some(index), call),
+ Some(mut streamer) => streamer.finish(index, &call),
+ None => oai_tool_call(Some(index), &call),
};
let sent = sender.send(Event::data(chunk_json(
&id,
@@ -928,7 +1066,7 @@ async fn run_stream(
None,
)));
if sent.is_err() {
- cancel_token.cancel();
+ cancel_and_drain(&stream).await;
log.fail("client disconnected");
return;
}
@@ -950,17 +1088,9 @@ async fn run_stream(
} => {
errored = true;
log.fail(&error.to_string());
- let _ = sender.send(Event::data(chunk_json(
- &id,
- &model,
- created,
- StreamDelta {
- content: Some(format!("Error: {error}")),
- ..StreamDelta::default()
- },
- Some("stop".to_string()),
- None,
- )));
+ // Committed 200 stream: report as an OpenAI-style error event.
+ let body = oai_error_body(Status::BadRequest, "backend_error", &error.to_string());
+ let _ = sender.send(Event::data(serde_json::to_string(&body).unwrap_or_default()));
break;
},
}
@@ -984,7 +1114,7 @@ async fn run_stream(
None,
)));
if sent.is_err() {
- cancel_token.cancel();
+ cancel_and_drain(&stream).await;
log.fail("client disconnected");
return;
}
@@ -1056,6 +1186,10 @@ pub async fn handle_chat_completions(
request.reasoning_effort.as_ref().and_then(serde_json::Value::as_str).or(Some("unspecified")),
);
+ if let Err((param, message)) = validate_sampling(&request) {
+ log.fail(&message);
+ return invalid_request_response(param, "out_of_range", message);
+ }
let config = match build_reply_config(&request) {
Ok(config) => config,
Err(error) => {
@@ -1074,6 +1208,10 @@ pub async fn handle_chat_completions(
},
};
+ // The parser cannot type scalar arguments; the declared schemas restore
+ // the types when replies cross back into the OpenAI wire format.
+ let parameter_types = ToolParameterTypes::from_tools(request.tools.as_deref());
+
if is_stream {
let session = Arc::clone(&state.session);
let (sender, receiver) = mpsc::unbounded_channel::<Event>();
@@ -1085,6 +1223,7 @@ pub async fn handle_chat_completions(
model,
created,
state.prefix_cache,
+ parameter_types,
sender,
log,
));
@@ -1092,8 +1231,24 @@ pub async fn handle_chat_completions(
ChatCompletionResult::Stream(EventStream::from(body))
} else {
let session = Arc::clone(&state.session);
- let response = run_blocking(session, messages, config, id, model, created, state.prefix_cache, log).await;
- ChatCompletionResult::Json(Json(response))
+ let (sender, receiver) = mpsc::unbounded_channel::<Vec<u8>>();
+ rocket::tokio::spawn(run_blocking(
+ session,
+ messages,
+ config,
+ id,
+ model,
+ created,
+ state.prefix_cache,
+ parameter_types,
+ sender,
+ log,
+ ));
+ let body: Pin<Box<dyn Stream<Item = Vec<u8>> + Send>> = Box::pin(UnboundedReceiverStream::new(receiver));
+ // Hand the body to Rocket immediately so it can write keepalives and
+ // drop the receiver when the client disconnects. Any backend error
+ // occurs after the 200 response is committed and is sent in the body.
+ ChatCompletionResult::Json(Status::Ok, ByteStream::from(body))
}
}
diff --git a/crates/legacy/cli/src/server/chat_tool_calls.rs b/crates/legacy/cli/src/server/chat_tool_calls.rs
index 5ac1e0851..cb8fd6fa4 100644
--- a/crates/legacy/cli/src/server/chat_tool_calls.rs
+++ b/crates/legacy/cli/src/server/chat_tool_calls.rs
@@ -15,16 +15,17 @@ pub struct OaiToolCall {
// Present only in streaming deltas, per the OpenAI wire format.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub index: Option<usize>,
- // Empty in fragments: only the announcement and the final delta carry the id.
+ // Empty after announcement; finish carries metadata only if no announcement occurred.
#[serde(default, skip_serializing_if = "String::is_empty")]
pub id: String,
- #[serde(rename = "type", default)]
+ #[serde(rename = "type", default, skip_serializing_if = "String::is_empty")]
pub kind: String,
pub function: OaiFunctionCall,
}
#[derive(Serialize, Deserialize, Clone)]
pub struct OaiFunctionCall {
+ #[serde(default, skip_serializing_if = "String::is_empty")]
pub name: String,
pub arguments: String,
}
@@ -91,6 +92,133 @@ pub fn choose_tools<'t>(
}
}
+/// Declared JSON-Schema types of each tool parameter, keyed by function name.
+/// Tool-call markup cannot carry scalar types — the parser keeps every scalar
+/// parameter a string and types JSON-shaped values by their braces — so the
+/// declared schema is what restores the wire types clients validate against.
+#[derive(Clone, Default)]
+pub struct ToolParameterTypes(HashMap<String, HashMap<String, Vec<String>>>);
+
+impl ToolParameterTypes {
+ pub fn from_tools(tools: Option<&[OaiTool]>) -> Self {
+ let mut functions = HashMap::new();
+ for tool in tools.unwrap_or_default() {
+ let Some(parameters) = &tool.function.parameters else {
+ continue;
+ };
+ let Ok(schema) = serde_json::from_str::<serde_json::Value>(&parameters.json) else {
+ continue;
+ };
+ let Some(properties) = schema.get("properties").and_then(serde_json::Value::as_object) else {
+ continue;
+ };
+ let parameters = properties
+ .iter()
+ .filter_map(|(name, property)| {
+ let types = match property.get("type")? {
+ serde_json::Value::String(kind) => vec![kind.clone()],
+ serde_json::Value::Array(kinds) => {
+ kinds.iter().filter_map(|kind| kind.as_str().map(str::to_string)).collect()
+ },
+ _ => return None,
+ };
+ (!types.is_empty()).then(|| (name.clone(), types))
+ })
+ .collect();
+ functions.insert(tool.function.name.clone(), parameters);
+ }
+ Self(functions)
+ }
+
+ fn declared(
+ &self,
+ function: &str,
+ parameter: &str,
+ ) -> Option<&[String]> {
+ Some(self.0.get(function)?.get(parameter)?.as_slice())
+ }
+}
+
+fn matches_declared_type(
+ value: &serde_json::Value,
+ declared: &[String],
+) -> bool {
+ declared.iter().any(|kind| match kind.as_str() {
+ "integer" => value.as_i64().is_some() || value.as_u64().is_some(),
+ "number" => value.is_number(),
+ "boolean" => value.is_boolean(),
+ "null" => value.is_null(),
+ "object" => value.is_object(),
+ "array" => value.is_array(),
+ "string" => value.is_string(),
+ _ => false,
+ })
+}
+
+/// Bare scalar text is strict JSON, plus the Python-style booleans some models
+/// emit in tool markup (qwen3.5 writes `True`/`False`).
+pub fn parse_scalar_text(text: &str) -> Option<serde_json::Value> {
+ if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(text) {
+ return Some(parsed);
+ }
+ match text.trim() {
+ text if text.eq_ignore_ascii_case("true") => Some(serde_json::Value::Bool(true)),
+ text if text.eq_ignore_ascii_case("false") => Some(serde_json::Value::Bool(false)),
+ _ => None,
+ }
+}
+
+fn coerce_parameter_value(
+ value: &serde_json::Value,
+ declared: &[String],
+) -> Option<serde_json::Value> {
+ let declares_string = declared.iter().any(|kind| kind == "string");
+ match value {
+ // The parser kept the bare markup text as a string; restore the
+ // declared type when the text reads as it. A union that includes
+ // "string" stays a string: the text is already schema-valid and the
+ // intended type is unknowable.
+ serde_json::Value::String(text) if !declares_string => {
+ let parsed = parse_scalar_text(text)?;
+ matches_declared_type(&parsed, declared).then_some(parsed)
+ },
+ // JSON-shaped markup text was typed by its braces although the
+ // parameter is a plain string.
+ value if declared == ["string"] && !value.is_string() => Some(serde_json::Value::String(value.to_string())),
+ _ => None,
+ }
+}
+
+/// Restores the declared scalar types the markup could not carry. Applied only
+/// at the OpenAI boundary: the session keeps the parser's values so its stored
+/// history stays consistent with what the template renders.
+pub fn coerce_tool_call(
+ tool_call: &ToolCall,
+ types: &ToolParameterTypes,
+) -> ToolCall {
+ let Ok(serde_json::Value::Object(mut object)) = serde_json::from_str(&tool_call.arguments.json) else {
+ return tool_call.clone();
+ };
+ let mut changed = false;
+ for (parameter, value) in object.iter_mut() {
+ if let Some(declared) = types.declared(&tool_call.name, parameter)
+ && let Some(coerced) = coerce_parameter_value(value, declared)
+ {
+ *value = coerced;
+ changed = true;
+ }
+ }
+ if !changed {
+ return tool_call.clone();
+ }
+ ToolCall {
+ arguments: Value {
+ json: serde_json::Value::Object(object).to_string(),
+ },
+ ..tool_call.clone()
+ }
+}
+
pub fn to_tool_call(tool_call: &OaiToolCall) -> ToolCall {
// Templates render arguments as key/value pairs; normalization guarantees
// an object no matter what the client echoes back.
@@ -192,9 +320,13 @@ pub fn oai_tool_call(
}
}
-pub fn reply_tool_calls(message: &ChatMessage) -> Option<Vec<OaiToolCall>> {
+pub fn reply_tool_calls(
+ message: &ChatMessage,
+ types: &ToolParameterTypes,
+) -> Option<Vec<OaiToolCall>> {
let tool_calls = message.tool_calls();
- (!tool_calls.is_empty()).then(|| tool_calls.iter().map(|tool_call| oai_tool_call(None, tool_call)).collect())
+ (!tool_calls.is_empty())
+ .then(|| tool_calls.iter().map(|tool_call| oai_tool_call(None, &coerce_tool_call(tool_call, types))).collect())
}
// Bare-JSON formats (e.g. llama-3) stream a tool call as ordinary text and only rewrite
@@ -267,12 +399,27 @@ fn parse_framed_call(raw: &str) -> FramedCall {
}
}
-// Mirrors the parser's synthesis rule: values starting with `{` or `[` are typed JSON,
-// everything else is a string.
-fn serialize_param_value(value: &str) -> String {
+// Mirrors the parser's synthesis rule composed with the schema coercion the
+// final call goes through: values starting with `{` or `[` are typed JSON
+// unless the parameter is declared a plain string, bare scalars take a
+// declared non-string type when they parse as it, everything else is a string.
+fn serialize_param_value(
+ value: &str,
+ declared: Option<&[String]>,
+) -> String {
let trimmed = value.trim_start();
- if (trimmed.starts_with('{') || trimmed.starts_with('['))
- && let Some(parsed) = parse_lenient_json(value)
+ let json_shaped = trimmed.starts_with('{') || trimmed.starts_with('[');
+ if json_shaped && let Some(parsed) = parse_lenient_json(value) {
+ if declared.is_some_and(|declared| declared == ["string"]) {
+ return serde_json::Value::String(parsed.to_string()).to_string();
+ }
+ return parsed.to_string();
+ }
+ if !json_shaped
+ && let Some(declared) = declared
+ && !declared.iter().any(|kind| kind == "string")
+ && let Some(parsed) = parse_scalar_text(value)
+ && matches_declared_type(&parsed, declared)
{
return parsed.to_string();
}
@@ -297,17 +444,29 @@ fn withhold_ambiguous_tail(content: &str) -> &str {
// The arguments text with no closing brace: completed parameters are frozen, the open
// parameter's value streams as it grows.
-fn open_arguments_text(params: &[(String, FramedParam)]) -> Option<String> {
+fn open_arguments_text(
+ params: &[(String, FramedParam)],
+ function: Option<&str>,
+ types: &ToolParameterTypes,
+) -> Option<String> {
let mut entries: Vec<(String, String)> = Vec::new();
for (key, param) in params {
+ let declared = function.and_then(|function| types.declared(function, key));
let serialized = match param {
- FramedParam::Complete(value) => serialize_param_value(value),
+ FramedParam::Complete(value) => serialize_param_value(value, declared),
FramedParam::Open(content) => {
- // an empty value has no known type yet, and typed values (JSON starting
- // with `{`/`[`, matching the parser's synthesis rule) can only be serialized
- // once complete: either way nothing about the parameter may be emitted
+ // an empty value has no known type yet, and typed values (JSON
+ // starting with `{`/`[`, or declared non-string so the final
+ // form is a bare literal) can only be serialized once
+ // complete: either way nothing about the parameter may be emitted
let streamable = withhold_ambiguous_tail(content);
- if streamable.is_empty() || streamable.starts_with('{') || streamable.starts_with('[') {
+ let declared_non_string =
+ declared.is_some_and(|declared| !declared.iter().any(|kind| kind == "string"));
+ if streamable.is_empty()
+ || streamable.starts_with('{')
+ || streamable.starts_with('[')
+ || declared_non_string
+ {
break;
}
let quoted = serde_json::Value::String(streamable.to_string()).to_string();
@@ -367,6 +526,7 @@ impl ToolCallStreamer {
&mut self,
index: usize,
raw: &str,
+ types: &ToolParameterTypes,
) -> Vec<OaiToolCall> {
let mut deltas = Vec::new();
let name;
@@ -374,14 +534,14 @@ impl ToolCallStreamer {
let call = parse_framed_call(raw);
name = call.name.filter(|name| !name.is_empty());
self.key_order = call.params.iter().map(|(key, _)| key.clone()).collect();
- if let Some(open_text) = open_arguments_text(&call.params)
+ if let Some(open_text) = open_arguments_text(&call.params, name.as_deref(), types)
&& open_text.len() > self.open_text.len()
&& open_text.starts_with(&self.open_text)
{
deltas.push(OaiToolCall {
index: Some(index),
id: String::new(),
- kind: "function".to_string(),
+ kind: String::new(),
function: OaiFunctionCall {
name: String::new(),
arguments: open_text[self.open_text.len()..].to_string(),
@@ -423,12 +583,17 @@ impl ToolCallStreamer {
} else {
self.closing_fragment(call)
};
+ let (id, kind, name) = if self.name_announced {
+ (String::new(), String::new(), String::new())
+ } else {
+ (self.id.clone(), "function".to_string(), call.name.clone())
+ };
OaiToolCall {
index: Some(index),
- id: self.id.clone(),
- kind: "function".to_string(),
+ id,
+ kind,
function: OaiFunctionCall {
- name: call.name.clone(),
+ name,
arguments,
},
}
diff --git a/crates/legacy/cli/unit/server/chat_completions_test.rs b/crates/legacy/cli/unit/server/chat_completions_test.rs
index eea23674b..be13a2c31 100644
--- a/crates/legacy/cli/unit/server/chat_completions_test.rs
+++ b/crates/legacy/cli/unit/server/chat_completions_test.rs
@@ -81,7 +81,7 @@ fn response_format_validation_errors_are_request_errors() {
let error = ResponseFormatError::GrammarUnsupported;
match invalid_request_response("response_format", error.code(), error.message()) {
ChatCompletionResult::Error(_) => {},
- ChatCompletionResult::Json(_) | ChatCompletionResult::Stream(_) => {
+ ChatCompletionResult::Json(_, _) | ChatCompletionResult::Stream(_) => {
panic!("response_format validation errors should be request errors")
},
}
@@ -429,3 +429,84 @@ fn prefix_match_ignores_tool_call_identifiers() {
let not_matching = vec![ChatMessage::user().with_text("hi".to_string()), different];
assert!(!messages_have_prefix(&not_matching, &current));
}
+
+#[test]
+fn prefix_match_treats_coerced_arguments_as_equal() {
+ let assistant_call = |arguments: &str| {
+ ChatMessage::assistant().with_tool_call(uzu::types::basic::ToolCall {
+ identifier: Some("id".to_string()),
+ name: "search".to_string(),
+ arguments: uzu::types::basic::Value {
+ json: arguments.to_string(),
+ },
+ })
+ };
+
+ // the session stores the parser's stringified scalar; the client echoes the
+ // coerced typed value it received
+ let current = vec![assistant_call(r#"{"query":"cats","limit":"5"}"#)];
+ let extending =
+ vec![assistant_call(r#"{"query":"cats","limit":5}"#), ChatMessage::user().with_text("more".to_string())];
+ assert!(messages_have_prefix(&extending, &current));
+
+ // a genuinely different value is still a mismatch
+ let different =
+ vec![assistant_call(r#"{"query":"cats","limit":6}"#), ChatMessage::user().with_text("more".to_string())];
+ assert!(!messages_have_prefix(&different, &current));
+
+ // string-declared values that the parser typed by their braces also match their echo
+ let stored_object = vec![assistant_call(r#"{"content":{"a":1}}"#)];
+ let echoed_string =
+ vec![assistant_call(r#"{"content":"{\"a\":1}"}"#), ChatMessage::user().with_text("more".to_string())];
+ assert!(messages_have_prefix(&echoed_string, &stored_object));
+
+ // the session stores a Python-style boolean (qwen3.5 markup); the client
+ // echoes the coerced JSON boolean it received
+ let stored_python = vec![assistant_call(r#"{"metric":"True"}"#)];
+ let echoed_boolean = vec![assistant_call(r#"{"metric":true}"#), ChatMessage::user().with_text("more".to_string())];
+ assert!(messages_have_prefix(&echoed_boolean, &stored_python));
+}
+
+#[test]
+fn sampling_values_out_of_range_are_rejected() {
+ for body in [
+ r#"{"messages":[],"temperature":0}"#,
+ r#"{"messages":[],"temperature":2}"#,
+ r#"{"messages":[],"top_p":1}"#,
+ r#"{"messages":[],"top_p":0.01}"#,
+ r#"{"messages":[],"top_k":1}"#,
+ ] {
+ validate_sampling(&request(body)).unwrap_or_else(|(_, message)| panic!("expected {body} to pass: {message}"));
+ }
+
+ for (body, param) in [
+ (r#"{"messages":[],"temperature":-5}"#, "temperature"),
+ (r#"{"messages":[],"temperature":100}"#, "temperature"),
+ (r#"{"messages":[],"top_p":0}"#, "top_p"),
+ (r#"{"messages":[],"top_p":7}"#, "top_p"),
+ (r#"{"messages":[],"top_k":0}"#, "top_k"),
+ (r#"{"messages":[],"top_k":-3}"#, "top_k"),
+ ] {
+ let error = validate_sampling(&request(body)).expect_err("out-of-range value should be rejected");
+ assert_eq!(error.0, param, "for {body}");
+ }
+}
+
+#[test]
+fn backend_error_bodies_are_openai_errors_with_collapsed_prefixes() {
+ let body = serde_json::to_value(oai_error_body(
+ Status::BadRequest,
+ "backend_error",
+ "Backend error: Backend error: Role 'wizard' is not supported",
+ ))
+ .expect("serializable error body");
+ assert_eq!(body["error"]["type"], "invalid_request_error");
+ assert_eq!(body["error"]["code"], "backend_error");
+ assert_eq!(body["error"]["message"], "Role 'wizard' is not supported");
+
+ let body =
+ serde_json::to_value(oai_error_body(Status::InternalServerError, "no_response", "No response generated"))
+ .expect("serializable error body");
+ assert_eq!(body["error"]["type"], "server_error");
+ assert_eq!(body["error"]["message"], "No response generated");
+}
diff --git a/crates/legacy/cli/unit/server/chat_tool_calls_test.rs b/crates/legacy/cli/unit/server/chat_tool_calls_test.rs
index e11889952..d6e05885e 100644
--- a/crates/legacy/cli/unit/server/chat_tool_calls_test.rs
+++ b/crates/legacy/cli/unit/server/chat_tool_calls_test.rs
@@ -153,16 +153,16 @@ fn reply_tool_calls_serialize_in_openai_format() {
#[test]
fn json_candidate_announces_name_once_it_is_complete() {
let mut streamer = ToolCallStreamer::new();
- assert!(streamer.update(0, r#"{"name": "wr"#).is_empty());
- assert!(streamer.update(0, r#"{"arguments": {}}"#).is_empty());
+ assert!(streamer.update(0, r#"{"name": "wr"#, &ToolParameterTypes::default()).is_empty());
+ assert!(streamer.update(0, r#"{"arguments": {}}"#, &ToolParameterTypes::default()).is_empty());
- let deltas = streamer.update(1, r#"{"name": "write", "arguments": {"path":"#);
+ let deltas = streamer.update(1, r#"{"name": "write", "arguments": {"path":"#, &ToolParameterTypes::default());
assert_eq!(deltas.len(), 1);
assert_eq!(deltas[0].index, Some(1));
assert_eq!(deltas[0].function.name, "write");
assert_eq!(deltas[0].function.arguments, "");
- // the id is assigned at announcement time and must never change mid-call
+ // Metadata is emitted exactly once, in the announcement delta.
assert!(!deltas[0].id.is_empty());
let call = ToolCall {
identifier: Some("server-side-id-must-not-leak".to_string()),
@@ -171,9 +171,56 @@ fn json_candidate_announces_name_once_it_is_complete() {
json: r#"{"path":"/tmp/a"}"#.to_string(),
},
};
- assert_eq!(streamer.finish(1, &call).id, deltas[0].id);
- assert!(streamer.update(1, r#"{"name": "write", "arguments": {"path": "/"#).is_empty());
+ let finish = serde_json::to_value(streamer.finish(1, &call)).expect("serializable final delta");
+ assert_eq!(
+ finish,
+ serde_json::json!({
+ "index": 1,
+ "function": {"arguments": r#"{"path":"/tmp/a"}"#}
+ })
+ );
+
+ assert!(
+ streamer.update(1, r#"{"name": "write", "arguments": {"path": "/"#, &ToolParameterTypes::default()).is_empty()
+ );
+}
+
+#[test]
+fn framed_argument_deltas_do_not_repeat_announced_metadata() {
+ let mut streamer = ToolCallStreamer::new();
+ let deltas = streamer.update(0, "<function=write>\n<parameter=path>\n/tmp/a", &ToolParameterTypes::default());
+ assert_eq!(deltas.len(), 2);
+
+ let announcement = serde_json::to_value(&deltas[0]).expect("serializable announcement");
+ assert_eq!(announcement["type"], "function");
+ assert_eq!(announcement["function"]["name"], "write");
+
+ let arguments = serde_json::to_value(&deltas[1]).expect("serializable argument delta");
+ assert_eq!(
+ arguments,
+ serde_json::json!({
+ "index": 0,
+ "function": {"arguments": r#"{"path":"/tmp/a"#}
+ })
+ );
+}
+
+#[test]
+fn finish_emits_metadata_when_no_announcement_was_possible() {
+ let mut streamer = ToolCallStreamer::new();
+ let call = ToolCall {
+ identifier: None,
+ name: "write".to_string(),
+ arguments: Value {
+ json: r#"{"path":"/tmp/a"}"#.to_string(),
+ },
+ };
+
+ let finish = serde_json::to_value(streamer.finish(0, &call)).expect("serializable final delta");
+ assert_eq!(finish["type"], "function");
+ assert_eq!(finish["function"]["name"], "write");
+ assert!(!finish["id"].as_str().expect("string id").is_empty());
}
#[test]
@@ -194,7 +241,7 @@ fn framed_tool_call_streams_arguments_incrementally() {
let chars = final_markup.chars().collect::<Vec<_>>();
for i in 0..=chars.len() {
let partial: String = chars[..i].iter().collect();
- for delta in streamer.update(0, &partial) {
+ for delta in streamer.update(0, &partial, &ToolParameterTypes::default()) {
if !delta.function.name.is_empty() {
announcements += 1;
assert_eq!(delta.function.name, "write");
@@ -212,10 +259,14 @@ fn framed_tool_call_streams_arguments_incrementally() {
#[test]
fn framed_tool_call_withholds_typed_parameter_until_complete() {
let mut streamer = ToolCallStreamer::new();
- let deltas = streamer.update(0, "\n<function=read>\n<parameter=options>\n{\"a\"");
+ let deltas = streamer.update(0, "\n<function=read>\n<parameter=options>\n{\"a\"", &ToolParameterTypes::default());
assert!(deltas.iter().all(|delta| delta.function.arguments.is_empty()));
- let deltas = streamer.update(0, "\n<function=read>\n<parameter=options>\n{\"a\": 1}\n</parameter>\n</function>\n");
+ let deltas = streamer.update(
+ 0,
+ "\n<function=read>\n<parameter=options>\n{\"a\": 1}\n</parameter>\n</function>\n",
+ &ToolParameterTypes::default(),
+ );
let fragment: String = deltas.iter().map(|delta| delta.function.arguments.as_str()).collect();
assert_eq!(fragment, r#"{"options":{"a":1}"#);
@@ -245,7 +296,7 @@ fn framed_tool_call_streams_multibyte_content_without_panicking() {
let chars = final_markup.chars().collect::<Vec<_>>();
for i in 0..=chars.len() {
let partial: String = chars[..i].iter().collect();
- for delta in streamer.update(0, &partial) {
+ for delta in streamer.update(0, &partial, &ToolParameterTypes::default()) {
fragments.push_str(&delta.function.arguments);
}
}
@@ -277,7 +328,7 @@ fn framed_tool_call_with_array_parameter_assembles_exactly() {
let chars = final_markup.chars().collect::<Vec<_>>();
for i in 0..=chars.len() {
let partial: String = chars[..i].iter().collect();
- for delta in streamer.update(0, &partial) {
+ for delta in streamer.update(0, &partial, &ToolParameterTypes::default()) {
fragment_count += 1;
fragments.push_str(&delta.function.arguments);
}
@@ -288,3 +339,173 @@ fn framed_tool_call_with_array_parameter_assembles_exactly() {
let parsed: serde_json::Value = serde_json::from_str(&fragments).expect("assembled arguments parse");
assert_eq!(parsed, final_arguments);
}
+
+fn parameter_types(tools_json: &str) -> ToolParameterTypes {
+ let tools: Vec<OaiTool> = serde_json::from_str(tools_json).expect("valid tools json");
+ ToolParameterTypes::from_tools(Some(&tools))
+}
+
+const SEARCH_TOOL: &str = r#"[{"type":"function","function":{"name":"search","description":"Search",
+ "parameters":{"type":"object","properties":{
+ "query":{"type":"string"},
+ "limit":{"type":"integer"},
+ "ratio":{"type":"number"},
+ "metric":{"type":"boolean"},
+ "note":{"type":["string","null"]},
+ "padded_id":{"type":"string"}
+ },"required":["query"]}}}]"#;
+
+#[test]
+fn declared_types_restore_scalar_arguments() {
+ let types = parameter_types(SEARCH_TOOL);
+ let call = ToolCall {
+ identifier: None,
+ name: "search".to_string(),
+ arguments: Value {
+ json: r#"{"query":"cats","limit":"5","ratio":"-0.75","metric":"true","note":"null","padded_id":"00123","unknown":"7"}"#.to_string(),
+ },
+ };
+
+ let coerced: serde_json::Value =
+ serde_json::from_str(&coerce_tool_call(&call, &types).arguments.json).expect("coerced arguments parse");
+ assert_eq!(
+ coerced,
+ serde_json::json!({
+ "query": "cats",
+ "limit": 5,
+ "ratio": -0.75,
+ "metric": true,
+ // a union that includes "string" keeps the string: it is already schema-valid
+ "note": "null",
+ // declared strings are never retyped, so padded ids survive
+ "padded_id": "00123",
+ // undeclared parameters are left alone
+ "unknown": "7"
+ })
+ );
+}
+
+#[test]
+fn python_style_booleans_coerce_to_declared_boolean() {
+ let types = parameter_types(SEARCH_TOOL);
+ // qwen3.5 writes Python-style booleans into its tool markup
+ let call = ToolCall {
+ identifier: None,
+ name: "search".to_string(),
+ arguments: Value {
+ json: r#"{"metric":"True","query":"False","limit":"True"}"#.to_string(),
+ },
+ };
+
+ let coerced: serde_json::Value =
+ serde_json::from_str(&coerce_tool_call(&call, &types).arguments.json).expect("coerced arguments parse");
+ assert_eq!(
+ coerced,
+ serde_json::json!({
+ "metric": true,
+ // declared strings are never retyped
+ "query": "False",
+ // a boolean does not read as the declared integer
+ "limit": "True"
+ })
+ );
+}
+
+#[test]
+fn framed_streamer_agrees_with_coerced_boolean_finish() {
+ let types = parameter_types(SEARCH_TOOL);
+ let final_markup =
+ "<function=search>\n<parameter=query>\ncats\n</parameter>\n<parameter=metric>\nTrue\n</parameter>\n</function>";
+ // what coerce_tool_call produces from the parser's Python-style boolean
+ let call = ToolCall {
+ identifier: Some("c1".to_string()),
+ name: "search".to_string(),
+ arguments: Value {
+ json: r#"{"query":"cats","metric":true}"#.to_string(),
+ },
+ };
+
+ let mut streamer = ToolCallStreamer::new();
+ let mut fragments = String::new();
+ let chars = final_markup.chars().collect::<Vec<_>>();
+ for i in 0..=chars.len() {
+ let partial: String = chars[..i].iter().collect();
+ for delta in streamer.update(0, &partial, &types) {
+ fragments.push_str(&delta.function.arguments);
+ }
+ }
+ fragments.push_str(&streamer.finish(0, &call).function.arguments);
+
+ let parsed: serde_json::Value = serde_json::from_str(&fragments).expect("assembled arguments parse");
+ assert_eq!(parsed, serde_json::json!({"query": "cats", "metric": true}));
+}
+
+#[test]
+fn declared_string_keeps_json_shaped_text() {
+ let types = parameter_types(
+ r#"[{"type":"function","function":{"name":"write_file","description":"Write",
+ "parameters":{"type":"object","properties":{"path":{"type":"string"},"content":{"type":"string"}},
+ "required":["path","content"]}}}]"#,
+ );
+ // the parser typed the JSON-shaped content by its braces
+ let call = ToolCall {
+ identifier: None,
+ name: "write_file".to_string(),
+ arguments: Value {
+ json: r#"{"path":"data.json","content":{"a":1}}"#.to_string(),
+ },
+ };
+
+ let coerced: serde_json::Value =
+ serde_json::from_str(&coerce_tool_call(&call, &types).arguments.json).expect("coerced arguments parse");
+ assert_eq!(coerced["content"], serde_json::json!(r#"{"a":1}"#));
+}
+
+#[test]
+fn coercion_leaves_unparseable_scalars_and_wrapped_arguments_alone() {
+ let types = parameter_types(SEARCH_TOOL);
+ let call = |json: &str| ToolCall {
+ identifier: None,
+ name: "search".to_string(),
+ arguments: Value {
+ json: json.to_string(),
+ },
+ };
+
+ // text that does not read as the declared type stays a string
+ assert_eq!(coerce_tool_call(&call(r#"{"limit":"lots"}"#), &types).arguments.json, r#"{"limit":"lots"}"#);
+ // a declared integer that reads as a float stays a string
+ assert_eq!(coerce_tool_call(&call(r#"{"limit":"5.5"}"#), &types).arguments.json, r#"{"limit":"5.5"}"#);
+ // wrapped unparseable arguments are not an object of parameters
+ let wrapped = r#"{"__uzu_unparsed_arguments":"{\"a\":"}"#;
+ assert_eq!(coerce_tool_call(&call(wrapped), &types).arguments.json, wrapped);
+}
+
+#[test]
+fn framed_streamer_agrees_with_coerced_finish() {
+ let types = parameter_types(SEARCH_TOOL);
+ let final_markup =
+ "<function=search>\n<parameter=query>\ncats\n</parameter>\n<parameter=limit>\n5\n</parameter>\n</function>";
+ // what coerce_tool_call produces from the parser's stringified scalars
+ let call = ToolCall {
+ identifier: Some("c1".to_string()),
+ name: "search".to_string(),
+ arguments: Value {
+ json: r#"{"query":"cats","limit":5}"#.to_string(),
+ },
+ };
+
+ let mut streamer = ToolCallStreamer::new();
+ let mut fragments = String::new();
+ let chars = final_markup.chars().collect::<Vec<_>>();
+ for i in 0..=chars.len() {
+ let partial: String = chars[..i].iter().collect();
+ for delta in streamer.update(0, &partial, &types) {
+ fragments.push_str(&delta.function.arguments);
+ }
+ }
+ fragments.push_str(&streamer.finish(0, &call).function.arguments);
+
+ let parsed: serde_json::Value = serde_json::from_str(&fragments).expect("assembled arguments parse");
+ assert_eq!(parsed, serde_json::json!({"query": "cats", "limit": 5}));
+}
diff --git a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs
index 21a324eb8..f310778e4 100644
--- a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs
+++ b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs
@@ -35,7 +35,9 @@ use uzu_engine::{
};
#[cfg(feature = "capability-grammar")]
-use crate::engine::bridge::helpers::get_grammar;
+use crate::engine::bridge::helpers::{
+ get_grammar, grammar_trigger_token_sequence, grammar_trigger_token_sequence_for_prompt,
+};
use crate::engine::bridge::{
chat_token_state::UzuChatTokenBackendInstanceState,
helpers::{error_stream, get_max_context_length, get_sampling_method},
@@ -47,6 +49,8 @@ pub struct UzuChatTokenBackendInstance<B: Backend> {
config: ChatConfig,
stop_token_ids: Vec<i32>,
max_context_length: Option<u32>,
+ #[cfg(feature = "capability-grammar")]
+ grammar_trigger_token_sequence: Option<Vec<u64>>,
}
impl<B: Backend> UzuChatTokenBackendInstance<B> {
@@ -60,6 +64,8 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
let stop_token_ids = model.generation_config().stop_token_ids.iter().map(|id| *id as i32).collect();
let max_context_length = get_max_context_length(&model, config.context_length.clone());
+ #[cfg(feature = "capability-grammar")]
+ let grammar_trigger_token_sequence = grammar_trigger_token_sequence(&model);
Ok(Self {
engine,
@@ -67,6 +73,8 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
config,
stop_token_ids,
max_context_length,
+ #[cfg(feature = "capability-grammar")]
+ grammar_trigger_token_sequence,
})
}
}
@@ -116,7 +124,12 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
#[cfg(feature = "capability-grammar")]
let grammar = if let Some(grammar_config) = config.grammar {
- match get_grammar(grammar_config, self.model.tokenizer(), &self.stop_token_ids) {
+ let trigger_token_sequence = grammar_trigger_token_sequence_for_prompt(
+ self.grammar_trigger_token_sequence.as_deref(),
+ input,
+ self.model.tokenizer(),
+ );
+ match get_grammar(grammar_config, self.model.tokenizer(), &self.stop_token_ids, trigger_token_sequence) {
Ok(grammar) => Some(grammar),
Err(err) => {
return Box::pin(NoMetricsStream::new(error_stream(err.to_string())));
diff --git a/crates/legacy/uzu/src/engine/bridge/helpers.rs b/crates/legacy/uzu/src/engine/bridge/helpers.rs
index 1fc4e61bb..1208b2b82 100644
--- a/crates/legacy/uzu/src/engine/bridge/helpers.rs
+++ b/crates/legacy/uzu/src/engine/bridge/helpers.rs
@@ -29,6 +29,7 @@ pub fn get_grammar(
grammar: ShojiGrammar,
tokenizer: &Tokenizer,
stop_token_ids: &[i32],
+ trigger_token_sequence: Option<Vec<u64>>,
) -> Result<UzuGrammar, GrammarError> {
let config = match grammar {
ShojiGrammar::JsonAny {
@@ -41,7 +42,44 @@ pub fn get_grammar(
pattern,
} => GrammarConfig::regex(pattern, false),
};
- UzuGrammar::new(&config, tokenizer, None, Some(stop_token_ids))
+ UzuGrammar::new(&config, tokenizer, trigger_token_sequence, Some(stop_token_ids))
+}
+
+/// The token sequence of the model's end-of-thinking tag. A reasoning model
+/// opens every reply with reasoning tokens that no grammar can accept, so the
+/// grammar must stay disengaged until this sequence has been generated.
+#[cfg(feature = "capability-grammar")]
+pub fn grammar_trigger_token_sequence<B: Backend>(model: &LanguageModel<B>) -> Option<Vec<u64>> {
+ let tag = model.end_of_thinking_tag()?;
+ let token_ids: Vec<u64> =
+ model.tokenizer().encode(tag, false).ok()?.get_ids().iter().map(|token_id| u64::from(*token_id)).collect();
+ (!token_ids.is_empty()).then_some(token_ids)
+}
+
+/// Returns the end-of-thinking trigger only when it still has to be generated.
+/// Toggle-capable models render the trigger into the generation prompt when
+/// reasoning is disabled, in which case the grammar must engage immediately.
+#[cfg(feature = "capability-grammar")]
+pub fn grammar_trigger_token_sequence_for_prompt(
+ trigger_token_sequence: Option<&[u64]>,
+ prompt: &[u64],
+ tokenizer: &Tokenizer,
+) -> Option<Vec<u64>> {
+ let trigger_token_sequence = trigger_token_sequence.filter(|sequence| !sequence.is_empty())?;
+ let trailing_tokens = prompt
+ .windows(trigger_token_sequence.len())
+ .rposition(|window| window == trigger_token_sequence)
+ .map(|start| &prompt[start + trigger_token_sequence.len()..]);
+
+ let trigger_is_in_generation_prompt = trailing_tokens.is_some_and(|trailing_tokens| {
+ let Ok(trailing_tokens) = trailing_tokens.iter().copied().map(u32::try_from).collect::<Result<Vec<_>, _>>()
+ else {
+ return false;
+ };
+ tokenizer.decode(&trailing_tokens, false).is_ok_and(|suffix| suffix.trim().is_empty())
+ });
+
+ (!trigger_is_in_generation_prompt).then(|| trigger_token_sequence.to_vec())
}
pub fn get_max_context_length<B: Backend>(
@@ -93,3 +131,43 @@ pub fn get_sampling_method<B: Backend>(
},
}
}
+
+#[cfg(all(test, feature = "capability-grammar"))]
+mod tests {
+ use tokenizers::{Tokenizer, decoders::fuse::Fuse, models::wordlevel::WordLevel};
+
+ use super::grammar_trigger_token_sequence_for_prompt;
+
+ fn tokenizer() -> Tokenizer {
+ let model = WordLevel::builder()
+ .vocab(
+ [
+ ("<unk>".to_string(), 0),
+ ("</think>".to_string(), 1),
+ ("\n\n".to_string(), 2),
+ ("next".to_string(), 3),
+ ]
+ .into_iter()
+ .collect(),
+ )
+ .build()
+ .unwrap();
+ let mut tokenizer = Tokenizer::new(model);
+ tokenizer.with_decoder(Some(Fuse::new()));
+ tokenizer
+ }
+
+ #[test]
+ fn grammar_engages_immediately_when_prompt_closes_reasoning() {
+ let tokenizer = tokenizer();
+
+ assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2], &tokenizer), None);
+ }
+
+ #[test]
+ fn grammar_waits_for_trigger_when_only_history_contains_it() {
+ let tokenizer = tokenizer();
+
+ assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2, 3], &tokenizer), Some(vec![1]));
+ }
+}
diff --git a/crates/uzu-engine/src/engine/language_model/grammar/engagement.rs b/crates/uzu-engine/src/engine/language_model/grammar/engagement.rs
index e1c7f9ae1..d530b85ea 100644
--- a/crates/uzu-engine/src/engine/language_model/grammar/engagement.rs
+++ b/crates/uzu-engine/src/engine/language_model/grammar/engagement.rs
@@ -1,19 +1,52 @@
+/// Decides when the grammar starts constraining sampling. `Always` engages from
+/// the first generated token. `Triggered` stays dormant until the generated
+/// stream contains the trigger token sequence (a reasoning model's
+/// end-of-thinking tag), so reasoning tokens flow unconstrained and only the
+/// final answer is grammar-constrained.
pub enum GrammarEngagementState {
Always,
Triggered {
- trigger_token_id: u64,
- trigger_distance: Option<usize>,
+ trigger_sequence: Vec<u64>,
+ /// Match-automaton state after each accepted token: the length of the
+ /// longest prefix of `trigger_sequence` that is a suffix of the stream,
+ /// held at `trigger_sequence.len()` once the trigger has completed. One
+ /// entry per accepted token so `rollback` restores any earlier state
+ /// exactly.
+ match_history: Vec<usize>,
},
}
+/// The last `matched` tokens of the stream equal `trigger_sequence[..matched]`,
+/// so the new match length is the longest prefix of `trigger_sequence` that is
+/// a suffix of `trigger_sequence[..matched]` followed by `token_id`.
+fn advance(
+ trigger_sequence: &[u64],
+ matched: usize,
+ token_id: u64,
+) -> usize {
+ if matched == trigger_sequence.len() {
+ return matched;
+ }
+ let mut length = matched + 1;
+ while length > 0 {
+ if trigger_sequence[length - 1] == token_id
+ && trigger_sequence[matched + 1 - length..matched] == trigger_sequence[..length - 1]
+ {
+ return length;
+ }
+ length -= 1;
+ }
+ 0
+}
+
impl GrammarEngagementState {
pub fn is_engaged(&self) -> bool {
match self {
Self::Always => true,
Self::Triggered {
- trigger_token_id: _,
- trigger_distance,
- } => trigger_distance.is_some(),
+ trigger_sequence,
+ match_history,
+ } => match_history.last() == Some(&trigger_sequence.len()),
}
}
@@ -24,14 +57,11 @@ impl GrammarEngagementState {
match self {
Self::Always => (),
Self::Triggered {
- trigger_token_id,
- trigger_distance,
+ trigger_sequence,
+ match_history,
} => {
- if let Some(trigger_distance) = trigger_distance {
- *trigger_distance += 1;
- } else if token_id == *trigger_token_id {
- *trigger_distance = Some(0);
- }
+ let matched = match_history.last().copied().unwrap_or(0);
+ match_history.push(advance(trigger_sequence, matched, token_id));
},
}
}
@@ -43,13 +73,28 @@ impl GrammarEngagementState {
match self {
Self::Always => num_tokens,
Self::Triggered {
- trigger_token_id: _,
- trigger_distance,
+ trigger_sequence,
+ match_history,
} => {
- let num_grammar_tokens = usize::min(trigger_distance.unwrap_or(0), num_tokens);
- *trigger_distance = trigger_distance.and_then(|x| x.checked_sub(num_tokens));
+ let full_match = trigger_sequence.len();
+ let mut num_grammar_tokens = 0;
+ for _ in 0..num_tokens {
+ if match_history.pop().is_none() {
+ break;
+ }
+ // A token was fed to the matcher iff the trigger was already
+ // complete when it was accepted, i.e. the state before it
+ // was a full match.
+ if match_history.last().copied().unwrap_or(0) == full_match {
+ num_grammar_tokens += 1;
+ }
+ }
num_grammar_tokens
},
}
}
}
+
+#[cfg(test)]
+#[path = "../../../../unit/engine/language_model/grammar/engagement_test.rs"]
+mod tests;
diff --git a/crates/uzu-engine/src/engine/language_model/grammar/mod.rs b/crates/uzu-engine/src/engine/language_model/grammar/mod.rs
index 0a70da241..59bcd5628 100644
--- a/crates/uzu-engine/src/engine/language_model/grammar/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/grammar/mod.rs
@@ -33,7 +33,7 @@ impl Grammar {
pub fn new(
config: &GrammarConfig,
tokenizer: &Tokenizer,
- trigger_token_id: Option<u64>,
+ trigger_token_sequence: Option<Vec<u64>>,
stop_token_ids: Option<&[i32]>,
) -> Result<Self, GrammarError> {
let tokenizer_info =
@@ -72,13 +72,12 @@ impl Grammar {
let compiled = compiler.compile_grammar(&grammar).map_err(GrammarError::XGrammar)?;
let matcher = GrammarMatcher::new(&compiled, None, true, -1).map_err(GrammarError::XGrammar)?;
- let engagement_state = if let Some(trigger_token_id) = trigger_token_id {
- GrammarEngagementState::Triggered {
- trigger_token_id,
- trigger_distance: None,
- }
- } else {
- GrammarEngagementState::Always
+ let engagement_state = match trigger_token_sequence.filter(|sequence| !sequence.is_empty()) {
+ Some(trigger_sequence) => GrammarEngagementState::Triggered {
+ trigger_sequence,
+ match_history: Vec::new(),
+ },
+ None => GrammarEngagementState::Always,
};
Ok(Self {
@@ -131,7 +130,13 @@ impl Grammar {
&mut self,
token_id: u64,
) -> Result<(), GrammarError> {
- if self.engagement_state.is_engaged() && !self.matcher.accept_token(token_id as i32) {
+ // A terminated matcher cannot advance and its bitmask only allows stop
+ // tokens, which close generation without being part of the grammar, so
+ // tokens sampled after termination must not be rejected.
+ if self.engagement_state.is_engaged()
+ && !self.matcher.is_terminated()
+ && !self.matcher.accept_token(token_id as i32)
+ {
return Err(GrammarError::GrammarReject);
}
diff --git a/crates/uzu-engine/src/engine/language_model/mod.rs b/crates/uzu-engine/src/engine/language_model/mod.rs
index 8e4c208bd..ba3486a08 100644
--- a/crates/uzu-engine/src/engine/language_model/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/mod.rs
@@ -5,7 +5,10 @@ use tokenizers::Tokenizer;
use crate::{
backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},
- config::model::{generation::GenerationConfig, language_model::LanguageModelConfig},
+ config::{
+ model::{generation::GenerationConfig, language_model::LanguageModelConfig},
+ token_codec::AnyTokenCodecConfig,
+ },
data_type::DataType,
encodable_block::{
decoder::{Decoder, DecoderError},
@@ -29,6 +32,9 @@ pub struct LanguageModel<B: Backend> {
sampling: Sampling<B>,
context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,
generation_config: GenerationConfig,
+ /// The literal text the model emits between its reasoning and its final
+ /// answer (e.g. "</think>"); None when the model does not separate them.
+ end_of_thinking_tag: Option<String>,
tokenizer: Arc<Tokenizer>,
#[cfg(grammar)]
vocab_size: usize,
@@ -99,6 +105,10 @@ impl<B: Backend> Engine<B> {
weight_loader.tree().assert_all_tensors_validated()?;
let generation_config = config.generation_config;
+ let end_of_thinking_tag = match &config.token_codec_config {
+ AnyTokenCodecConfig::ChatCodecConfig(token_codec_config) => token_codec_config.end_of_thinking_tag.clone(),
+ AnyTokenCodecConfig::RawTextCodecConfig(_) => None,
+ };
#[cfg(grammar)]
let vocab_size = config.decoder_config.vocab_size as usize;
@@ -110,6 +120,7 @@ impl<B: Backend> Engine<B> {
sampling,
context_ring_update,
generation_config,
+ end_of_thinking_tag,
tokenizer,
#[cfg(grammar)]
vocab_size,
@@ -168,6 +179,10 @@ impl<B: Backend> LanguageModel<B> {
&self.generation_config
}
+ pub fn end_of_thinking_tag(&self) -> Option<&str> {
+ self.end_of_thinking_tag.as_deref()
+ }
+
pub fn tokenizer(&self) -> &Arc<Tokenizer> {
&self.tokenizer
}
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index f8316410b..f18c31edf 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -347,9 +347,11 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
node.logprob,
);
for &child_index in &node.child_indices {
+ // Grammar-illegal proposals are dropped; nothing may be proposed
+ // past termination either, since only stop tokens can follow.
#[cfg(grammar)]
if let Some(grammar) = grammar.as_mut()
- && grammar.accept_token(nodes[child_index].token_id as u64).is_err()
+ && (grammar.is_terminated() || grammar.accept_token(nodes[child_index].token_id as u64).is_err())
{
continue;
}
diff --git a/crates/uzu-engine/src/trie.rs b/crates/uzu-engine/src/trie.rs
index a48b06e9a..e00c3fa5f 100644
--- a/crates/uzu-engine/src/trie.rs
+++ b/crates/uzu-engine/src/trie.rs
@@ -291,11 +291,9 @@ impl<'a> FlatTrie<'a> {
break;
};
- #[cfg(grammar)]
- if let Some(grammar) = grammar.as_mut() {
- assert!(!grammar.is_terminated(), "Grammar has terminated but llm continued generation");
- }
-
+ // The grammar may already be terminated here: when the draft model
+ // proposed the grammar-closing token the trie descends one node
+ // further, where the mask only allows stop tokens.
current_token = next_token;
}
diff --git a/crates/uzu-engine/unit/engine/language_model/grammar/engagement_test.rs b/crates/uzu-engine/unit/engine/language_model/grammar/engagement_test.rs
new file mode 100644
index 000000000..4ebeb23ac
--- /dev/null
+++ b/crates/uzu-engine/unit/engine/language_model/grammar/engagement_test.rs
@@ -0,0 +1,77 @@
+use uzu_engine_macros::uzu_test;
+
+use super::GrammarEngagementState;
+
+// The muse-glimmer end-of-thinking tag:
+// <|eom|> <|start|> assistant " to" "=user" <|message|>
+const TRIGGER: [u64; 6] = [200007, 200022, 140680, 328, 76976, 200023];
+
+fn triggered() -> GrammarEngagementState {
+ GrammarEngagementState::Triggered {
+ trigger_sequence: TRIGGER.to_vec(),
+ match_history: Vec::new(),
+ }
+}
+
+#[uzu_test]
+fn always_is_engaged_and_rolls_back_everything() {
+ let mut state = GrammarEngagementState::Always;
+ assert!(state.is_engaged());
+ state.accept_token(1);
+ assert!(state.is_engaged());
+ assert_eq!(state.rollback(3), 3);
+}
+
+#[uzu_test]
+fn engages_only_after_the_full_trigger_sequence() {
+ let mut state = triggered();
+ // reasoning tokens, including a stray prefix of the trigger
+ for token in [11, 200007, 200022, 42, 13] {
+ state.accept_token(token);
+ assert!(!state.is_engaged());
+ }
+ for token in TRIGGER {
+ assert!(!state.is_engaged());
+ state.accept_token(token);
+ }
+ assert!(state.is_engaged());
+ // stays engaged for the rest of the stream
+ state.accept_token(99);
+ assert!(state.is_engaged());
+}
+
+#[uzu_test]
+fn restarts_matching_on_a_mismatch_mid_sequence() {
+ let mut state = triggered();
+ for token in [TRIGGER[0], TRIGGER[1], TRIGGER[0], TRIGGER[1], TRIGGER[2], TRIGGER[3], TRIGGER[4], TRIGGER[5]] {
+ state.accept_token(token);
+ }
+ // ..., eom, start, eom, start, assistant, to, =user, message: the second
+ // eom restarts the match, which then runs to completion
+ assert!(state.is_engaged());
+}
+
+#[uzu_test]
+fn rollback_counts_only_tokens_accepted_while_engaged() {
+ let mut state = triggered();
+ state.accept_token(11);
+ for token in TRIGGER {
+ state.accept_token(token);
+ }
+ state.accept_token(90); // grammar token
+ state.accept_token(91); // grammar token
+ // 91, 90 were matcher-fed; <|message|> completed the trigger and was not
+ assert_eq!(state.rollback(3), 2);
+ assert!(!state.is_engaged());
+ // the partial trigger state is restored: finishing it re-engages
+ state.accept_token(TRIGGER[5]);
+ assert!(state.is_engaged());
+}
+
+#[uzu_test]
+fn rollback_past_the_start_is_safe() {
+ let mut state = triggered();
+ state.accept_token(11);
+ assert_eq!(state.rollback(5), 0);
+ assert!(!state.is_engaged());
+}