A high-performance inference engine for AI models
Add checking for omitted reasoning (#947)
Aleksandr Golokoz3c3b4dd5ecad
diff --git a/crates/legacy/cli/src/server/chat_completions.rs b/crates/legacy/cli/src/server/chat_completions.rsindex c2881e45b..87c472330 100644--- a/crates/legacy/cli/src/server/chat_completions.rs+++ b/crates/legacy/cli/src/server/chat_completions.rs@@ -674,40 +674,47 @@ fn json_values_equivalent(}}-/// Tool call identifiers are assigned per turn and never rendered into tokens, so they are-/// excluded from the comparison; everything else about the messages must match.+/// Clients may omit assistant reasoning when echoing a previous turn, keep the session's reasoning in that case.+/// Tool call identifiers are assigned per turn and never rendered into tokens, so they are excluded from the comparison.+/// All other content must match.fn messages_have_prefix(messages: &[ChatMessage],prefix: &[ChatMessage],) -> bool {prefix.len() <= messages.len()&& prefix.iter().zip(messages).all(|(prefix_message, message)| {+ let omitted_reasoning = matches!(message.role, ChatRole::Assistant {})+ && !message.content.iter().any(|block| matches!(block, ChatContentBlock::Reasoning { .. }));+ let prefix_content = prefix_message+ .content+ .iter()+ .filter(|block| !omitted_reasoning || !matches!(block, ChatContentBlock::Reasoning { .. }));prefix_message.role == message.role&& prefix_message.metadata == message.metadata- && prefix_message.content.len() == message.content.len()- && prefix_message.content.iter().zip(&message.content).all(|(prefix_block, block)| {- match (prefix_block, block) {- (- ChatContentBlock::ToolCall {- value: prefix_call,- },- ChatContentBlock::ToolCall {- value: call,- },- ) => {- prefix_call.name == call.name- && tool_call_arguments_equivalent(&prefix_call.arguments.json, &call.arguments.json)+ && prefix_content.clone().count() == message.content.len()+ && prefix_content.zip(&message.content).all(|(prefix_block, block)| match (prefix_block, block) {+ (+ ChatContentBlock::ToolCall {+ value: prefix_call,},- _ => prefix_block == block,- }+ ChatContentBlock::ToolCall {+ value: call,+ },+ ) => {+ prefix_call.name == call.name+ && tool_call_arguments_equivalent(&prefix_call.arguments.json, &call.arguments.json)+ },+ _ => prefix_block == block,})})}/// Returns only the messages the session has not seen yet when the request/// extends the session's current history; otherwise resets the session and-/// returns the full list. Token-level reuse itself is decided inside the-/// session, which falls back to a full prefill if the rendered tokens diverge.+/// returns the full list. Retaining the session history also retains assistant+/// reasoning omitted by the client. Token-level reuse itself is decided inside+/// the session, which falls back to a full prefill for incompatible boundaries+/// (including truncated replies) or divergent rendered tokens.async fn prepare_input(session: &ChatSession,mut messages: Vec<ChatMessage>,diff --git a/crates/legacy/cli/unit/server/chat_completions_test.rs b/crates/legacy/cli/unit/server/chat_completions_test.rsindex 9912c7c5d..921b617a3 100644--- a/crates/legacy/cli/unit/server/chat_completions_test.rs+++ b/crates/legacy/cli/unit/server/chat_completions_test.rs@@ -404,6 +404,54 @@ fn assistant_reasoning_content_round_trips_in_session_block_order() {assert_eq!(messages[2].text().as_deref(), Some("no thinking"));}+#[test]+fn prefix_match_allows_omitted_assistant_reasoning_across_turns() {+ let current = vec![+ ChatMessage::system().with_text("instructions".to_string()),+ ChatMessage::user().with_text("hi".to_string()),+ ChatMessage::assistant().with_reasoning("first thoughts".to_string()).with_text("hello".to_string()),+ ChatMessage::user().with_text("again".to_string()),+ ChatMessage::assistant().with_reasoning("second thoughts".to_string()).with_text("hello again".to_string()),+ ];+ // OpenAI clients commonly omit the reasoning field or echo null/empty text.+ for reasoning in [None, Some(serde_json::Value::Null), Some(serde_json::json!(""))] {+ let mut body = serde_json::json!({"messages": [+ {"role": "system", "content": "instructions"},+ {"role": "user", "content": "hi"},+ {"role": "assistant", "content": "hello"},+ {"role": "user", "content": "again"},+ {"role": "assistant", "content": "hello again"},+ {"role": "user", "content": "once more"}+ ]});+ if let Some(reasoning) = reasoning {+ body["messages"][2]["reasoning_content"] = reasoning.clone();+ body["messages"][4]["reasoning_content"] = reasoning;+ }+ let request: ChatCompletionRequest = serde_json::from_value(body).unwrap();+ assert!(messages_have_prefix(&to_chat_messages(&request.messages), ¤t));+ }+}++#[test]+fn prefix_match_still_rejects_edits_when_assistant_reasoning_is_omitted() {+ let current = vec![ChatMessage::assistant().with_reasoning("thoughts".to_string()).with_text("hello".to_string())];+ for edited in [+ ChatMessage::assistant().with_text("changed".to_string()),+ ChatMessage::assistant(),+ ChatMessage::user().with_text("hello".to_string()),+ ChatMessage::assistant().with_text("hello".to_string()).with_reasoning_effort(ReasoningEffort::Disabled),+ ChatMessage::assistant().with_reasoning("different thoughts".to_string()).with_text("hello".to_string()),+ ] {+ assert!(!messages_have_prefix(&[edited, ChatMessage::user().with_text("more".to_string())], ¤t));+ }++ let without_reasoning = vec![ChatMessage::assistant().with_text("hello".to_string())];+ assert!(!messages_have_prefix(¤t, &without_reasoning), "new reasoning must not be ignored");++ let system = vec![ChatMessage::system().with_reasoning("thoughts".to_string()).with_text("hello".to_string())];+ assert!(!messages_have_prefix(&[ChatMessage::system().with_text("hello".to_string())], &system));+}+#[test]fn prefix_match_preserves_tool_calls_with_empty_assistant_content() {let tool_call = uzu::types::basic::ToolCall {@@ -431,10 +479,15 @@ fn prefix_match_preserves_tool_calls_with_empty_assistant_content() {if let Some(content) = content {body["messages"][1]["content"] = serde_json::from_str(content).unwrap();}- let request: ChatCompletionRequest = serde_json::from_value(body).unwrap();- let messages = to_chat_messages(&request.messages);- assert!(messages.len() > current.len());- assert!(messages_have_prefix(&messages, ¤t), "content: {content:?}, reasoning: {reasoning:?}");+ for echo_reasoning in [true, false] {+ if !echo_reasoning {+ body["messages"][1].as_object_mut().unwrap().remove("reasoning_content");+ }+ let request: ChatCompletionRequest = serde_json::from_value(body.clone()).unwrap();+ let messages = to_chat_messages(&request.messages);+ assert!(messages.len() > current.len());+ assert!(messages_have_prefix(&messages, ¤t), "content: {content:?}, reasoning: {reasoning:?}");+ }}}}