Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Add checking for omitted reasoning (#947)

Aleksandr Golokoz3c3b4dd5ecad
diff --git a/crates/legacy/cli/src/server/chat_completions.rs b/crates/legacy/cli/src/server/chat_completions.rs
index c2881e45b..87c472330 100644
--- a/crates/legacy/cli/src/server/chat_completions.rs
+++ b/crates/legacy/cli/src/server/chat_completions.rs
@@ -674,40 +674,47 @@ fn json_values_equivalent(
}
}
-/// Tool call identifiers are assigned per turn and never rendered into tokens, so they are
-/// excluded from the comparison; everything else about the messages must match.
+/// Clients may omit assistant reasoning when echoing a previous turn, keep the session's reasoning in that case.
+/// Tool call identifiers are assigned per turn and never rendered into tokens, so they are excluded from the comparison.
+/// All other content must match.
fn messages_have_prefix(
messages: &[ChatMessage],
prefix: &[ChatMessage],
) -> bool {
prefix.len() <= messages.len()
&& prefix.iter().zip(messages).all(|(prefix_message, message)| {
+ let omitted_reasoning = matches!(message.role, ChatRole::Assistant {})
+ && !message.content.iter().any(|block| matches!(block, ChatContentBlock::Reasoning { .. }));
+ let prefix_content = prefix_message
+ .content
+ .iter()
+ .filter(|block| !omitted_reasoning || !matches!(block, ChatContentBlock::Reasoning { .. }));
prefix_message.role == message.role
&& prefix_message.metadata == message.metadata
- && prefix_message.content.len() == message.content.len()
- && prefix_message.content.iter().zip(&message.content).all(|(prefix_block, block)| {
- match (prefix_block, block) {
- (
- ChatContentBlock::ToolCall {
- value: prefix_call,
- },
- ChatContentBlock::ToolCall {
- value: call,
- },
- ) => {
- prefix_call.name == call.name
- && tool_call_arguments_equivalent(&prefix_call.arguments.json, &call.arguments.json)
+ && prefix_content.clone().count() == message.content.len()
+ && prefix_content.zip(&message.content).all(|(prefix_block, block)| match (prefix_block, block) {
+ (
+ ChatContentBlock::ToolCall {
+ value: prefix_call,
},
- _ => prefix_block == block,
- }
+ ChatContentBlock::ToolCall {
+ value: call,
+ },
+ ) => {
+ prefix_call.name == call.name
+ && tool_call_arguments_equivalent(&prefix_call.arguments.json, &call.arguments.json)
+ },
+ _ => prefix_block == block,
})
})
}
/// Returns only the messages the session has not seen yet when the request
/// extends the session's current history; otherwise resets the session and
-/// returns the full list. Token-level reuse itself is decided inside the
-/// session, which falls back to a full prefill if the rendered tokens diverge.
+/// returns the full list. Retaining the session history also retains assistant
+/// reasoning omitted by the client. Token-level reuse itself is decided inside
+/// the session, which falls back to a full prefill for incompatible boundaries
+/// (including truncated replies) or divergent rendered tokens.
async fn prepare_input(
session: &ChatSession,
mut messages: Vec<ChatMessage>,
diff --git a/crates/legacy/cli/unit/server/chat_completions_test.rs b/crates/legacy/cli/unit/server/chat_completions_test.rs
index 9912c7c5d..921b617a3 100644
--- a/crates/legacy/cli/unit/server/chat_completions_test.rs
+++ b/crates/legacy/cli/unit/server/chat_completions_test.rs
@@ -404,6 +404,54 @@ fn assistant_reasoning_content_round_trips_in_session_block_order() {
assert_eq!(messages[2].text().as_deref(), Some("no thinking"));
}
+#[test]
+fn prefix_match_allows_omitted_assistant_reasoning_across_turns() {
+ let current = vec![
+ ChatMessage::system().with_text("instructions".to_string()),
+ ChatMessage::user().with_text("hi".to_string()),
+ ChatMessage::assistant().with_reasoning("first thoughts".to_string()).with_text("hello".to_string()),
+ ChatMessage::user().with_text("again".to_string()),
+ ChatMessage::assistant().with_reasoning("second thoughts".to_string()).with_text("hello again".to_string()),
+ ];
+ // OpenAI clients commonly omit the reasoning field or echo null/empty text.
+ for reasoning in [None, Some(serde_json::Value::Null), Some(serde_json::json!(""))] {
+ let mut body = serde_json::json!({"messages": [
+ {"role": "system", "content": "instructions"},
+ {"role": "user", "content": "hi"},
+ {"role": "assistant", "content": "hello"},
+ {"role": "user", "content": "again"},
+ {"role": "assistant", "content": "hello again"},
+ {"role": "user", "content": "once more"}
+ ]});
+ if let Some(reasoning) = reasoning {
+ body["messages"][2]["reasoning_content"] = reasoning.clone();
+ body["messages"][4]["reasoning_content"] = reasoning;
+ }
+ let request: ChatCompletionRequest = serde_json::from_value(body).unwrap();
+ assert!(messages_have_prefix(&to_chat_messages(&request.messages), &current));
+ }
+}
+
+#[test]
+fn prefix_match_still_rejects_edits_when_assistant_reasoning_is_omitted() {
+ let current = vec![ChatMessage::assistant().with_reasoning("thoughts".to_string()).with_text("hello".to_string())];
+ for edited in [
+ ChatMessage::assistant().with_text("changed".to_string()),
+ ChatMessage::assistant(),
+ ChatMessage::user().with_text("hello".to_string()),
+ ChatMessage::assistant().with_text("hello".to_string()).with_reasoning_effort(ReasoningEffort::Disabled),
+ ChatMessage::assistant().with_reasoning("different thoughts".to_string()).with_text("hello".to_string()),
+ ] {
+ assert!(!messages_have_prefix(&[edited, ChatMessage::user().with_text("more".to_string())], &current));
+ }
+
+ let without_reasoning = vec![ChatMessage::assistant().with_text("hello".to_string())];
+ assert!(!messages_have_prefix(&current, &without_reasoning), "new reasoning must not be ignored");
+
+ let system = vec![ChatMessage::system().with_reasoning("thoughts".to_string()).with_text("hello".to_string())];
+ assert!(!messages_have_prefix(&[ChatMessage::system().with_text("hello".to_string())], &system));
+}
+
#[test]
fn prefix_match_preserves_tool_calls_with_empty_assistant_content() {
let tool_call = uzu::types::basic::ToolCall {
@@ -431,10 +479,15 @@ fn prefix_match_preserves_tool_calls_with_empty_assistant_content() {
if let Some(content) = content {
body["messages"][1]["content"] = serde_json::from_str(content).unwrap();
}
- let request: ChatCompletionRequest = serde_json::from_value(body).unwrap();
- let messages = to_chat_messages(&request.messages);
- assert!(messages.len() > current.len());
- assert!(messages_have_prefix(&messages, &current), "content: {content:?}, reasoning: {reasoning:?}");
+ for echo_reasoning in [true, false] {
+ if !echo_reasoning {
+ body["messages"][1].as_object_mut().unwrap().remove("reasoning_content");
+ }
+ let request: ChatCompletionRequest = serde_json::from_value(body.clone()).unwrap();
+ let messages = to_chat_messages(&request.messages);
+ assert!(messages.len() > current.len());
+ assert!(messages_have_prefix(&messages, &current), "content: {content:?}, reasoning: {reasoning:?}");
+ }
}
}
}