A high-performance inference engine for AI models
Honor reasoning effort in benchmark tasks (#945)
Artur Mullakhmetovd40a48214e2d
diff --git a/crates/legacy/cli/src/bench/model.rs b/crates/legacy/cli/src/bench/model.rsindex 326f418f1..4a48fe13e 100644--- a/crates/legacy/cli/src/bench/model.rs+++ b/crates/legacy/cli/src/bench/model.rs@@ -1,11 +1,17 @@use anyhow::{Result, anyhow, ensure};use rocket::serde::{Deserialize, Serialize};-use uzu::types::session::chat::{ChatMessage, ChatReplyEnergy};+use uzu::types::{+ basic::ReasoningEffort,+ session::chat::{ChatMessage, ChatReplyEnergy, ChatRole},+};use uzu_engine::data_type::DataType;-use crate::server::{- chat_completions::{OaiMessage, to_chat_messages},- chat_tool_calls::{OaiTool, backfill_tool_result_names, choose_tools, insert_tools_message},+use crate::{+ common::thinking::ThinkingSupport,+ server::{+ chat_completions::{OaiMessage, to_chat_messages},+ chat_tool_calls::{OaiTool, backfill_tool_result_names, choose_tools, insert_tools_message},+ },};#[derive(Debug, Clone, Serialize, Deserialize)]@@ -17,13 +23,18 @@ pub struct BenchTask {pub messages: Vec<BenchMessage>,pub greedy: bool,#[serde(default, skip_serializing_if = "Option::is_none")]+ pub reasoning_effort: Option<ReasoningEffort>,+ #[serde(default, skip_serializing_if = "Option::is_none")]pub tools: Option<Vec<serde_json::Value>>,#[serde(default, skip_serializing_if = "Option::is_none")]pub tool_choice: Option<serde_json::Value>,}impl BenchTask {- pub fn to_chat_messages(&self) -> Result<Vec<ChatMessage>> {+ pub fn to_chat_messages(+ &self,+ thinking_support: ThinkingSupport,+ ) -> Result<Vec<ChatMessage>> {let messages: Vec<OaiMessage> = serde_json::from_value(serde_json::to_value(&self.messages)?).map_err(|_| anyhow!("Invalid benchmark messages"))?;let tools: Vec<OaiTool> =@@ -39,6 +50,18 @@ impl BenchTask {);}let mut messages = to_chat_messages(&messages);+ if let Some(effort) = self.reasoning_effort+ && let Some(effort) = thinking_support+ .fulfill_requested_effort(effort)+ .map_err(|error| anyhow!("Invalid benchmark reasoning_effort: {error}"))?+ {+ match messages.first_mut() {+ Some(first) if first.role == (ChatRole::System {}) => {+ *first = first.clone().with_reasoning_effort(effort);+ },+ _ => messages.insert(0, ChatMessage::system().with_reasoning_effort(effort)),+ }+ }backfill_tool_result_names(&mut messages);let tools = choose_tools(Some(&tools), self.tool_choice.as_ref()).map_err(|_| anyhow!("Invalid benchmark tool_choice"))?;diff --git a/crates/legacy/cli/src/bench/runner.rs b/crates/legacy/cli/src/bench/runner.rsindex 3a15b9b42..0a9f5745c 100644--- a/crates/legacy/cli/src/bench/runner.rs+++ b/crates/legacy/cli/src/bench/runner.rs@@ -16,9 +16,12 @@ use uzu::{};use uzu_engine::{VERSION, data_type::DataType};-use crate::bench::{- model::{BenchDevice, BenchResult, BenchTask},- stat::mean,+use crate::{+ bench::{+ model::{BenchDevice, BenchResult, BenchTask},+ stat::mean,+ },+ common::thinking::ThinkingSupport,};pub struct BenchRunner {@@ -41,7 +44,6 @@ impl BenchRunner {&self,mut progress: Option<F>,) -> Result<Vec<BenchResult>> {- let messages = self.task.to_chat_messages()?;let model_path = Path::new(&self.model_path).canonicalize().with_context(|| format!("Can not open model path: {}", self.model_path))?;@@ -61,6 +63,7 @@ impl BenchRunner {model.family = Some(model_family);}+ let messages = self.task.to_chat_messages(ThinkingSupport::for_model(&model))?;let device = self.get_device_info();let session_config = ChatConfig::default();diff --git a/crates/legacy/cli/unit/bench/model_test.rs b/crates/legacy/cli/unit/bench/model_test.rsindex 1aad2e861..ffc81047b 100644--- a/crates/legacy/cli/unit/bench/model_test.rs+++ b/crates/legacy/cli/unit/bench/model_test.rs@@ -1,10 +1,11 @@use serde_json::json;use uzu::types::{- basic::ToolDescription,+ basic::{ReasoningEffort, ToolDescription},session::chat::{ChatContentBlock, ChatRole},};use super::*;+use crate::common::thinking::ReasoningEffortSet;fn task(messages: serde_json::Value) -> serde_json::Value {json!({@@ -17,12 +18,42 @@ fn task(messages: serde_json::Value) -> serde_json::Value {fn legacy_task_still_converts_without_tools() {let input = task(json!([{"role": "user", "content": "Hello"}]));let task: BenchTask = serde_json::from_value(input).unwrap();- let messages = task.to_chat_messages().unwrap();+ let messages = task.to_chat_messages(ThinkingSupport::Unsupported).unwrap();assert_eq!(messages.len(), 1);assert_eq!(messages[0].role, ChatRole::User {});assert_eq!(messages[0].text().as_deref(), Some("Hello"));}+#[test]+fn reasoning_effort_is_applied_and_preserved_in_echo() {+ for (context, system_text) in [+ (json!([{"role": "user", "content": "Hello"}]), None),+ (+ json!([{"role": "system", "content": "Be helpful"}, {"role": "user", "content": "Hello"}]),+ Some("Be helpful"),+ ),+ ] {+ let mut input = task(context);+ input["reasoning_effort"] = json!("low");+ let task: BenchTask = serde_json::from_value(input).unwrap();+ let messages = task+ .to_chat_messages(ThinkingSupport::Levels {+ effort: ReasoningEffort::Default,+ supported: ReasoningEffortSet::ALL,+ })+ .unwrap();+ assert_eq!(messages.len(), 2);+ assert_eq!(messages[0].role, ChatRole::System {});+ assert_eq!(messages[0].reasoning_effort(), Some(ReasoningEffort::Low));+ assert_eq!(messages[0].text().as_deref(), system_text);+ assert_eq!(messages[1].text().as_deref(), Some("Hello"));+ assert_eq!(serde_json::to_value(&task).unwrap()["reasoning_effort"], "low");+ assert!(+ task.to_chat_messages(ThinkingSupport::Unsupported).unwrap_err().to_string().contains("reasoning_effort")+ );+ }+}+#[test]fn assistant_tool_call_accepts_null_content() {let input = task(json!([{"role": "assistant", "content": null, "tool_calls": [@@ -32,7 +63,7 @@ fn assistant_tool_call_accepts_null_content() {let echoed = serde_json::to_value(&task).unwrap();assert_eq!(echoed["messages"][0]["content"], serde_json::Value::Null);assert_eq!(echoed["messages"][0]["tool_calls"], input["messages"][0]["tool_calls"]);- let messages = task.to_chat_messages().unwrap();+ let messages = task.to_chat_messages(ThinkingSupport::Unsupported).unwrap();assert_eq!(messages[0].role, ChatRole::Assistant {});assert_eq!(messages[0].text(), None);assert_eq!(messages[0].tool_calls()[0].identifier.as_deref(), Some("call_clock"));@@ -55,7 +86,7 @@ fn tool_replay_preserves_wire_input_and_builds_complete_context() {assert_eq!(echoed["tools"], input["tools"]);assert_eq!(echoed["messages"][2]["tool_calls"], input["messages"][2]["tool_calls"]);assert_eq!(echoed["messages"][3]["tool_call_id"], "call_clock");- let messages = task.to_chat_messages().unwrap();+ let messages = task.to_chat_messages(ThinkingSupport::Unsupported).unwrap();assert_eq!(messages.len(), 6);assert_eq!(messages[0].role, ChatRole::System {});assert_eq!(messages[1].role, ChatRole::Developer {});@@ -91,7 +122,7 @@ fn tool_choice_selects_benchmark_tools_and_is_preserved_in_echo() {]);input["tool_choice"] = choice.clone();let task: BenchTask = serde_json::from_value(input).unwrap();- let messages = task.to_chat_messages().unwrap();+ let messages = task.to_chat_messages(ThinkingSupport::Unsupported).unwrap();let names: Vec<_> = messages.iter().flat_map(|message| message.tool_namespaces())@@ -112,7 +143,7 @@ fn invalid_tool_choice_is_rejected_without_exposing_private_values() {let mut input = task(json!([{"role": "user", "content": "Hello"}]));input["tool_choice"] = json!("PRIVATE_SENTINEL");let task: BenchTask = serde_json::from_value(input).unwrap();- let error = task.to_chat_messages().unwrap_err();+ let error = task.to_chat_messages(ThinkingSupport::Unsupported).unwrap_err();assert!(error.to_string().contains("tool_choice"));assert!(!format!("{error:?}").contains("PRIVATE_SENTINEL"));}@@ -122,7 +153,7 @@ fn malformed_tool_definition_returns_contextual_error() {let mut input = task(json!([{"role": "user", "content": "Hello"}]));input["tools"] = json!([{"type": "function", "function": {"parameters": {}}}]);let task: BenchTask = serde_json::from_value(input).unwrap();- assert!(task.to_chat_messages().unwrap_err().to_string().contains("benchmark tools"));+ assert!(task.to_chat_messages(ThinkingSupport::Unsupported).unwrap_err().to_string().contains("benchmark tools"));}#[test]@@ -131,7 +162,9 @@ fn malformed_tool_call_returns_contextual_error() {{"id": "call_clock", "type": "function", "function": {"name": "clock", "arguments": {}}}]}]));let task: BenchTask = serde_json::from_value(input).unwrap();- assert!(task.to_chat_messages().unwrap_err().to_string().contains("benchmark messages"));+ assert!(+ task.to_chat_messages(ThinkingSupport::Unsupported).unwrap_err().to_string().contains("benchmark messages")+ );}#[test]@@ -141,7 +174,9 @@ fn invalid_tool_arguments_are_rejected_without_rewriting_history() {{"id": "call_clock", "type": "function", "function": {"name": "clock", "arguments": arguments}}]}]));let task: BenchTask = serde_json::from_value(input).unwrap();- assert!(task.to_chat_messages().unwrap_err().to_string().contains("tool arguments"));+ assert!(+ task.to_chat_messages(ThinkingSupport::Unsupported).unwrap_err().to_string().contains("tool arguments")+ );}}@@ -152,7 +187,7 @@ fn nested_object_tool_arguments_are_preserved() {{"id": "call_clock", "type": "function", "function": {"name": "clock", "arguments": arguments.to_string()}}]}]));let task: BenchTask = serde_json::from_value(input).unwrap();- let messages = task.to_chat_messages().unwrap();+ let messages = task.to_chat_messages(ThinkingSupport::Unsupported).unwrap();assert_eq!(serde_json::from_str::<serde_json::Value>(&messages[0].tool_calls()[0].arguments.json).unwrap(),arguments@@ -166,7 +201,7 @@ fn conversion_errors_do_not_include_private_values_in_debug_chain() {let calls_input = task(json!([{"role": "assistant", "content": "", "tool_calls": ["PRIVATE_SENTINEL"]}]));for input in [tools_input, calls_input] {let task: BenchTask = serde_json::from_value(input).unwrap();- let error = task.to_chat_messages().unwrap_err();+ let error = task.to_chat_messages(ThinkingSupport::Unsupported).unwrap_err();assert!(!format!("{error:?}").contains("PRIVATE_SENTINEL"));}}