A high-performance inference engine for AI models
Expose structured benchmark replies on request (#948)
Artur Mullakhmetove61e355a1caa
diff --git a/crates/legacy/cli/src/bench/model.rs b/crates/legacy/cli/src/bench/model.rsindex 4a48fe13e..6345fafa7 100644--- a/crates/legacy/cli/src/bench/model.rs+++ b/crates/legacy/cli/src/bench/model.rs@@ -2,7 +2,7 @@ use anyhow::{Result, anyhow, ensure};use rocket::serde::{Deserialize, Serialize};use uzu::types::{basic::ReasoningEffort,- session::chat::{ChatMessage, ChatReplyEnergy, ChatRole},+ session::chat::{ChatMessage, ChatReply, ChatReplyEnergy, ChatRole},};use uzu_engine::data_type::DataType;@@ -22,6 +22,8 @@ pub struct BenchTask {pub tokens_limit: u64,pub messages: Vec<BenchMessage>,pub greedy: bool,+ #[serde(default)]+ pub include_replies: bool,#[serde(default, skip_serializing_if = "Option::is_none")]pub reasoning_effort: Option<ReasoningEffort>,#[serde(default, skip_serializing_if = "Option::is_none")]@@ -114,6 +116,8 @@ pub struct BenchResult {pub output_energy: Option<ChatReplyEnergy>,pub joules_per_token: Option<f64>,pub text: String,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub replies: Option<Vec<ChatReply>>,}#[cfg(test)]diff --git a/crates/legacy/cli/src/bench/runner.rs b/crates/legacy/cli/src/bench/runner.rsindex 0a9f5745c..3875553bb 100644--- a/crates/legacy/cli/src/bench/runner.rs+++ b/crates/legacy/cli/src/bench/runner.rs@@ -132,6 +132,7 @@ impl BenchRunner {output_energy,joules_per_token,text: text.unwrap_or("".to_string()),+ replies: self.task.include_replies.then_some(replies),};results.push(result);diff --git a/crates/legacy/cli/unit/bench/model_test.rs b/crates/legacy/cli/unit/bench/model_test.rsindex ffc81047b..e5c95c7d0 100644--- a/crates/legacy/cli/unit/bench/model_test.rs+++ b/crates/legacy/cli/unit/bench/model_test.rs@@ -1,7 +1,7 @@use serde_json::json;use uzu::types::{- basic::{ReasoningEffort, ToolDescription},- session::chat::{ChatContentBlock, ChatRole},+ basic::{ReasoningEffort, ToolCall, ToolDescription},+ session::chat::{ChatContentBlock, ChatReply, ChatReplyFinishReason, ChatRole},};use super::*;@@ -24,6 +24,42 @@ fn legacy_task_still_converts_without_tools() {assert_eq!(messages[0].text().as_deref(), Some("Hello"));}+#[test]+fn benchmark_result_preserves_optional_structured_replies() {+ let mut input = json!({+ "task": task(json!([{"role": "user", "content": "Hello"}])),+ "device": {"memory_total": 1024}, "engine_version": "test", "timestamp": 0,+ "data_type": "bfloat16", "tokens_count_input": 1, "tokens_count_output": 2,+ "time_to_first_token": 1.0, "prompt_tokens_per_second": 1.0, "text": "Done"+ });+ let legacy: BenchResult = serde_json::from_value(input.clone()).unwrap();+ assert!(!legacy.task.include_replies);+ assert!(serde_json::to_value(&legacy).unwrap().get("replies").is_none());++ input["task"]["include_replies"] = json!(true);+ input["replies"] = json!([+ ChatReply {+ message: ChatMessage::assistant().with_reasoning("Check the clock".to_string()).with_tool_call(ToolCall {+ identifier: Some("call_clock".to_string()),+ name: "clock".to_string(),+ arguments: json!({"zone": "UTC"}).into(),+ }),+ stats: Default::default(),+ finish_reason: Some(ChatReplyFinishReason::ToolCalls),+ },+ ChatReply {+ message: ChatMessage::assistant().with_text("Done".to_string()),+ stats: Default::default(),+ finish_reason: Some(ChatReplyFinishReason::Stop),+ },+ ]);+ let result: BenchResult = serde_json::from_value(input.clone()).unwrap();+ let saved = serde_json::to_value(result).unwrap();+ assert_eq!(saved["replies"], input["replies"]);+ assert_eq!(saved["task"]["include_replies"], true);+ assert_eq!(saved["text"], "Done");+}+#[test]fn reasoning_effort_is_applied_and_preserved_in_echo() {for (context, system_text) in [