Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Expose structured benchmark replies on request (#948)

Artur Mullakhmetove61e355a1caa
diff --git a/crates/legacy/cli/src/bench/model.rs b/crates/legacy/cli/src/bench/model.rs
index 4a48fe13e..6345fafa7 100644
--- a/crates/legacy/cli/src/bench/model.rs
+++ b/crates/legacy/cli/src/bench/model.rs
@@ -2,7 +2,7 @@ use anyhow::{Result, anyhow, ensure};
use rocket::serde::{Deserialize, Serialize};
use uzu::types::{
basic::ReasoningEffort,
- session::chat::{ChatMessage, ChatReplyEnergy, ChatRole},
+ session::chat::{ChatMessage, ChatReply, ChatReplyEnergy, ChatRole},
};
use uzu_engine::data_type::DataType;
@@ -22,6 +22,8 @@ pub struct BenchTask {
pub tokens_limit: u64,
pub messages: Vec<BenchMessage>,
pub greedy: bool,
+ #[serde(default)]
+ pub include_replies: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reasoning_effort: Option<ReasoningEffort>,
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -114,6 +116,8 @@ pub struct BenchResult {
pub output_energy: Option<ChatReplyEnergy>,
pub joules_per_token: Option<f64>,
pub text: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub replies: Option<Vec<ChatReply>>,
}
#[cfg(test)]
diff --git a/crates/legacy/cli/src/bench/runner.rs b/crates/legacy/cli/src/bench/runner.rs
index 0a9f5745c..3875553bb 100644
--- a/crates/legacy/cli/src/bench/runner.rs
+++ b/crates/legacy/cli/src/bench/runner.rs
@@ -132,6 +132,7 @@ impl BenchRunner {
output_energy,
joules_per_token,
text: text.unwrap_or("".to_string()),
+ replies: self.task.include_replies.then_some(replies),
};
results.push(result);
diff --git a/crates/legacy/cli/unit/bench/model_test.rs b/crates/legacy/cli/unit/bench/model_test.rs
index ffc81047b..e5c95c7d0 100644
--- a/crates/legacy/cli/unit/bench/model_test.rs
+++ b/crates/legacy/cli/unit/bench/model_test.rs
@@ -1,7 +1,7 @@
use serde_json::json;
use uzu::types::{
- basic::{ReasoningEffort, ToolDescription},
- session::chat::{ChatContentBlock, ChatRole},
+ basic::{ReasoningEffort, ToolCall, ToolDescription},
+ session::chat::{ChatContentBlock, ChatReply, ChatReplyFinishReason, ChatRole},
};
use super::*;
@@ -24,6 +24,42 @@ fn legacy_task_still_converts_without_tools() {
assert_eq!(messages[0].text().as_deref(), Some("Hello"));
}
+#[test]
+fn benchmark_result_preserves_optional_structured_replies() {
+ let mut input = json!({
+ "task": task(json!([{"role": "user", "content": "Hello"}])),
+ "device": {"memory_total": 1024}, "engine_version": "test", "timestamp": 0,
+ "data_type": "bfloat16", "tokens_count_input": 1, "tokens_count_output": 2,
+ "time_to_first_token": 1.0, "prompt_tokens_per_second": 1.0, "text": "Done"
+ });
+ let legacy: BenchResult = serde_json::from_value(input.clone()).unwrap();
+ assert!(!legacy.task.include_replies);
+ assert!(serde_json::to_value(&legacy).unwrap().get("replies").is_none());
+
+ input["task"]["include_replies"] = json!(true);
+ input["replies"] = json!([
+ ChatReply {
+ message: ChatMessage::assistant().with_reasoning("Check the clock".to_string()).with_tool_call(ToolCall {
+ identifier: Some("call_clock".to_string()),
+ name: "clock".to_string(),
+ arguments: json!({"zone": "UTC"}).into(),
+ }),
+ stats: Default::default(),
+ finish_reason: Some(ChatReplyFinishReason::ToolCalls),
+ },
+ ChatReply {
+ message: ChatMessage::assistant().with_text("Done".to_string()),
+ stats: Default::default(),
+ finish_reason: Some(ChatReplyFinishReason::Stop),
+ },
+ ]);
+ let result: BenchResult = serde_json::from_value(input.clone()).unwrap();
+ let saved = serde_json::to_value(result).unwrap();
+ assert_eq!(saved["replies"], input["replies"]);
+ assert_eq!(saved["task"]["include_replies"], true);
+ assert_eq!(saved["text"], "Done");
+}
+
#[test]
fn reasoning_effort_is_applied_and_preserved_in_echo() {
for (context, system_text) in [