Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

4.8 KB
Raw Blame
use anyhow::{Result, anyhow, ensure};
use rocket::serde::{Deserialize, Serialize};
use sysinfo::System;
use uzu::types::{
    basic::ReasoningEffort,
    session::chat::{ChatMessage, ChatReply, ChatRole, ReplyEnergy},
};

use crate::{
    common::thinking::ThinkingSupport,
    server::{
        chat_completions::{OaiMessage, to_chat_messages},
        chat_tool_calls::{OaiTool, backfill_tool_result_names, choose_tools, insert_tools_message},
    },
};

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchTask {
    pub identifier: String,
    pub repo_id: String,
    pub number_of_runs: u64,
    pub tokens_limit: u64,
    pub messages: Vec<BenchMessage>,
    pub greedy: bool,
    #[serde(default)]
    pub include_replies: bool,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub reasoning_effort: Option<ReasoningEffort>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub tools: Option<Vec<serde_json::Value>>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub tool_choice: Option<serde_json::Value>,
}

impl BenchTask {
    pub fn to_chat_messages(
        &self,
        thinking_support: ThinkingSupport,
    ) -> Result<Vec<ChatMessage>> {
        let messages: Vec<OaiMessage> = serde_json::from_value(serde_json::to_value(&self.messages)?)
            .map_err(|_| anyhow!("Invalid benchmark messages"))?;
        let tools: Vec<OaiTool> =
            serde_json::from_value(serde_json::to_value(self.tools.as_deref().unwrap_or_default())?)
                .map_err(|_| anyhow!("Invalid benchmark tools"))?;
        // Reject arguments that the server converter would repair or wrap;
        // replay must preserve the recorded calls.
        for call in messages.iter().flat_map(|message| message.tool_calls.iter().flatten()) {
            ensure!(
                serde_json::from_str::<serde_json::Value>(&call.function.arguments)
                    .is_ok_and(|arguments| arguments.is_object()),
                "Invalid benchmark tool arguments: expected a JSON object"
            );
        }
        let mut messages = to_chat_messages(&messages);
        if let Some(effort) = self.reasoning_effort
            && let Some(effort) = thinking_support
                .fulfill_requested_effort(effort)
                .map_err(|error| anyhow!("Invalid benchmark reasoning_effort: {error}"))?
        {
            match messages.first_mut() {
                Some(first) if first.role == (ChatRole::System {}) => {
                    *first = first.clone().with_reasoning_effort(effort);
                },
                _ => messages.insert(0, ChatMessage::system().with_reasoning_effort(effort)),
            }
        }
        backfill_tool_result_names(&mut messages);
        let tools = choose_tools(Some(&tools), self.tool_choice.as_ref())
            .map_err(|_| anyhow!("Invalid benchmark tool_choice"))?;
        insert_tools_message(&mut messages, &tools);
        Ok(messages)
    }
}

#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct BenchMessage {
    pub role: BenchMessageRole,
    pub content: Option<String>,
    pub reasoning_content: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub tool_calls: Option<Vec<serde_json::Value>>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub tool_call_id: Option<String>,
}

#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum BenchMessageRole {
    System,
    User,
    Assistant,
    Tool,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchDevice {
    pub os_name: Option<String>,
    pub cpu_name: Option<String>,
    pub memory_total: u64,
}

impl BenchDevice {
    pub fn current() -> Self {
        let mut system_info = System::new_all();
        system_info.refresh_all();

        let os_name = System::long_os_version();
        let cpu_name = system_info.cpus().first().map(|cpu| cpu.brand().to_string());
        let memory_total = system_info.total_memory();

        Self {
            os_name,
            cpu_name,
            memory_total,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchResult {
    pub task: BenchTask,
    pub device: BenchDevice,
    pub engine_version: String,
    pub timestamp: u64,
    pub data_type: String,
    pub memory_used: Option<usize>,
    pub tokens_count_input: u64,
    pub tokens_count_output: u64,
    pub time_to_first_token: f64,
    pub prompt_tokens_per_second: f64,
    pub generate_tokens_per_second: Option<f64>,
    pub input_energy: Option<ReplyEnergy>,
    pub output_energy: Option<ReplyEnergy>,
    pub joules_per_token: Option<f64>,
    pub text: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub replies: Option<Vec<ChatReply>>,
}

#[cfg(test)]
#[path = "../../unit/bench/model_test.rs"]
mod tests;