A high-performance inference engine for AI models
use std::{
path::Path,
time::{SystemTime, UNIX_EPOCH},
};
use anyhow::{Context, Result};
use shoji::types::model::{ModelFamily, ModelSource};
use uzu::{
engine::{Engine, EngineConfig},
types::{
basic::SamplingMethod,
model::ModelAccessibility,
session::chat::{ChatConfig, ChatReplyConfig, ReplyEnergy},
},
};
use uzu_engine::VERSION;
use crate::{
bench::{
model::{BenchDevice, BenchResult, BenchTask},
stat::mean,
},
common::thinking::ThinkingSupport,
};
pub struct BenchRunner {
pub task: BenchTask,
pub model_path: String,
}
impl BenchRunner {
pub fn new(
task: BenchTask,
model_path: String,
) -> Self {
Self {
task,
model_path,
}
}
pub async fn run<F: FnMut(f64)>(
&self,
mut progress: Option<F>,
) -> Result<Vec<BenchResult>> {
// Windows canonical paths carry a verbatim prefix no cached model path has, so there the path is only absolute.
let model_path = if cfg!(windows) {
std::path::absolute(&self.model_path)
} else {
Path::new(&self.model_path).canonicalize()
}
.with_context(|| format!("Can not resolve model path: {}", self.model_path))?;
let model_path_string = model_path.to_string_lossy().into_owned();
// Path lookup already handles unregistered directories. Registering the
// parent would duplicate models that are also in the download cache.
let engine_config = EngineConfig::default();
let engine = Engine::new(engine_config).await.with_context(|| "Can not create engine".to_string())?;
let mut model = engine
.model_by_path(model_path_string.clone())
.await?
.with_context(|| format!("Model not found at path: {model_path_string}"))?;
if model.family.is_none()
&& let Some(model_family) = self.get_remote_model_family(&model_path, &engine).await?
{
model.family = Some(model_family);
}
let messages = self.task.to_chat_messages(ThinkingSupport::for_model(&model))?;
let device = BenchDevice::current();
let session_config = ChatConfig::default();
let session = engine.chat(model, session_config).await?;
let warmup_config = ChatReplyConfig::default().with_token_limit(Some(1));
let _ = session.reply(messages.clone(), warmup_config).await?;
let mut results = Vec::<BenchResult>::new();
for run_idx in 0..self.task.number_of_runs {
session.reset().await?;
let mut reply_config = ChatReplyConfig::default().with_token_limit(Some(self.task.tokens_limit as u32));
if self.task.greedy {
reply_config = reply_config.with_sampling_method(SamplingMethod::Greedy {})
}
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
let replies = session.reply(messages.clone(), reply_config).await?;
let mut tokens_count_input = 0u64;
let mut tokens_count_output = 0u64;
let mut time_to_first_token = 0.0f64;
let mut prompt_tokens_per_second = 0.0f64;
let mut generate_tokens_per_second = Vec::new();
for reply in replies.iter() {
tokens_count_input += reply.stats.tokens_count_input.unwrap_or(0) as u64;
tokens_count_output += reply.stats.tokens_count_output.unwrap_or(0) as u64;
time_to_first_token += reply.stats.time_to_first_token.unwrap_or(0.0f64);
prompt_tokens_per_second += reply.stats.prefill_tokens_per_second.unwrap_or(0.0f64);
if let Some(value) = reply.stats.generate_tokens_per_second {
generate_tokens_per_second.push(value);
}
}
let mut text: Option<String> = None;
if !replies.is_empty() {
let replies_count = replies.len() as f64;
time_to_first_token /= replies_count;
prompt_tokens_per_second /= replies_count;
text = replies.last().unwrap().message.text();
}
let generate_tokens_per_second = mean(&generate_tokens_per_second);
let input_energy = aggregate_energy(replies.iter().filter_map(|reply| reply.stats.input_energy.as_ref()));
let output_energy = aggregate_energy(replies.iter().filter_map(|reply| reply.stats.output_energy.as_ref()));
let total_joules =
aggregate_energy(input_energy.iter().chain(output_energy.iter())).map(|energy| energy.total());
let tokens_count = tokens_count_input + tokens_count_output;
let joules_per_token =
total_joules.and_then(|joules| (tokens_count > 0).then_some(joules / tokens_count as f64));
let result = BenchResult {
task: self.task.clone(),
device: device.clone(),
engine_version: VERSION.to_string(),
timestamp,
data_type: "bfloat16".to_string(),
memory_used: session.peak_memory_usage().await,
tokens_count_input,
tokens_count_output,
time_to_first_token,
prompt_tokens_per_second,
generate_tokens_per_second,
input_energy,
output_energy,
joules_per_token,
text: text.unwrap_or("".to_string()),
replies: self.task.include_replies.then_some(replies),
};
results.push(result);
if let Some(progress) = progress.as_mut() {
progress((run_idx + 1) as f64 / self.task.number_of_runs as f64);
}
}
Ok(results)
}
async fn get_remote_model_family(
&self,
model_path: &Path,
engine: &Engine,
) -> Result<Option<ModelFamily>> {
let dir_name = model_path
.file_name()
.ok_or(anyhow::format_err!("Can not get directory name"))?
.to_string_lossy()
.into_owned()
.to_lowercase();
let all_models = engine.models().await?;
for model in all_models {
if let ModelAccessibility::OnDevice {
source:
ModelSource::Registry {
toolchain_version: _toolchain_version,
repository,
source_repository,
files: _files,
},
} = model.accessibility
{
if let Some(repo) = source_repository
&& repo.identifier.to_lowercase().contains(&dir_name)
{
return Ok(model.family);
}
if let Some(repo) = repository
&& repo.identifier.to_lowercase().contains(&dir_name)
{
return Ok(model.family);
}
}
}
Ok(None)
}
}
fn aggregate_energy<'a>(energy: impl IntoIterator<Item = &'a ReplyEnergy>) -> Option<ReplyEnergy> {
energy.into_iter().cloned().reduce(|total, energy| total + energy)
}