Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

7.2 KB
Raw Blame
use std::{
    path::Path,
    time::{SystemTime, UNIX_EPOCH},
};

use anyhow::{Context, Result};
use shoji::types::model::{ModelFamily, ModelSource};
use uzu::{
    engine::{Engine, EngineConfig},
    types::{
        basic::SamplingMethod,
        model::ModelAccessibility,
        session::chat::{ChatConfig, ChatReplyConfig, ReplyEnergy},
    },
};
use uzu_engine::VERSION;

use crate::{
    bench::{
        model::{BenchDevice, BenchResult, BenchTask},
        stat::mean,
    },
    common::thinking::ThinkingSupport,
};

pub struct BenchRunner {
    pub task: BenchTask,
    pub model_path: String,
}

impl BenchRunner {
    pub fn new(
        task: BenchTask,
        model_path: String,
    ) -> Self {
        Self {
            task,
            model_path,
        }
    }

    pub async fn run<F: FnMut(f64)>(
        &self,
        mut progress: Option<F>,
    ) -> Result<Vec<BenchResult>> {
        // Windows canonical paths carry a verbatim prefix no cached model path has, so there the path is only absolute.
        let model_path = if cfg!(windows) {
            std::path::absolute(&self.model_path)
        } else {
            Path::new(&self.model_path).canonicalize()
        }
        .with_context(|| format!("Can not resolve model path: {}", self.model_path))?;
        let model_path_string = model_path.to_string_lossy().into_owned();
        // Path lookup already handles unregistered directories. Registering the
        // parent would duplicate models that are also in the download cache.
        let engine_config = EngineConfig::default();
        let engine = Engine::new(engine_config).await.with_context(|| "Can not create engine".to_string())?;

        let mut model = engine
            .model_by_path(model_path_string.clone())
            .await?
            .with_context(|| format!("Model not found at path: {model_path_string}"))?;
        if model.family.is_none()
            && let Some(model_family) = self.get_remote_model_family(&model_path, &engine).await?
        {
            model.family = Some(model_family);
        }

        let messages = self.task.to_chat_messages(ThinkingSupport::for_model(&model))?;
        let device = BenchDevice::current();

        let session_config = ChatConfig::default();
        let session = engine.chat(model, session_config).await?;

        let warmup_config = ChatReplyConfig::default().with_token_limit(Some(1));
        let _ = session.reply(messages.clone(), warmup_config).await?;

        let mut results = Vec::<BenchResult>::new();
        for run_idx in 0..self.task.number_of_runs {
            session.reset().await?;

            let mut reply_config = ChatReplyConfig::default().with_token_limit(Some(self.task.tokens_limit as u32));
            if self.task.greedy {
                reply_config = reply_config.with_sampling_method(SamplingMethod::Greedy {})
            }

            let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
            let replies = session.reply(messages.clone(), reply_config).await?;

            let mut tokens_count_input = 0u64;
            let mut tokens_count_output = 0u64;
            let mut time_to_first_token = 0.0f64;
            let mut prompt_tokens_per_second = 0.0f64;
            let mut generate_tokens_per_second = Vec::new();
            for reply in replies.iter() {
                tokens_count_input += reply.stats.tokens_count_input.unwrap_or(0) as u64;
                tokens_count_output += reply.stats.tokens_count_output.unwrap_or(0) as u64;
                time_to_first_token += reply.stats.time_to_first_token.unwrap_or(0.0f64);
                prompt_tokens_per_second += reply.stats.prefill_tokens_per_second.unwrap_or(0.0f64);
                if let Some(value) = reply.stats.generate_tokens_per_second {
                    generate_tokens_per_second.push(value);
                }
            }

            let mut text: Option<String> = None;
            if !replies.is_empty() {
                let replies_count = replies.len() as f64;
                time_to_first_token /= replies_count;
                prompt_tokens_per_second /= replies_count;
                text = replies.last().unwrap().message.text();
            }
            let generate_tokens_per_second = mean(&generate_tokens_per_second);

            let input_energy = aggregate_energy(replies.iter().filter_map(|reply| reply.stats.input_energy.as_ref()));
            let output_energy = aggregate_energy(replies.iter().filter_map(|reply| reply.stats.output_energy.as_ref()));
            let total_joules =
                aggregate_energy(input_energy.iter().chain(output_energy.iter())).map(|energy| energy.total());
            let tokens_count = tokens_count_input + tokens_count_output;
            let joules_per_token =
                total_joules.and_then(|joules| (tokens_count > 0).then_some(joules / tokens_count as f64));

            let result = BenchResult {
                task: self.task.clone(),
                device: device.clone(),
                engine_version: VERSION.to_string(),
                timestamp,
                data_type: "bfloat16".to_string(),
                memory_used: session.peak_memory_usage().await,
                tokens_count_input,
                tokens_count_output,
                time_to_first_token,
                prompt_tokens_per_second,
                generate_tokens_per_second,
                input_energy,
                output_energy,
                joules_per_token,
                text: text.unwrap_or("".to_string()),
                replies: self.task.include_replies.then_some(replies),
            };
            results.push(result);

            if let Some(progress) = progress.as_mut() {
                progress((run_idx + 1) as f64 / self.task.number_of_runs as f64);
            }
        }

        Ok(results)
    }

    async fn get_remote_model_family(
        &self,
        model_path: &Path,
        engine: &Engine,
    ) -> Result<Option<ModelFamily>> {
        let dir_name = model_path
            .file_name()
            .ok_or(anyhow::format_err!("Can not get directory name"))?
            .to_string_lossy()
            .into_owned()
            .to_lowercase();
        let all_models = engine.models().await?;
        for model in all_models {
            if let ModelAccessibility::OnDevice {
                source:
                    ModelSource::Registry {
                        toolchain_version: _toolchain_version,
                        repository,
                        source_repository,
                        files: _files,
                    },
            } = model.accessibility
            {
                if let Some(repo) = source_repository
                    && repo.identifier.to_lowercase().contains(&dir_name)
                {
                    return Ok(model.family);
                }
                if let Some(repo) = repository
                    && repo.identifier.to_lowercase().contains(&dir_name)
                {
                    return Ok(model.family);
                }
            }
        }

        Ok(None)
    }
}

fn aggregate_energy<'a>(energy: impl IntoIterator<Item = &'a ReplyEnergy>) -> Option<ReplyEnergy> {
    energy.into_iter().cloned().reduce(|total, energy| total + energy)
}