Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

4.3 KB
Raw Blame
mod decision;
mod model;
mod runner;
mod stat;

use anyhow::{Context, Result, bail};
use comfy_table::{CellAlignment, ContentArrangement, Table, presets::UTF8_FULL};
use indicatif::ProgressBar;
use tokio::fs;

use crate::bench::{decision::DecisionBenchTask, model::BenchTask, runner::BenchRunner, stat::calculate_metric};

pub async fn run_bench(
    model_path: String,
    task_path: String,
    output_path: String,
) -> Result<()> {
    let task_content =
        fs::read_to_string(&task_path).await.with_context(|| format!("Failed to read task file: {task_path}"))?;
    let task_value: serde_json::Value =
        serde_json::from_str(&task_content).with_context(|| format!("Failed to parse task file: {task_path}"))?;
    if task_value.get("questions").is_some() {
        let task: DecisionBenchTask = serde_json::from_value(task_value)
            .with_context(|| format!("Failed to parse decision task file: {task_path}"))?;
        return decision::run(task, model_path, output_path).await;
    }

    let task: BenchTask =
        serde_json::from_value(task_value).with_context(|| format!("Failed to parse task file: {task_path}"))?;
    println!("Model: {}", task.repo_id);

    let progress_bar = ProgressBar::new(task.number_of_runs);
    progress_bar.set_position(0);

    let runner = BenchRunner::new(task.clone(), model_path);
    let results = runner
        .run(Some(|_progress| {
            progress_bar.inc(1);
        }))
        .await
        .map_err(|error| anyhow::anyhow!(error.to_string()))?;
    progress_bar.finish();

    let results_data = serde_json::to_string_pretty(&results).context("Failed to serialize benchmark results")?;
    fs::write(&output_path, results_data)
        .await
        .with_context(|| format!("Failed to write output file: {output_path}"))?;

    let Some(first_result) = results.first() else {
        bail!("Benchmark produced no results");
    };
    println!("Input tokens count: {}", first_result.tokens_count_input);

    let memory_used_list = results
        .iter()
        .filter_map(|result| result.memory_used)
        .map(|memory_used| (memory_used as f64) / 1024.0 / 1024.0 / 1024.0)
        .collect::<Vec<f64>>();
    let time_to_first_token_list = results.iter().map(|result| result.time_to_first_token).collect::<Vec<f64>>();
    let prompt_tokens_per_second_list =
        results.iter().map(|result| result.prompt_tokens_per_second).collect::<Vec<f64>>();
    let generate_tokens_per_second_list =
        results.iter().filter_map(|result| result.generate_tokens_per_second).collect::<Vec<f64>>();
    let energy_joules_list = results
        .iter()
        .filter_map(|result| {
            let mut phases = result.input_energy.iter().chain(result.output_energy.iter()).map(|energy| energy.total());
            let first = phases.next()?;
            Some(first + phases.sum::<f64>())
        })
        .collect::<Vec<f64>>();
    let joules_per_token_list = results.iter().filter_map(|result| result.joules_per_token).collect::<Vec<f64>>();

    let memory_used_metric = calculate_metric(&memory_used_list);
    let time_to_first_token_metric = calculate_metric(&time_to_first_token_list);
    let prompt_tokens_per_second_metric = calculate_metric(&prompt_tokens_per_second_list);
    let generate_tokens_per_second_metric = calculate_metric(&generate_tokens_per_second_list);
    let energy_joules_metric = calculate_metric(&energy_joules_list);
    let joules_per_token_metric = calculate_metric(&joules_per_token_list);

    let mut table = Table::new();
    table
        .load_style(UTF8_FULL.with_rounded_corners())
        .set_content_arrangement(ContentArrangement::Dynamic)
        .set_header(vec!["Metric", "Value"])
        .add_row(vec!["Used memory, GB", memory_used_metric.as_str()])
        .add_row(vec!["TTFT, s", time_to_first_token_metric.as_str()])
        .add_row(vec!["Prompt, t/s", prompt_tokens_per_second_metric.as_str()])
        .add_row(vec!["Generate, t/s", generate_tokens_per_second_metric.as_str()])
        .add_row(vec!["Total energy, J", energy_joules_metric.as_str()])
        .add_row(vec!["Energy per token, J/tok", joules_per_token_metric.as_str()]);
    let Some(column) = table.column_mut(1) else {
        bail!("Benchmark summary table value column is missing");
    };
    column.set_cell_alignment(CellAlignment::Right);
    println!("{table}");

    Ok(())
}