Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

972 B
Raw Blame
use std::time::Duration;

use criterion::{BenchmarkId, Criterion};
use futures::executor::block_on;
use uzu_engine_macros::uzu_bench;

use crate::{
    engine::Engine,
    tests::{helpers::for_each_non_cpu_backend, path::get_test_model_path},
};

#[uzu_bench]
fn bench_model_loading(c: &mut Criterion) {
    let mut group = c.benchmark_group("Model loading");
    group.sample_size(10);
    group.warm_up_time(Duration::from_secs(1));
    for_each_non_cpu_backend!(|B| {
        group.bench_function(BenchmarkId::from_parameter("test model path"), |b| {
            b.iter(|| {
                // TODO: Should this reuse engine not measure creation?
                let model_path = get_test_model_path();
                let engine = block_on(Engine::<B>::new()).unwrap();
                // TODO: This should flush caches (maybe both cold/hot benchmarks?)
                block_on(engine.load_language_model(&model_path)).unwrap();
            });
        });
    });
}