A high-performance inference engine for AI models
Expose prefill_step_size for benchmarks
Aleksei Savinf183797183f5
diff --git a/crates/benchmarks/src/runner/runner.rs b/crates/benchmarks/src/runner/runner.rsindex 4a09540db..9d4232674 100644--- a/crates/benchmarks/src/runner/runner.rs+++ b/crates/benchmarks/src/runner/runner.rs@@ -9,7 +9,9 @@ use uzu::{session::{ChatSession,config::{DecodingConfig, RunConfig},- parameter::{ContextLength, SamplingMethod, SamplingPolicy},+ parameter::{+ ContextLength, PrefillStepSize, SamplingMethod, SamplingPolicy,+ },types::{Input, Output},},};@@ -22,16 +24,19 @@ use crate::runner::{pub struct Runner {pub task: Task,pub model_path: String,+ pub prefill_step_size: Option<usize>,}impl Runner {pub fn new(task: Task,model_path: String,+ prefill_step_size: Option<usize>,) -> Self {Self {task,model_path,+ prefill_step_size,}}@@ -64,10 +69,17 @@ impl Runner {{let context_length = self.minimal_context_length()?;+ let mut decoding_config = DecodingConfig::default()+ .with_context_length(ContextLength::Custom(context_length));+ if let Some(prefill_step_size) = self.prefill_step_size {+ decoding_config = decoding_config.with_prefill_step_size(+ PrefillStepSize::Custom(prefill_step_size),+ );+ }+let mut session = ChatSession::new(PathBuf::from(self.model_path.clone()),- DecodingConfig::default()- .with_context_length(ContextLength::Custom(context_length)),+ decoding_config,)?;let precision =diff --git a/crates/cli/src/handlers/bench.rs b/crates/cli/src/handlers/bench.rsindex 7aff71622..4c420383e 100644--- a/crates/cli/src/handlers/bench.rs+++ b/crates/cli/src/handlers/bench.rs@@ -18,7 +18,7 @@ pub fn handle_bench(println!("Model: {}", task.repo_id);let progress_bar = ProgressBar::new(task.number_of_runs);progress_bar.set_position(0);- let runner = Runner::new(task.clone(), model_path);+ let runner = Runner::new(task.clone(), model_path, None);let results = runner.run(Some(|_: f64| {progress_bar.inc(1);}))?;