Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Expose prefill_step_size for benchmarks

Aleksei Savinf183797183f5
diff --git a/crates/benchmarks/src/runner/runner.rs b/crates/benchmarks/src/runner/runner.rs
index 4a09540db..9d4232674 100644
--- a/crates/benchmarks/src/runner/runner.rs
+++ b/crates/benchmarks/src/runner/runner.rs
@@ -9,7 +9,9 @@ use uzu::{
session::{
ChatSession,
config::{DecodingConfig, RunConfig},
- parameter::{ContextLength, SamplingMethod, SamplingPolicy},
+ parameter::{
+ ContextLength, PrefillStepSize, SamplingMethod, SamplingPolicy,
+ },
types::{Input, Output},
},
};
@@ -22,16 +24,19 @@ use crate::runner::{
pub struct Runner {
pub task: Task,
pub model_path: String,
+ pub prefill_step_size: Option<usize>,
}
impl Runner {
pub fn new(
task: Task,
model_path: String,
+ prefill_step_size: Option<usize>,
) -> Self {
Self {
task,
model_path,
+ prefill_step_size,
}
}
@@ -64,10 +69,17 @@ impl Runner {
{
let context_length = self.minimal_context_length()?;
+ let mut decoding_config = DecodingConfig::default()
+ .with_context_length(ContextLength::Custom(context_length));
+ if let Some(prefill_step_size) = self.prefill_step_size {
+ decoding_config = decoding_config.with_prefill_step_size(
+ PrefillStepSize::Custom(prefill_step_size),
+ );
+ }
+
let mut session = ChatSession::new(
PathBuf::from(self.model_path.clone()),
- DecodingConfig::default()
- .with_context_length(ContextLength::Custom(context_length)),
+ decoding_config,
)?;
let precision =
diff --git a/crates/cli/src/handlers/bench.rs b/crates/cli/src/handlers/bench.rs
index 7aff71622..4c420383e 100644
--- a/crates/cli/src/handlers/bench.rs
+++ b/crates/cli/src/handlers/bench.rs
@@ -18,7 +18,7 @@ pub fn handle_bench(
println!("Model: {}", task.repo_id);
let progress_bar = ProgressBar::new(task.number_of_runs);
progress_bar.set_position(0);
- let runner = Runner::new(task.clone(), model_path);
+ let runner = Runner::new(task.clone(), model_path, None);
let results = runner.run(Some(|_: f64| {
progress_bar.inc(1);
}))?;