Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

remove warmup (#252)

uuuvn4dfd4df11c7e
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 7202de709..f2655a3e0 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -253,13 +253,8 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
is_prefilling: !should_sample_after_step,
};
- let (state, run_time) = self.run_model(
- task,
- false,
- self.allow_pre_encode(),
- sampling_method,
- self.should_fill_attention_bias(),
- )?;
+ let (state, run_time) =
+ self.run_model(task, self.allow_pre_encode(), sampling_method, self.should_fill_attention_bias())?;
if should_capture {
self.gpu_capture.stop_capture(&self.context.context, "prefill").map_err(|_| Error::CaptureFailed)?;
@@ -390,7 +385,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
};
let (mut state, run_time) =
- self.run_model(task, false, self.allow_pre_encode(), sampling_method, self.should_fill_attention_bias())?;
+ self.run_model(task, self.allow_pre_encode(), sampling_method, self.should_fill_attention_bias())?;
let sampled_tokens = self.read_sampling_output(&mut state)?;
@@ -729,53 +724,20 @@ impl<B: Backend> LanguageModelGenerator<B> {
let gpu_capture = GpuCaptureManager::new();
let context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;
- let prefill_step_size = decoding_config.prefill_step_size.resolve(&context.model_config);
- let generate_suffix_length = decoding_config.generate_suffix_length();
- let mut generator = Self {
+ Ok(Self {
decoding_config,
tokens: Vec::new(),
context,
pre_encoded_task: None,
registered_prefix_len: 0,
gpu_capture,
- };
-
- //Warmup
- generator.warmup(prefill_step_size)?;
- generator.warmup(generate_suffix_length)?;
-
- Ok(generator)
- }
-
- fn warmup(
- &mut self,
- suffix_length: usize,
- ) -> Result<(), Error> {
- let token_ids: Vec<u64> = vec![0; suffix_length];
- let token_positions: Vec<usize> = (0..suffix_length).collect();
- let token_seeds: Vec<u64> = vec![0; suffix_length];
-
- let task = Task {
- token_ids: &token_ids,
- token_positions: &token_positions,
- token_bitmask: None,
- token_seeds: &token_seeds,
- expected_number_of_new_tokens: suffix_length,
- active_suffix_length: suffix_length,
- sampling_start: 0,
- sampling_length: suffix_length,
- is_prefilling: false,
- };
-
- self.run_model(task, true, false, SamplingMethod::default(), false)?;
- Ok(())
+ })
}
fn run_model(
&mut self,
task: Task,
- warmup: bool,
allow_pre_encode: bool,
sampling_method: SamplingMethod,
should_fill_attention_bias: bool,
@@ -821,7 +783,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
is_prefilling: task.is_prefilling,
};
- let is_first_decode = !warmup && task.token_ids.len() == 1;
+ let is_first_decode = task.token_ids.len() == 1;
let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
if should_capture {
@@ -829,7 +791,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
self.pre_encoded_task = None;
}
- let sample = !warmup && !task.is_prefilling;
+ let sample = !task.is_prefilling;
let executable = if let Some((pre_encoded_key, pre_encoded_executable)) = self.pre_encoded_task.take()
&& pre_encoded_key == encoding_key