A high-performance inference engine for AI models
Use max context length for backends with sparse buffers (#438)
Aleksandr Golokoz571d76717c96
diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rsindex 5a0716400..111f6f5f3 100644--- a/crates/backend-uzu/src/language_model/language_model_generator.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs@@ -124,6 +124,8 @@ pub trait LanguageModelGeneratorTrait {&mut self,context: &dyn Any,);++ fn get_context_length(&self) -> usize;}impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {@@ -647,6 +649,10 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.tokens = ctx.tokens.clone();self.registered_prefix_len = self.tokens.len().saturating_sub(1);}++ fn get_context_length(&self) -> usize {+ self.context.get_context_length(&self.decoding_config)+ }}impl<B: Backend> LanguageModelGenerator<B> {diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rsindex adad1a3e5..d7f7e75e6 100644--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs@@ -1,5 +1,6 @@use std::{cell::{Cell, RefCell},+ cmp::min,fs::File,path::Path,rc::Rc,@@ -12,11 +13,12 @@ use crate::{Backend, Context, Kernels,kernel::{TokenCopySampledKernel, kv_cache_update::KVCacheUpdate},},- config::{LanguageModelConfig, ModelMetadata},+ config::{DecoderConfig, LanguageModelConfig, ModelMetadata},encodable_block::{Decoder, Sampling},forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},language_model::rng::PRng,parameters::ParameterLoader,+ prelude::ContextLength,session::{config::DecodingConfig,parameter::{ConfigResolvableValue, ResolvableValue},@@ -124,7 +126,8 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {let prefill_step_size = decoding_config.prefill_step_size.resolve(&model_metadata.model_config);let generate_suffix_length = decoding_config.generate_suffix_length();- let max_prefix_length: usize = decoding_config.context_length.resolve(&model_metadata.model_config);+ let max_prefix_length: usize =+ Self::get_context_length_internal(&context, &model_metadata.model_config.model_config, decoding_config);let max_suffix_length: usize = std::cmp::max(prefill_step_size, generate_suffix_length);let weights_path = model_path.join("model.safetensors");@@ -166,7 +169,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {let seed = PRng::new(decoding_config.sampling_seed.resolve());- let context = Self {+ Ok(Self {context,cache_layers,shared_buffers,@@ -178,8 +181,35 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {seed,token_copy_sampled,async_buffers,- };+ })+ }++ pub fn get_context_length(+ &self,+ decoding_config: &DecodingConfig,+ ) -> usize {+ Self::get_context_length_internal(&self.context, &self.model_config.model_config, decoding_config)+ }- Ok(context)+ fn get_context_length_internal(+ context: &B::Context,+ decoder_config: &DecoderConfig,+ decoding_config: &DecodingConfig,+ ) -> usize {+ let model_length = decoder_config.transformer_config.context_length;+ let proposed_value = match decoding_config.context_length {+ ContextLength::Default => {+ if context.sparse_buffers_supported() {+ model_length+ } else if cfg!(target_os = "ios") {+ 8192+ } else {+ 16384+ }+ },+ ContextLength::Maximal => model_length,+ ContextLength::Custom(value) => value,+ };+ min(proposed_value, model_length)}}diff --git a/crates/backend-uzu/src/session/chat_session.rs b/crates/backend-uzu/src/session/chat_session.rsindex 4aa0121a5..9987ba81a 100644--- a/crates/backend-uzu/src/session/chat_session.rs+++ b/crates/backend-uzu/src/session/chat_session.rs@@ -231,7 +231,7 @@ impl ChatSession {.collect();let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- let context_length = self.decoding_config.context_length.resolve(&self.model_metadata.model_config);+ let context_length = language_model_generator.get_context_length();if tokens.len() >= context_length {return Err(Error::ContextLengthExceeded);}diff --git a/crates/backend-uzu/src/session/parameter/context_length.rs b/crates/backend-uzu/src/session/parameter/context_length.rsindex 017e0d363..1de733c17 100644--- a/crates/backend-uzu/src/session/parameter/context_length.rs+++ b/crates/backend-uzu/src/session/parameter/context_length.rs@@ -1,5 +1,3 @@-use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};-#[derive(Debug, Clone, Copy)]pub enum ContextLength {Default,@@ -12,24 +10,3 @@ impl Default for ContextLength {ContextLength::Default}}--impl ConfigResolvableValue<LanguageModelConfig, usize> for ContextLength {- fn resolve(- &self,- config: &LanguageModelConfig,- ) -> usize {- let model_context_length = config.model_config.transformer_config.context_length;- let proposed_value = match self {- ContextLength::Default => {- if cfg!(target_os = "ios") {- 8192- } else {- 16384- }- },- ContextLength::Maximal => model_context_length,- ContextLength::Custom(value) => *value,- };- std::cmp::min(proposed_value, model_context_length)- }-}diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator.rsindex ec16386df..9ce73b8f1 100644--- a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs@@ -942,7 +942,7 @@ impl<B: Backend> TraceValidator<B> {desired_suffix_length: usize,context: &mut LanguageModelGeneratorContext<B>,) {- let resolved_prefix_length = decoding_config.context_length.resolve(&context.model_config);+ let resolved_prefix_length = context.get_context_length(decoding_config);let current_suffix_length = std::cmp::max(decoding_config.prefill_step_size.resolve(&context.model_config),decoding_config.generate_suffix_length(),diff --git a/crates/benchmarks/src/runner/runner.rs b/crates/benchmarks/src/runner/runner.rsindex 7f4eea8cc..7b4b716dd 100644--- a/crates/benchmarks/src/runner/runner.rs+++ b/crates/benchmarks/src/runner/runner.rs@@ -8,7 +8,7 @@ use backend_uzu::{session::{ChatSession,config::{DecodingConfig, RunConfig},- parameter::{ContextLength, PrefillStepSize, SamplingMethod, SamplingPolicy},+ parameter::{PrefillStepSize, SamplingMethod, SamplingPolicy},types::{Input, Output},},};@@ -38,20 +38,6 @@ impl Runner {}}- fn minimal_context_length(&self) -> Result<usize, Box<dyn std::error::Error>> {- let input = Input::Messages(self.task.messages.clone());- let mut session = ChatSession::new(PathBuf::from(self.model_path.clone()), DecodingConfig::default())?;- let run_config = RunConfig::default().tokens_limit(1);- let output = session.run(- input.clone(),- run_config,- Some(|_: Output| {- return true;- }),- )?;- return Ok(output.stats.total_stats.tokens_count_input as usize + self.task.tokens_limit as usize);- }-pub fn run<F>(&self,mut progress: Option<F>,@@ -59,9 +45,7 @@ impl Runner {whereF: FnMut(f64),{- let context_length = self.minimal_context_length()?;-- let mut decoding_config = DecodingConfig::default().with_context_length(ContextLength::Custom(context_length));+ let mut decoding_config = DecodingConfig::default();if let Some(prefill_step_size) = self.prefill_step_size {decoding_config = decoding_config.with_prefill_step_size(PrefillStepSize::Custom(prefill_step_size));}