Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Use max context length for backends with sparse buffers (#438)

Aleksandr Golokoz571d76717c96
diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rs
index 5a0716400..111f6f5f3 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs
@@ -124,6 +124,8 @@ pub trait LanguageModelGeneratorTrait {
&mut self,
context: &dyn Any,
);
+
+ fn get_context_length(&self) -> usize;
}
impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
@@ -647,6 +649,10 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.tokens = ctx.tokens.clone();
self.registered_prefix_len = self.tokens.len().saturating_sub(1);
}
+
+ fn get_context_length(&self) -> usize {
+ self.context.get_context_length(&self.decoding_config)
+ }
}
impl<B: Backend> LanguageModelGenerator<B> {
diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
index adad1a3e5..d7f7e75e6 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
@@ -1,5 +1,6 @@
use std::{
cell::{Cell, RefCell},
+ cmp::min,
fs::File,
path::Path,
rc::Rc,
@@ -12,11 +13,12 @@ use crate::{
Backend, Context, Kernels,
kernel::{TokenCopySampledKernel, kv_cache_update::KVCacheUpdate},
},
- config::{LanguageModelConfig, ModelMetadata},
+ config::{DecoderConfig, LanguageModelConfig, ModelMetadata},
encodable_block::{Decoder, Sampling},
forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},
language_model::rng::PRng,
parameters::ParameterLoader,
+ prelude::ContextLength,
session::{
config::DecodingConfig,
parameter::{ConfigResolvableValue, ResolvableValue},
@@ -124,7 +126,8 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
let prefill_step_size = decoding_config.prefill_step_size.resolve(&model_metadata.model_config);
let generate_suffix_length = decoding_config.generate_suffix_length();
- let max_prefix_length: usize = decoding_config.context_length.resolve(&model_metadata.model_config);
+ let max_prefix_length: usize =
+ Self::get_context_length_internal(&context, &model_metadata.model_config.model_config, decoding_config);
let max_suffix_length: usize = std::cmp::max(prefill_step_size, generate_suffix_length);
let weights_path = model_path.join("model.safetensors");
@@ -166,7 +169,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
let seed = PRng::new(decoding_config.sampling_seed.resolve());
- let context = Self {
+ Ok(Self {
context,
cache_layers,
shared_buffers,
@@ -178,8 +181,35 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
seed,
token_copy_sampled,
async_buffers,
- };
+ })
+ }
+
+ pub fn get_context_length(
+ &self,
+ decoding_config: &DecodingConfig,
+ ) -> usize {
+ Self::get_context_length_internal(&self.context, &self.model_config.model_config, decoding_config)
+ }
- Ok(context)
+ fn get_context_length_internal(
+ context: &B::Context,
+ decoder_config: &DecoderConfig,
+ decoding_config: &DecodingConfig,
+ ) -> usize {
+ let model_length = decoder_config.transformer_config.context_length;
+ let proposed_value = match decoding_config.context_length {
+ ContextLength::Default => {
+ if context.sparse_buffers_supported() {
+ model_length
+ } else if cfg!(target_os = "ios") {
+ 8192
+ } else {
+ 16384
+ }
+ },
+ ContextLength::Maximal => model_length,
+ ContextLength::Custom(value) => value,
+ };
+ min(proposed_value, model_length)
}
}
diff --git a/crates/backend-uzu/src/session/chat_session.rs b/crates/backend-uzu/src/session/chat_session.rs
index 4aa0121a5..9987ba81a 100644
--- a/crates/backend-uzu/src/session/chat_session.rs
+++ b/crates/backend-uzu/src/session/chat_session.rs
@@ -231,7 +231,7 @@ impl ChatSession {
.collect();
let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- let context_length = self.decoding_config.context_length.resolve(&self.model_metadata.model_config);
+ let context_length = language_model_generator.get_context_length();
if tokens.len() >= context_length {
return Err(Error::ContextLengthExceeded);
}
diff --git a/crates/backend-uzu/src/session/parameter/context_length.rs b/crates/backend-uzu/src/session/parameter/context_length.rs
index 017e0d363..1de733c17 100644
--- a/crates/backend-uzu/src/session/parameter/context_length.rs
+++ b/crates/backend-uzu/src/session/parameter/context_length.rs
@@ -1,5 +1,3 @@
-use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};
-
#[derive(Debug, Clone, Copy)]
pub enum ContextLength {
Default,
@@ -12,24 +10,3 @@ impl Default for ContextLength {
ContextLength::Default
}
}
-
-impl ConfigResolvableValue<LanguageModelConfig, usize> for ContextLength {
- fn resolve(
- &self,
- config: &LanguageModelConfig,
- ) -> usize {
- let model_context_length = config.model_config.transformer_config.context_length;
- let proposed_value = match self {
- ContextLength::Default => {
- if cfg!(target_os = "ios") {
- 8192
- } else {
- 16384
- }
- },
- ContextLength::Maximal => model_context_length,
- ContextLength::Custom(value) => *value,
- };
- std::cmp::min(proposed_value, model_context_length)
- }
-}
diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
index ec16386df..9ce73b8f1 100644
--- a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
@@ -942,7 +942,7 @@ impl<B: Backend> TraceValidator<B> {
desired_suffix_length: usize,
context: &mut LanguageModelGeneratorContext<B>,
) {
- let resolved_prefix_length = decoding_config.context_length.resolve(&context.model_config);
+ let resolved_prefix_length = context.get_context_length(decoding_config);
let current_suffix_length = std::cmp::max(
decoding_config.prefill_step_size.resolve(&context.model_config),
decoding_config.generate_suffix_length(),
diff --git a/crates/benchmarks/src/runner/runner.rs b/crates/benchmarks/src/runner/runner.rs
index 7f4eea8cc..7b4b716dd 100644
--- a/crates/benchmarks/src/runner/runner.rs
+++ b/crates/benchmarks/src/runner/runner.rs
@@ -8,7 +8,7 @@ use backend_uzu::{
session::{
ChatSession,
config::{DecodingConfig, RunConfig},
- parameter::{ContextLength, PrefillStepSize, SamplingMethod, SamplingPolicy},
+ parameter::{PrefillStepSize, SamplingMethod, SamplingPolicy},
types::{Input, Output},
},
};
@@ -38,20 +38,6 @@ impl Runner {
}
}
- fn minimal_context_length(&self) -> Result<usize, Box<dyn std::error::Error>> {
- let input = Input::Messages(self.task.messages.clone());
- let mut session = ChatSession::new(PathBuf::from(self.model_path.clone()), DecodingConfig::default())?;
- let run_config = RunConfig::default().tokens_limit(1);
- let output = session.run(
- input.clone(),
- run_config,
- Some(|_: Output| {
- return true;
- }),
- )?;
- return Ok(output.stats.total_stats.tokens_count_input as usize + self.task.tokens_limit as usize);
- }
-
pub fn run<F>(
&self,
mut progress: Option<F>,
@@ -59,9 +45,7 @@ impl Runner {
where
F: FnMut(f64),
{
- let context_length = self.minimal_context_length()?;
-
- let mut decoding_config = DecodingConfig::default().with_context_length(ContextLength::Custom(context_length));
+ let mut decoding_config = DecodingConfig::default();
if let Some(prefill_step_size) = self.prefill_step_size {
decoding_config = decoding_config.with_prefill_step_size(PrefillStepSize::Custom(prefill_step_size));
}