A high-performance inference engine for AI models
Per-layer MLP hidden_dim + per-layer KV cache shape (#426)
CCYeh763255e7e7ec
diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rsindex a9effd686..5a2dfc8b5 100644--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs@@ -122,7 +122,7 @@ impl<B: Backend> ClassifierLayer<B> {let (mlp, mlp_hadamard_factors) = <dyn Mlp<B>>::new(&layer_config.mlp_config,transformer_config.model_dim,- transformer_config.hidden_dim,+ layer_config.hidden_dim.unwrap_or(transformer_config.hidden_dim),context,&layer_loader.subtree("mlp").unwrap(),)diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rsindex 308e64da8..07ed4e5bf 100644--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs@@ -263,7 +263,7 @@ impl<B: Backend> LayerExecutables<B> {let (mlp, mlp_input_hadamard_factors) = <dyn Mlp<B>>::new(&layer_config.mlp_config,transformer_config.model_dim,- transformer_config.hidden_dim,+ layer_config.hidden_dim.unwrap_or(transformer_config.hidden_dim),context,&decoder_layer_loader.subtree("mlp").unwrap(),)diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rsindex 94e314a36..7b904cd03 100644--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs@@ -103,18 +103,17 @@ impl<B: Backend> CacheLayers<B> {max_suffix_length: usize,) -> Self {let total_context_length = max_prefix_length.max(max_suffix_length);- let kv_shapes: Vec<[usize; 3]> =- model_shape.kv_cache_layer_shapes(max_prefix_length, max_suffix_length).collect();let mut data: Box<[CacheLayer<B>]> = model_shape.layer_mixers().iter()- .enumerate()- .map(|(layer_index, mixer)| match mixer {- MixerConfig::Attention(_) => {- let shape = kv_shapes[layer_index];- let window_length = model_shape.sliding_window_length_per_layer[layer_index]- .filter(|&window_size| window_size < total_context_length);+ .map(|mixer| match mixer {+ MixerConfig::Attention(attn) => {+ let sliding_window = attn.sliding_window_size;+ let length = sliding_window.unwrap_or(max_prefix_length);+ let shape = [length + max_suffix_length, attn.num_groups, attn.head_dim];+ let window_length =+ sliding_window.filter(|&window_size| window_size < total_context_length);let state = if let Some(w) = window_length {KVCacheLayerState::Windowed {diff --git a/crates/backend-uzu/src/forward_pass/model_shape.rs b/crates/backend-uzu/src/forward_pass/model_shape.rsindex 9fa30e4cc..0d2e95887 100644--- a/crates/backend-uzu/src/forward_pass/model_shape.rs+++ b/crates/backend-uzu/src/forward_pass/model_shape.rs@@ -13,9 +13,7 @@ pub struct ModelShape {context_length: usize,num_groups: usize,- head_dim: usize,pub num_layers: usize,- pub sliding_window_length_per_layer: Box<[Option<usize>]>,layer_mixers: Box<[MixerConfig]>,}@@ -25,12 +23,11 @@ impl ModelShape {let layer_configs = &tf.layer_configs;let num_layers = layer_configs.len();- let (num_heads, num_groups, head_dim) =- decoder_config.first_attention().map(|a| (a.num_heads, a.num_groups, a.head_dim)).unwrap_or_default();+ let (num_heads, num_groups) =+ decoder_config.first_attention().map(|a| (a.num_heads, a.num_groups)).unwrap_or_default();for attn in layer_configs.iter().filter_map(|l| l.mixer_config.as_attention()) {assert_eq!(attn.num_heads, num_heads, "attention layers must share num_heads");assert_eq!(attn.num_groups, num_groups, "attention layers must share num_groups");- assert_eq!(attn.head_dim, head_dim, "attention layers must share head_dim");}let activation_type: DataType = match &layer_configs[0].mlp_config {@@ -38,8 +35,6 @@ impl ModelShape {MLPConfig::MixtureOfExperts(m) => m.expert_config.linear_config.activation_precision().into(),};- let sliding_window_length_per_layer: Box<[Option<usize>]> =- layer_configs.iter().map(|l| l.mixer_config.sliding_window_size()).collect();let layer_mixers: Box<[MixerConfig]> = layer_configs.iter().map(|l| l.mixer_config.clone()).collect();Self {@@ -49,9 +44,7 @@ impl ModelShape {model_dim: tf.model_dim,context_length: tf.context_length,num_groups,- head_dim,num_layers,- sliding_window_length_per_layer,layer_mixers,}}@@ -105,17 +98,6 @@ impl ModelShape {[suffix_length, self.vocabulary_size]}- pub fn kv_cache_layer_shapes(- &self,- max_prefix_length: usize,- max_suffix_length: usize,- ) -> impl Iterator<Item = [usize; 3]> {- self.sliding_window_length_per_layer.iter().map(move |length| {- let length = length.unwrap_or(max_prefix_length);- [length + max_suffix_length, self.num_groups, self.head_dim]- })- }-pub fn context_length(&self) -> usize {self.context_length}