Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Per-layer MLP hidden_dim + per-layer KV cache shape (#426)

CCYeh763255e7e7ec
diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
index a9effd686..5a2dfc8b5 100644
--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
@@ -122,7 +122,7 @@ impl<B: Backend> ClassifierLayer<B> {
let (mlp, mlp_hadamard_factors) = <dyn Mlp<B>>::new(
&layer_config.mlp_config,
transformer_config.model_dim,
- transformer_config.hidden_dim,
+ layer_config.hidden_dim.unwrap_or(transformer_config.hidden_dim),
context,
&layer_loader.subtree("mlp").unwrap(),
)
diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rs
index 308e64da8..07ed4e5bf 100644
--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs
@@ -263,7 +263,7 @@ impl<B: Backend> LayerExecutables<B> {
let (mlp, mlp_input_hadamard_factors) = <dyn Mlp<B>>::new(
&layer_config.mlp_config,
transformer_config.model_dim,
- transformer_config.hidden_dim,
+ layer_config.hidden_dim.unwrap_or(transformer_config.hidden_dim),
context,
&decoder_layer_loader.subtree("mlp").unwrap(),
)
diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rs
index 94e314a36..7b904cd03 100644
--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs
+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs
@@ -103,18 +103,17 @@ impl<B: Backend> CacheLayers<B> {
max_suffix_length: usize,
) -> Self {
let total_context_length = max_prefix_length.max(max_suffix_length);
- let kv_shapes: Vec<[usize; 3]> =
- model_shape.kv_cache_layer_shapes(max_prefix_length, max_suffix_length).collect();
let mut data: Box<[CacheLayer<B>]> = model_shape
.layer_mixers()
.iter()
- .enumerate()
- .map(|(layer_index, mixer)| match mixer {
- MixerConfig::Attention(_) => {
- let shape = kv_shapes[layer_index];
- let window_length = model_shape.sliding_window_length_per_layer[layer_index]
- .filter(|&window_size| window_size < total_context_length);
+ .map(|mixer| match mixer {
+ MixerConfig::Attention(attn) => {
+ let sliding_window = attn.sliding_window_size;
+ let length = sliding_window.unwrap_or(max_prefix_length);
+ let shape = [length + max_suffix_length, attn.num_groups, attn.head_dim];
+ let window_length =
+ sliding_window.filter(|&window_size| window_size < total_context_length);
let state = if let Some(w) = window_length {
KVCacheLayerState::Windowed {
diff --git a/crates/backend-uzu/src/forward_pass/model_shape.rs b/crates/backend-uzu/src/forward_pass/model_shape.rs
index 9fa30e4cc..0d2e95887 100644
--- a/crates/backend-uzu/src/forward_pass/model_shape.rs
+++ b/crates/backend-uzu/src/forward_pass/model_shape.rs
@@ -13,9 +13,7 @@ pub struct ModelShape {
context_length: usize,
num_groups: usize,
- head_dim: usize,
pub num_layers: usize,
- pub sliding_window_length_per_layer: Box<[Option<usize>]>,
layer_mixers: Box<[MixerConfig]>,
}
@@ -25,12 +23,11 @@ impl ModelShape {
let layer_configs = &tf.layer_configs;
let num_layers = layer_configs.len();
- let (num_heads, num_groups, head_dim) =
- decoder_config.first_attention().map(|a| (a.num_heads, a.num_groups, a.head_dim)).unwrap_or_default();
+ let (num_heads, num_groups) =
+ decoder_config.first_attention().map(|a| (a.num_heads, a.num_groups)).unwrap_or_default();
for attn in layer_configs.iter().filter_map(|l| l.mixer_config.as_attention()) {
assert_eq!(attn.num_heads, num_heads, "attention layers must share num_heads");
assert_eq!(attn.num_groups, num_groups, "attention layers must share num_groups");
- assert_eq!(attn.head_dim, head_dim, "attention layers must share head_dim");
}
let activation_type: DataType = match &layer_configs[0].mlp_config {
@@ -38,8 +35,6 @@ impl ModelShape {
MLPConfig::MixtureOfExperts(m) => m.expert_config.linear_config.activation_precision().into(),
};
- let sliding_window_length_per_layer: Box<[Option<usize>]> =
- layer_configs.iter().map(|l| l.mixer_config.sliding_window_size()).collect();
let layer_mixers: Box<[MixerConfig]> = layer_configs.iter().map(|l| l.mixer_config.clone()).collect();
Self {
@@ -49,9 +44,7 @@ impl ModelShape {
model_dim: tf.model_dim,
context_length: tf.context_length,
num_groups,
- head_dim,
num_layers,
- sliding_window_length_per_layer,
layer_mixers,
}
}
@@ -105,17 +98,6 @@ impl ModelShape {
[suffix_length, self.vocabulary_size]
}
- pub fn kv_cache_layer_shapes(
- &self,
- max_prefix_length: usize,
- max_suffix_length: usize,
- ) -> impl Iterator<Item = [usize; 3]> {
- self.sliding_window_length_per_layer.iter().map(move |length| {
- let length = length.unwrap_or(max_prefix_length);
- [length + max_suffix_length, self.num_groups, self.head_dim]
- })
- }
-
pub fn context_length(&self) -> usize {
self.context_length
}