A high-performance inference engine for AI models
q-only attention (#482)
CCYeh44f8a55c214e
diff --git a/crates/backend-uzu/src/classifier/classifier_context.rs b/crates/backend-uzu/src/classifier/classifier_context.rsindex 9e1ded8d0..81c8492c0 100644--- a/crates/backend-uzu/src/classifier/classifier_context.rs+++ b/crates/backend-uzu/src/classifier/classifier_context.rs@@ -83,7 +83,7 @@ impl<B: Backend> ClassifierContext<B> {}let rope = Rc::new(- Rope::<B>::new(context.as_ref(), &model_shape)+ Rope::<B>::new(context.as_ref(), &model_shape, false).map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e))))?,);let qk_unpack = Rc::new(diff --git a/crates/backend-uzu/src/encodable_block/attention/mod.rs b/crates/backend-uzu/src/encodable_block/attention/mod.rsindex 221c3a614..05f8f0294 100644--- a/crates/backend-uzu/src/encodable_block/attention/mod.rs+++ b/crates/backend-uzu/src/encodable_block/attention/mod.rs@@ -99,15 +99,22 @@ impl<B: Backend> Attention<B> {rope: Rc<Rope<B>>,qk_unpack: Rc<QkUnpack<B>>,extract_input_hadamard: bool,+ is_kv_sharing: bool,) -> Result<(Self, Option<Allocation<B>>), AttentionError<B>> {let q_dim = config.num_heads * config.head_dim;let kv_dim = config.num_groups * config.head_dim;+ // KV-sharing layers reuse the source layer's K/V: project queries only, with no key/value norm.+ let (qkv_output_dim, key_norm_config, value_norm_config, kv_norm_groups) = if is_kv_sharing {+ (q_dim, None, None, 0)+ } else {+ (q_dim + 2 * kv_dim, config.key_norm_config.clone(), config.value_norm_config(), config.num_groups)+ };let qkv_projection_tree = parameter_tree.subtree("qkv_projection")?;let (qkv_projection, input_hadamard_factors) = if extract_input_hadamard {<dyn Linear<B>>::new_extracting_input_hadamard(model_dim,- [q_dim, kv_dim, kv_dim],+ [qkv_output_dim],config.has_qkv_biases,context,data_type,@@ -117,7 +124,7 @@ impl<B: Backend> Attention<B> {(<dyn Linear<B>>::new(model_dim,- [q_dim, kv_dim, kv_dim],+ [qkv_output_dim],config.has_qkv_biases,context,data_type,@@ -136,23 +143,22 @@ impl<B: Backend> Attention<B> {}).transpose()?;- let value_norm_config = config.value_norm_config();- let qkv_norm =- if config.query_norm_config.is_some() || config.key_norm_config.is_some() || value_norm_config.is_some() {- Some(QKVNorm::new(- context,- data_type,- config.query_norm_config.clone(),- config.key_norm_config.clone(),- value_norm_config,- parameter_tree,- config.num_heads,- config.num_groups,- config.head_dim,- )?)- } else {- None- };+ let qkv_norm = if config.query_norm_config.is_some() || key_norm_config.is_some() || value_norm_config.is_some()+ {+ Some(QKVNorm::new(+ context,+ data_type,+ config.query_norm_config.clone(),+ key_norm_config,+ value_norm_config,+ parameter_tree,+ config.num_heads,+ kv_norm_groups,+ config.head_dim,+ )?)+ } else {+ None+ };let out_projection_tree = parameter_tree.subtree("out_projection")?;let out_projection =@@ -361,7 +367,15 @@ impl<B: Backend> Attention<B> {encoder,)?,None => {- self.qk_unpack.encode(&qkv, suffix_length, self.num_heads, self.num_groups, self.head_dim, encoder)?+ let (queries, keys) = self.qk_unpack.encode(+ &qkv,+ suffix_length,+ self.num_heads,+ self.num_groups,+ self.head_dim,+ encoder,+ )?;+ (queries, Some(keys))},};let attention_output = self.encode_core(@@ -383,7 +397,7 @@ impl<B: Backend> Attention<B> {cache_access: Option<LayerCacheAccess<B>>,qkv: &Allocation<B>,queries: &Allocation<B>,- mut rotated_keys: Allocation<B>,+ mut rotated_keys: Option<Allocation<B>>,gate: Option<&Allocation<B>>,suffix_length: usize,encoder: &mut Encoder<B>,@@ -453,7 +467,7 @@ impl<B: Backend> Attention<B> {self.update_kv_cache_inplace_kernel.encode(None::<&Allocation<B>>,qkv,- &mut rotated_keys,+ rotated_keys.as_mut().expect("classifier attention requires rotated keys"),&mut values,self.num_groups as u32,self.num_heads as u32,@@ -528,7 +542,7 @@ impl<B: Backend> Attention<B> {let layer = entry.as_transformer_mut().expect("Attention layer expects transformer cache");if let Some(layer) = layer.as_any_mut().downcast_mut::<KVCacheLayer<B, B::SparseBuffer>>() {self.update_kv_cache_kernel.encode(- Some(&rotated_keys),+ rotated_keys.as_ref(),qkv,&mut layer.keys,&mut layer.values,@@ -543,7 +557,7 @@ impl<B: Backend> Attention<B> {encode_cached_attention!(&layer.keys, &layer.values)} else if let Some(layer) = layer.as_any_mut().downcast_mut::<KVCacheLayer<B, B::DenseBuffer>>() {self.update_kv_cache_kernel.encode(- Some(&rotated_keys),+ rotated_keys.as_ref(),qkv,&mut layer.keys,&mut layer.values,@@ -575,7 +589,8 @@ impl<B: Backend> Attention<B> {}} else {let values = extracted_values.as_ref().expect("Missing extracted values for classifier attention");- encode_cached_attention!(&rotated_keys, values)+ let rotated_keys = rotated_keys.as_ref().expect("classifier attention requires rotated keys");+ encode_cached_attention!(rotated_keys, values)};if let Some(gate_kernel) = &self.gate_kernel {diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rsindex f183ae3bc..168ae5e25 100644--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs@@ -79,6 +79,7 @@ impl<B: Backend> ClassifierLayer<B> {rope,qk_unpack,false,+ false,)?;if attention_hadamard_factors.is_some() {return Err(ClassifierLayerError::AttentionHadamardUnsupported);diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rsindex 2ff72a269..c64d6e70c 100644--- a/crates/backend-uzu/src/encodable_block/decoder.rs+++ b/crates/backend-uzu/src/encodable_block/decoder.rs@@ -152,7 +152,13 @@ impl<B: Backend> Decoder<B> {let decoder_weight_loader = root_weight_loader.subtree(transformer_subtree)?;let tf = &decoder_config.transformer_config;- let rope = Rc::new(Rope::<B>::new(context, model_shape).map_err(DecoderError::BackendError)?);+ let rope = Rc::new(Rope::<B>::new(context, model_shape, false).map_err(DecoderError::BackendError)?);+ // Built only when some layer projects queries only (reuses an earlier layer's KV cache).+ let rope_query_only = if tf.layer_configs.iter().any(|l| l.kv_source_layer_index.is_some()) {+ Some(Rc::new(Rope::<B>::new(context, model_shape, true).map_err(DecoderError::BackendError)?))+ } else {+ None+ };let qk_unpack =Rc::new(QkUnpack::<B>::new(context, model_shape.data_type).map_err(DecoderError::BackendError)?);@@ -162,6 +168,11 @@ impl<B: Backend> Decoder<B> {.enumerate().map(|(layer_index, layer_config)| {let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index))?;+ let layer_rope = if layer_config.kv_source_layer_index.is_some() {+ rope_query_only.as_ref().expect("query-only rope is built whenever a layer shares KV")+ } else {+ &rope+ };LayerExecutables::new(context,@@ -169,7 +180,7 @@ impl<B: Backend> Decoder<B> {layer_config,layer_index,&layer_loader,- &rope,+ layer_rope,&qk_unpack,model_shape.data_type,)diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rsindex 07bc6461e..b5981c917 100644--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs@@ -128,6 +128,7 @@ impl<B: Backend> LayerExecutables<B> {rope.clone(),qk_unpack.clone(),attention_config.gate_projection_config.is_none(),+ layer_config.kv_source_layer_index.is_some(),)?;(diff --git a/crates/backend-uzu/src/encodable_block/rope.rs b/crates/backend-uzu/src/encodable_block/rope.rsindex 82bde7618..472a4607d 100644--- a/crates/backend-uzu/src/encodable_block/rope.rs+++ b/crates/backend-uzu/src/encodable_block/rope.rs@@ -13,21 +13,24 @@ use crate::{pub struct Rope<B: Backend> {kernel: <B::Kernels as Kernels>::RopeKernel,data_type: DataType,+ query_only: bool,}impl<B: Backend> Rope<B> {pub fn new(context: &B::Context,model_shape: &ModelShape,+ query_only: bool,) -> Result<Self, B::Error> {Ok(Self {kernel: <B::Kernels as Kernels>::RopeKernel::new(context,model_shape.data_type,model_shape.rope_data_type,- false,+ query_only,)?,data_type: model_shape.data_type,+ query_only,})}@@ -44,22 +47,25 @@ impl<B: Backend> Rope<B> {rope_max_seq_len: usize,rope_dim: usize,encoder: &mut Encoder<B>,- ) -> Result<(Allocation<B>, Allocation<B>), B::Error> {+ ) -> Result<(Allocation<B>, Option<Allocation<B>>), B::Error> {let mut rotated_queries =encoder.allocate_scratch(size_for_shape(&[num_heads, suffix_length, head_dim], self.data_type))?;- let mut rotated_keys =- encoder.allocate_scratch(size_for_shape(&[num_groups, suffix_length, head_dim], self.data_type))?;+ let mut rotated_keys = if self.query_only {+ None+ } else {+ Some(encoder.allocate_scratch(size_for_shape(&[num_groups, suffix_length, head_dim], self.data_type))?)+ };self.kernel.encode(qkv,cosines,sines,token_positions,&mut rotated_queries,- Some(&mut rotated_keys),+ rotated_keys.as_mut(),head_dim as u32,rope_dim as u32,num_heads as u32,- Some(num_groups as u32),+ (!self.query_only).then_some(num_groups as u32),suffix_length as u32,rope_max_seq_len as u32,encoder,