Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

q-only attention (#482)

CCYeh44f8a55c214e
diff --git a/crates/backend-uzu/src/classifier/classifier_context.rs b/crates/backend-uzu/src/classifier/classifier_context.rs
index 9e1ded8d0..81c8492c0 100644
--- a/crates/backend-uzu/src/classifier/classifier_context.rs
+++ b/crates/backend-uzu/src/classifier/classifier_context.rs
@@ -83,7 +83,7 @@ impl<B: Backend> ClassifierContext<B> {
}
let rope = Rc::new(
- Rope::<B>::new(context.as_ref(), &model_shape)
+ Rope::<B>::new(context.as_ref(), &model_shape, false)
.map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e))))?,
);
let qk_unpack = Rc::new(
diff --git a/crates/backend-uzu/src/encodable_block/attention/mod.rs b/crates/backend-uzu/src/encodable_block/attention/mod.rs
index 221c3a614..05f8f0294 100644
--- a/crates/backend-uzu/src/encodable_block/attention/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/attention/mod.rs
@@ -99,15 +99,22 @@ impl<B: Backend> Attention<B> {
rope: Rc<Rope<B>>,
qk_unpack: Rc<QkUnpack<B>>,
extract_input_hadamard: bool,
+ is_kv_sharing: bool,
) -> Result<(Self, Option<Allocation<B>>), AttentionError<B>> {
let q_dim = config.num_heads * config.head_dim;
let kv_dim = config.num_groups * config.head_dim;
+ // KV-sharing layers reuse the source layer's K/V: project queries only, with no key/value norm.
+ let (qkv_output_dim, key_norm_config, value_norm_config, kv_norm_groups) = if is_kv_sharing {
+ (q_dim, None, None, 0)
+ } else {
+ (q_dim + 2 * kv_dim, config.key_norm_config.clone(), config.value_norm_config(), config.num_groups)
+ };
let qkv_projection_tree = parameter_tree.subtree("qkv_projection")?;
let (qkv_projection, input_hadamard_factors) = if extract_input_hadamard {
<dyn Linear<B>>::new_extracting_input_hadamard(
model_dim,
- [q_dim, kv_dim, kv_dim],
+ [qkv_output_dim],
config.has_qkv_biases,
context,
data_type,
@@ -117,7 +124,7 @@ impl<B: Backend> Attention<B> {
(
<dyn Linear<B>>::new(
model_dim,
- [q_dim, kv_dim, kv_dim],
+ [qkv_output_dim],
config.has_qkv_biases,
context,
data_type,
@@ -136,23 +143,22 @@ impl<B: Backend> Attention<B> {
})
.transpose()?;
- let value_norm_config = config.value_norm_config();
- let qkv_norm =
- if config.query_norm_config.is_some() || config.key_norm_config.is_some() || value_norm_config.is_some() {
- Some(QKVNorm::new(
- context,
- data_type,
- config.query_norm_config.clone(),
- config.key_norm_config.clone(),
- value_norm_config,
- parameter_tree,
- config.num_heads,
- config.num_groups,
- config.head_dim,
- )?)
- } else {
- None
- };
+ let qkv_norm = if config.query_norm_config.is_some() || key_norm_config.is_some() || value_norm_config.is_some()
+ {
+ Some(QKVNorm::new(
+ context,
+ data_type,
+ config.query_norm_config.clone(),
+ key_norm_config,
+ value_norm_config,
+ parameter_tree,
+ config.num_heads,
+ kv_norm_groups,
+ config.head_dim,
+ )?)
+ } else {
+ None
+ };
let out_projection_tree = parameter_tree.subtree("out_projection")?;
let out_projection =
@@ -361,7 +367,15 @@ impl<B: Backend> Attention<B> {
encoder,
)?,
None => {
- self.qk_unpack.encode(&qkv, suffix_length, self.num_heads, self.num_groups, self.head_dim, encoder)?
+ let (queries, keys) = self.qk_unpack.encode(
+ &qkv,
+ suffix_length,
+ self.num_heads,
+ self.num_groups,
+ self.head_dim,
+ encoder,
+ )?;
+ (queries, Some(keys))
},
};
let attention_output = self.encode_core(
@@ -383,7 +397,7 @@ impl<B: Backend> Attention<B> {
cache_access: Option<LayerCacheAccess<B>>,
qkv: &Allocation<B>,
queries: &Allocation<B>,
- mut rotated_keys: Allocation<B>,
+ mut rotated_keys: Option<Allocation<B>>,
gate: Option<&Allocation<B>>,
suffix_length: usize,
encoder: &mut Encoder<B>,
@@ -453,7 +467,7 @@ impl<B: Backend> Attention<B> {
self.update_kv_cache_inplace_kernel.encode(
None::<&Allocation<B>>,
qkv,
- &mut rotated_keys,
+ rotated_keys.as_mut().expect("classifier attention requires rotated keys"),
&mut values,
self.num_groups as u32,
self.num_heads as u32,
@@ -528,7 +542,7 @@ impl<B: Backend> Attention<B> {
let layer = entry.as_transformer_mut().expect("Attention layer expects transformer cache");
if let Some(layer) = layer.as_any_mut().downcast_mut::<KVCacheLayer<B, B::SparseBuffer>>() {
self.update_kv_cache_kernel.encode(
- Some(&rotated_keys),
+ rotated_keys.as_ref(),
qkv,
&mut layer.keys,
&mut layer.values,
@@ -543,7 +557,7 @@ impl<B: Backend> Attention<B> {
encode_cached_attention!(&layer.keys, &layer.values)
} else if let Some(layer) = layer.as_any_mut().downcast_mut::<KVCacheLayer<B, B::DenseBuffer>>() {
self.update_kv_cache_kernel.encode(
- Some(&rotated_keys),
+ rotated_keys.as_ref(),
qkv,
&mut layer.keys,
&mut layer.values,
@@ -575,7 +589,8 @@ impl<B: Backend> Attention<B> {
}
} else {
let values = extracted_values.as_ref().expect("Missing extracted values for classifier attention");
- encode_cached_attention!(&rotated_keys, values)
+ let rotated_keys = rotated_keys.as_ref().expect("classifier attention requires rotated keys");
+ encode_cached_attention!(rotated_keys, values)
};
if let Some(gate_kernel) = &self.gate_kernel {
diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
index f183ae3bc..168ae5e25 100644
--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
@@ -79,6 +79,7 @@ impl<B: Backend> ClassifierLayer<B> {
rope,
qk_unpack,
false,
+ false,
)?;
if attention_hadamard_factors.is_some() {
return Err(ClassifierLayerError::AttentionHadamardUnsupported);
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index 2ff72a269..c64d6e70c 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -152,7 +152,13 @@ impl<B: Backend> Decoder<B> {
let decoder_weight_loader = root_weight_loader.subtree(transformer_subtree)?;
let tf = &decoder_config.transformer_config;
- let rope = Rc::new(Rope::<B>::new(context, model_shape).map_err(DecoderError::BackendError)?);
+ let rope = Rc::new(Rope::<B>::new(context, model_shape, false).map_err(DecoderError::BackendError)?);
+ // Built only when some layer projects queries only (reuses an earlier layer's KV cache).
+ let rope_query_only = if tf.layer_configs.iter().any(|l| l.kv_source_layer_index.is_some()) {
+ Some(Rc::new(Rope::<B>::new(context, model_shape, true).map_err(DecoderError::BackendError)?))
+ } else {
+ None
+ };
let qk_unpack =
Rc::new(QkUnpack::<B>::new(context, model_shape.data_type).map_err(DecoderError::BackendError)?);
@@ -162,6 +168,11 @@ impl<B: Backend> Decoder<B> {
.enumerate()
.map(|(layer_index, layer_config)| {
let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index))?;
+ let layer_rope = if layer_config.kv_source_layer_index.is_some() {
+ rope_query_only.as_ref().expect("query-only rope is built whenever a layer shares KV")
+ } else {
+ &rope
+ };
LayerExecutables::new(
context,
@@ -169,7 +180,7 @@ impl<B: Backend> Decoder<B> {
layer_config,
layer_index,
&layer_loader,
- &rope,
+ layer_rope,
&qk_unpack,
model_shape.data_type,
)
diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rs
index 07bc6461e..b5981c917 100644
--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs
@@ -128,6 +128,7 @@ impl<B: Backend> LayerExecutables<B> {
rope.clone(),
qk_unpack.clone(),
attention_config.gate_projection_config.is_none(),
+ layer_config.kv_source_layer_index.is_some(),
)?;
(
diff --git a/crates/backend-uzu/src/encodable_block/rope.rs b/crates/backend-uzu/src/encodable_block/rope.rs
index 82bde7618..472a4607d 100644
--- a/crates/backend-uzu/src/encodable_block/rope.rs
+++ b/crates/backend-uzu/src/encodable_block/rope.rs
@@ -13,21 +13,24 @@ use crate::{
pub struct Rope<B: Backend> {
kernel: <B::Kernels as Kernels>::RopeKernel,
data_type: DataType,
+ query_only: bool,
}
impl<B: Backend> Rope<B> {
pub fn new(
context: &B::Context,
model_shape: &ModelShape,
+ query_only: bool,
) -> Result<Self, B::Error> {
Ok(Self {
kernel: <B::Kernels as Kernels>::RopeKernel::new(
context,
model_shape.data_type,
model_shape.rope_data_type,
- false,
+ query_only,
)?,
data_type: model_shape.data_type,
+ query_only,
})
}
@@ -44,22 +47,25 @@ impl<B: Backend> Rope<B> {
rope_max_seq_len: usize,
rope_dim: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(Allocation<B>, Allocation<B>), B::Error> {
+ ) -> Result<(Allocation<B>, Option<Allocation<B>>), B::Error> {
let mut rotated_queries =
encoder.allocate_scratch(size_for_shape(&[num_heads, suffix_length, head_dim], self.data_type))?;
- let mut rotated_keys =
- encoder.allocate_scratch(size_for_shape(&[num_groups, suffix_length, head_dim], self.data_type))?;
+ let mut rotated_keys = if self.query_only {
+ None
+ } else {
+ Some(encoder.allocate_scratch(size_for_shape(&[num_groups, suffix_length, head_dim], self.data_type))?)
+ };
self.kernel.encode(
qkv,
cosines,
sines,
token_positions,
&mut rotated_queries,
- Some(&mut rotated_keys),
+ rotated_keys.as_mut(),
head_dim as u32,
rope_dim as u32,
num_heads as u32,
- Some(num_groups as u32),
+ (!self.query_only).then_some(num_groups as u32),
suffix_length as u32,
rope_max_seq_len as u32,
encoder,