A high-performance inference engine for AI models
use mask only when it's needed, better kv update
LuckyIYI7dee120c54ec
diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rsindex ce1819a32..c222f944c 100644--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs@@ -144,6 +144,8 @@ impl EncodableBlock for Attention {(0, self.sliding_window_size)};+ let use_mask = window_length.is_some();+let sequence_length = segment_prefix_length + suffix_length;let gqa_factor = num_heads / num_groups;@@ -169,6 +171,7 @@ impl EncodableBlock for Attention {sequence_length,head_dim,self.is_causal,+ use_mask,)};@@ -180,7 +183,6 @@ impl EncodableBlock for Attention {let attention_output_binding =state.arrays(&[ArrayId::AttentionOutput]);- let attention_bias_map = attention_bias_binding[0].clone();let mask_kv_seq_stride = 1;let mask_q_seq_stride = sequence_length as i32;let mask_head_stride = 0;@@ -252,18 +254,25 @@ impl EncodableBlock for Attention {let attention_output_buffer =unsafe { attention_output_array.mtl_buffer() };- let attention_bias_buffer = attention_bias_map- .get(&window_length)- .map(|array| {- let mut array_ref = array.borrow_mut();- unsafe { array_ref.mtl_buffer().clone() }- })- .unwrap_or_else(|| {- panic!(- "Attention bias buffer not found for window length {:?}",- window_length- );- });+ let attention_bias_buffer = if use_mask {+ let attention_bias_map = attention_bias_binding[0].clone();+ Some(+ attention_bias_map+ .get(&window_length)+ .map(|array| {+ let mut array_ref = array.borrow_mut();+ unsafe { array_ref.mtl_buffer().clone() }+ })+ .unwrap_or_else(|| {+ panic!(+ "Attention bias buffer not found for window length {:?}",+ window_length+ );+ }),+ )+ } else {+ None+ };let partials_binding = state.arrays(&[ArrayId::AttentionPartials]);let sums_binding = state.arrays(&[ArrayId::AttentionSums]);@@ -321,7 +330,7 @@ impl EncodableBlock for Attention {keys_buffer: &key_cache_buffer,values_buffer: &value_cache_buffer,output_buffer: &attention_output_buffer,- mask_buffer: Some(&attention_bias_buffer),+ mask_buffer: attention_bias_buffer.as_ref(),sinks_buffer: sinks_buffer.as_ref(),num_heads,num_groups,@@ -350,7 +359,7 @@ impl EncodableBlock for Attention {v_head_stride,v_seq_stride,scale,- mask_buffer: Some(&attention_bias_buffer),+ mask_buffer: attention_bias_buffer.as_ref(),mask_kv_seq_stride,mask_q_seq_stride,mask_head_stride,@@ -385,7 +394,7 @@ impl EncodableBlock for Attention {v_head_stride,v_seq_stride,scale,- mask_buffer: Some(&attention_bias_buffer),+ mask_buffer: attention_bias_buffer.as_ref(),mask_kv_seq_stride,mask_q_seq_stride,mask_head_stride,diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention.metalindex 271b97341..e813e54f3 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention.metal@@ -817,36 +817,27 @@ void update_kv_cache(return;}- TensorView3D<const T> rotatedKeysTensorView =- TensorView3D<const T>(rotated_keys, num_groups, suffix_length, head_dim);- TensorView3D<T> keyCacheTensorView =- TensorView3D<T>(key_cache, num_groups, max_sequence_length, head_dim);- TensorView3D<T> valueCacheTensorView =- TensorView3D<T>(value_cache, num_groups, max_sequence_length, head_dim);+ const uint cacheTokenIndex = prefix_segment_length + tokenIndex;// Copy rotated key to cache- keyCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, dimIndex) =- rotatedKeysTensorView(groupIndex, tokenIndex, dimIndex);-- // Update value cache (only first thread in each token processes values to- // avoid redundant work)- if (dimIndex == 0) {- const uint totalQueryDim = num_heads * head_dim;- const uint totalKeyValueDim = num_groups * head_dim;-- const int qkvStride = totalQueryDim + 2 * totalKeyValueDim;- TensorView2D<const T> qkvTensorView =- TensorView2D<const T>(qkv, suffix_length, qkvStride);-- // Extract values from QKV tensor- // Values start at offset: total_query_dim + total_key_value_dim- for (uint d = 0; d < head_dim; d++) {- const uint valueOffset =- totalQueryDim + totalKeyValueDim + groupIndex * head_dim + d;- valueCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, d) =- qkvTensorView(tokenIndex, valueOffset);- }- }+ const uint rotatedKeyOffset =+ (groupIndex * suffix_length + tokenIndex) * head_dim + dimIndex;+ const uint keyCacheOffset =+ (groupIndex * max_sequence_length + cacheTokenIndex) * head_dim ++ dimIndex;+ key_cache[keyCacheOffset] = rotated_keys[rotatedKeyOffset];++ // Copy value to cache+ // qkv layout: [suffix_length, (num_heads + 2*num_groups) * head_dim]+ // Values start at offset: (num_heads * head_dim) + (num_groups * head_dim)+ const uint qkvStride = (num_heads + 2 * num_groups) * head_dim;+ const uint valueBaseOffset = (num_heads + num_groups) * head_dim;+ const uint valueOffset = tokenIndex * qkvStride + valueBaseOffset ++ groupIndex * head_dim + dimIndex;+ const uint valueCacheOffset =+ (groupIndex * max_sequence_length + cacheTokenIndex) * head_dim ++ dimIndex;+ value_cache[valueCacheOffset] = qkv[valueOffset];}#define outerArguments(T) \diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rsindex 2ee51879d..88f988a8c 100644--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs@@ -18,7 +18,7 @@ pub enum AttentionKernelVariant {TwoPass,}-type PipelineKey = (usize, bool, bool); // (head_dim, has_sinks, is_causal)+type PipelineKey = (usize, bool, bool, bool); // (head_dim, has_sinks, is_causal, has_mask)pub struct AttentionKernelPipelines {single_pass: HashMap<PipelineKey, MTLComputePipelineState>,@@ -123,15 +123,15 @@ pub struct AttentionGemmArguments<'a> {}fn make_function_constants(+ has_mask_value: bool,has_sinks_value: bool,is_causal_value: bool,) -> FunctionConstantValues {let function_constants = FunctionConstantValues::new();- let has_mask_value = true;let query_transposed_value = false;let bool_mask_value = false;- let float_mask_value = true;+ let float_mask_value = has_mask_value;function_constants.set_constant_value_at_index(&has_mask_value as *const bool as *const std::ffi::c_void,@@ -182,51 +182,66 @@ impl AttentionKernel {// Pre-generate all supported variants for sinks and causal configurationsfor &has_sinks_value in &[false, true] {for &is_causal_value in &[false, true] {- let function_constants =- make_function_constants(has_sinks_value, is_causal_value);-- for &head_dim in &supported_head_dims {- if let Ok((pipeline, _)) = context- .compute_pipeline_state_with_reflection(- &format!(- "attention_single_pass_{}_{}",- data_suffix, head_dim- ),- Some(&function_constants),- )- {- single_pass.insert(- (head_dim, has_sinks_value, is_causal_value),- pipeline,- );- }-- if let Ok((pipeline, _)) = context- .compute_pipeline_state_with_reflection(- &format!(- "attention_2pass_1_{}_{}",- data_suffix, head_dim- ),- Some(&function_constants),- )- {- two_pass_1.insert(- (head_dim, has_sinks_value, is_causal_value),- pipeline,- );- }+ for &has_mask_value in &[false, true] {+ let function_constants = make_function_constants(+ has_mask_value,+ has_sinks_value,+ is_causal_value,+ );++ for &head_dim in &supported_head_dims {+ if let Ok((pipeline, _)) = context+ .compute_pipeline_state_with_reflection(+ &format!(+ "attention_single_pass_{}_{}",+ data_suffix, head_dim+ ),+ Some(&function_constants),+ )+ {+ single_pass.insert(+ (+ head_dim,+ has_sinks_value,+ is_causal_value,+ has_mask_value,+ ),+ pipeline,+ );+ }- if !two_pass_2.contains_key(&head_dim) {if let Ok((pipeline, _)) = context.compute_pipeline_state_with_reflection(&format!(- "attention_2pass_2_{}_{}",+ "attention_2pass_1_{}_{}",data_suffix, head_dim),Some(&function_constants),){- two_pass_2.insert(head_dim, pipeline);+ two_pass_1.insert(+ (+ head_dim,+ has_sinks_value,+ is_causal_value,+ has_mask_value,+ ),+ pipeline,+ );+ }++ if !two_pass_2.contains_key(&head_dim) {+ if let Ok((pipeline, _)) = context+ .compute_pipeline_state_with_reflection(+ &format!(+ "attention_2pass_2_{}_{}",+ data_suffix, head_dim+ ),+ Some(&function_constants),+ )+ {+ two_pass_2.insert(head_dim, pipeline);+ }}}}@@ -256,16 +271,22 @@ impl AttentionKernel {&self,head_dim: usize,is_causal: bool,+ has_mask: bool,) -> bool {- self.pipelines.single_pass.contains_key(&(head_dim, false, is_causal))+ self.pipelines+ .single_pass+ .contains_key(&(head_dim, false, is_causal, has_mask))}pub fn supports_two_pass(&self,head_dim: usize,is_causal: bool,+ has_mask: bool,) -> bool {- self.pipelines.two_pass_1.contains_key(&(head_dim, false, is_causal))+ self.pipelines+ .two_pass_1+ .contains_key(&(head_dim, false, is_causal, has_mask))&& self.pipelines.two_pass_2.contains_key(&head_dim)}@@ -274,8 +295,10 @@ impl AttentionKernel {sequence_length: usize,head_dim: usize,is_causal: bool,+ has_mask: bool,) -> AttentionKernelVariant {- if self.supports_two_pass(head_dim, is_causal) && sequence_length > 1024+ if self.supports_two_pass(head_dim, is_causal, has_mask)+ && sequence_length > 1024{AttentionKernelVariant::TwoPass} else {@@ -289,10 +312,11 @@ impl AttentionKernel {args: AttentionSinglePassArguments,) -> Result<(), AttentionError> {let has_sinks = args.sinks_buffer.is_some();+ let has_mask = args.mask_buffer.is_some();let pipeline = self.pipelines.single_pass- .get(&(args.head_dim, has_sinks, args.is_causal))+ .get(&(args.head_dim, has_sinks, args.is_causal, has_mask)).ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;compute_encoder.set_compute_pipeline_state(pipeline);@@ -386,10 +410,11 @@ impl AttentionKernel {args: AttentionTwoPassArguments,) -> Result<(), AttentionError> {let has_sinks = args.sinks_buffer.is_some();+ let has_mask = args.mask_buffer.is_some();let pass1_pipeline = self.pipelines.two_pass_1- .get(&(args.head_dim, has_sinks, args.is_causal))+ .get(&(args.head_dim, has_sinks, args.is_causal, has_mask)).ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;let pass2_pipeline =@@ -567,12 +592,14 @@ impl AttentionKernel {depth: args.head_dim as u64,};+ let threadgroup_depth = std::cmp::min(args.head_dim.max(1), 64) as u64;+compute_encoder.dispatch_threads(threads_per_grid,MTLSize {width: 1,height: 1,- depth: 1,+ depth: threadgroup_depth,},);Ok(())