Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

use mask only when it's needed, better kv update

LuckyIYI7dee120c54ec
diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rs
index ce1819a32..c222f944c 100644
--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs
@@ -144,6 +144,8 @@ impl EncodableBlock for Attention {
(0, self.sliding_window_size)
};
+ let use_mask = window_length.is_some();
+
let sequence_length = segment_prefix_length + suffix_length;
let gqa_factor = num_heads / num_groups;
@@ -169,6 +171,7 @@ impl EncodableBlock for Attention {
sequence_length,
head_dim,
self.is_causal,
+ use_mask,
)
};
@@ -180,7 +183,6 @@ impl EncodableBlock for Attention {
let attention_output_binding =
state.arrays(&[ArrayId::AttentionOutput]);
- let attention_bias_map = attention_bias_binding[0].clone();
let mask_kv_seq_stride = 1;
let mask_q_seq_stride = sequence_length as i32;
let mask_head_stride = 0;
@@ -252,18 +254,25 @@ impl EncodableBlock for Attention {
let attention_output_buffer =
unsafe { attention_output_array.mtl_buffer() };
- let attention_bias_buffer = attention_bias_map
- .get(&window_length)
- .map(|array| {
- let mut array_ref = array.borrow_mut();
- unsafe { array_ref.mtl_buffer().clone() }
- })
- .unwrap_or_else(|| {
- panic!(
- "Attention bias buffer not found for window length {:?}",
- window_length
- );
- });
+ let attention_bias_buffer = if use_mask {
+ let attention_bias_map = attention_bias_binding[0].clone();
+ Some(
+ attention_bias_map
+ .get(&window_length)
+ .map(|array| {
+ let mut array_ref = array.borrow_mut();
+ unsafe { array_ref.mtl_buffer().clone() }
+ })
+ .unwrap_or_else(|| {
+ panic!(
+ "Attention bias buffer not found for window length {:?}",
+ window_length
+ );
+ }),
+ )
+ } else {
+ None
+ };
let partials_binding = state.arrays(&[ArrayId::AttentionPartials]);
let sums_binding = state.arrays(&[ArrayId::AttentionSums]);
@@ -321,7 +330,7 @@ impl EncodableBlock for Attention {
keys_buffer: &key_cache_buffer,
values_buffer: &value_cache_buffer,
output_buffer: &attention_output_buffer,
- mask_buffer: Some(&attention_bias_buffer),
+ mask_buffer: attention_bias_buffer.as_ref(),
sinks_buffer: sinks_buffer.as_ref(),
num_heads,
num_groups,
@@ -350,7 +359,7 @@ impl EncodableBlock for Attention {
v_head_stride,
v_seq_stride,
scale,
- mask_buffer: Some(&attention_bias_buffer),
+ mask_buffer: attention_bias_buffer.as_ref(),
mask_kv_seq_stride,
mask_q_seq_stride,
mask_head_stride,
@@ -385,7 +394,7 @@ impl EncodableBlock for Attention {
v_head_stride,
v_seq_stride,
scale,
- mask_buffer: Some(&attention_bias_buffer),
+ mask_buffer: attention_bias_buffer.as_ref(),
mask_kv_seq_stride,
mask_q_seq_stride,
mask_head_stride,
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention.metal
index 271b97341..e813e54f3 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention.metal
@@ -817,36 +817,27 @@ void update_kv_cache(
return;
}
- TensorView3D<const T> rotatedKeysTensorView =
- TensorView3D<const T>(rotated_keys, num_groups, suffix_length, head_dim);
- TensorView3D<T> keyCacheTensorView =
- TensorView3D<T>(key_cache, num_groups, max_sequence_length, head_dim);
- TensorView3D<T> valueCacheTensorView =
- TensorView3D<T>(value_cache, num_groups, max_sequence_length, head_dim);
+ const uint cacheTokenIndex = prefix_segment_length + tokenIndex;
// Copy rotated key to cache
- keyCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, dimIndex) =
- rotatedKeysTensorView(groupIndex, tokenIndex, dimIndex);
-
- // Update value cache (only first thread in each token processes values to
- // avoid redundant work)
- if (dimIndex == 0) {
- const uint totalQueryDim = num_heads * head_dim;
- const uint totalKeyValueDim = num_groups * head_dim;
-
- const int qkvStride = totalQueryDim + 2 * totalKeyValueDim;
- TensorView2D<const T> qkvTensorView =
- TensorView2D<const T>(qkv, suffix_length, qkvStride);
-
- // Extract values from QKV tensor
- // Values start at offset: total_query_dim + total_key_value_dim
- for (uint d = 0; d < head_dim; d++) {
- const uint valueOffset =
- totalQueryDim + totalKeyValueDim + groupIndex * head_dim + d;
- valueCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, d) =
- qkvTensorView(tokenIndex, valueOffset);
- }
- }
+ const uint rotatedKeyOffset =
+ (groupIndex * suffix_length + tokenIndex) * head_dim + dimIndex;
+ const uint keyCacheOffset =
+ (groupIndex * max_sequence_length + cacheTokenIndex) * head_dim +
+ dimIndex;
+ key_cache[keyCacheOffset] = rotated_keys[rotatedKeyOffset];
+
+ // Copy value to cache
+ // qkv layout: [suffix_length, (num_heads + 2*num_groups) * head_dim]
+ // Values start at offset: (num_heads * head_dim) + (num_groups * head_dim)
+ const uint qkvStride = (num_heads + 2 * num_groups) * head_dim;
+ const uint valueBaseOffset = (num_heads + num_groups) * head_dim;
+ const uint valueOffset = tokenIndex * qkvStride + valueBaseOffset +
+ groupIndex * head_dim + dimIndex;
+ const uint valueCacheOffset =
+ (groupIndex * max_sequence_length + cacheTokenIndex) * head_dim +
+ dimIndex;
+ value_cache[valueCacheOffset] = qkv[valueOffset];
}
#define outerArguments(T) \
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
index 2ee51879d..88f988a8c 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
@@ -18,7 +18,7 @@ pub enum AttentionKernelVariant {
TwoPass,
}
-type PipelineKey = (usize, bool, bool); // (head_dim, has_sinks, is_causal)
+type PipelineKey = (usize, bool, bool, bool); // (head_dim, has_sinks, is_causal, has_mask)
pub struct AttentionKernelPipelines {
single_pass: HashMap<PipelineKey, MTLComputePipelineState>,
@@ -123,15 +123,15 @@ pub struct AttentionGemmArguments<'a> {
}
fn make_function_constants(
+ has_mask_value: bool,
has_sinks_value: bool,
is_causal_value: bool,
) -> FunctionConstantValues {
let function_constants = FunctionConstantValues::new();
- let has_mask_value = true;
let query_transposed_value = false;
let bool_mask_value = false;
- let float_mask_value = true;
+ let float_mask_value = has_mask_value;
function_constants.set_constant_value_at_index(
&has_mask_value as *const bool as *const std::ffi::c_void,
@@ -182,51 +182,66 @@ impl AttentionKernel {
// Pre-generate all supported variants for sinks and causal configurations
for &has_sinks_value in &[false, true] {
for &is_causal_value in &[false, true] {
- let function_constants =
- make_function_constants(has_sinks_value, is_causal_value);
-
- for &head_dim in &supported_head_dims {
- if let Ok((pipeline, _)) = context
- .compute_pipeline_state_with_reflection(
- &format!(
- "attention_single_pass_{}_{}",
- data_suffix, head_dim
- ),
- Some(&function_constants),
- )
- {
- single_pass.insert(
- (head_dim, has_sinks_value, is_causal_value),
- pipeline,
- );
- }
-
- if let Ok((pipeline, _)) = context
- .compute_pipeline_state_with_reflection(
- &format!(
- "attention_2pass_1_{}_{}",
- data_suffix, head_dim
- ),
- Some(&function_constants),
- )
- {
- two_pass_1.insert(
- (head_dim, has_sinks_value, is_causal_value),
- pipeline,
- );
- }
+ for &has_mask_value in &[false, true] {
+ let function_constants = make_function_constants(
+ has_mask_value,
+ has_sinks_value,
+ is_causal_value,
+ );
+
+ for &head_dim in &supported_head_dims {
+ if let Ok((pipeline, _)) = context
+ .compute_pipeline_state_with_reflection(
+ &format!(
+ "attention_single_pass_{}_{}",
+ data_suffix, head_dim
+ ),
+ Some(&function_constants),
+ )
+ {
+ single_pass.insert(
+ (
+ head_dim,
+ has_sinks_value,
+ is_causal_value,
+ has_mask_value,
+ ),
+ pipeline,
+ );
+ }
- if !two_pass_2.contains_key(&head_dim) {
if let Ok((pipeline, _)) = context
.compute_pipeline_state_with_reflection(
&format!(
- "attention_2pass_2_{}_{}",
+ "attention_2pass_1_{}_{}",
data_suffix, head_dim
),
Some(&function_constants),
)
{
- two_pass_2.insert(head_dim, pipeline);
+ two_pass_1.insert(
+ (
+ head_dim,
+ has_sinks_value,
+ is_causal_value,
+ has_mask_value,
+ ),
+ pipeline,
+ );
+ }
+
+ if !two_pass_2.contains_key(&head_dim) {
+ if let Ok((pipeline, _)) = context
+ .compute_pipeline_state_with_reflection(
+ &format!(
+ "attention_2pass_2_{}_{}",
+ data_suffix, head_dim
+ ),
+ Some(&function_constants),
+ )
+ {
+ two_pass_2.insert(head_dim, pipeline);
+ }
}
}
}
@@ -256,16 +271,22 @@ impl AttentionKernel {
&self,
head_dim: usize,
is_causal: bool,
+ has_mask: bool,
) -> bool {
- self.pipelines.single_pass.contains_key(&(head_dim, false, is_causal))
+ self.pipelines
+ .single_pass
+ .contains_key(&(head_dim, false, is_causal, has_mask))
}
pub fn supports_two_pass(
&self,
head_dim: usize,
is_causal: bool,
+ has_mask: bool,
) -> bool {
- self.pipelines.two_pass_1.contains_key(&(head_dim, false, is_causal))
+ self.pipelines
+ .two_pass_1
+ .contains_key(&(head_dim, false, is_causal, has_mask))
&& self.pipelines.two_pass_2.contains_key(&head_dim)
}
@@ -274,8 +295,10 @@ impl AttentionKernel {
sequence_length: usize,
head_dim: usize,
is_causal: bool,
+ has_mask: bool,
) -> AttentionKernelVariant {
- if self.supports_two_pass(head_dim, is_causal) && sequence_length > 1024
+ if self.supports_two_pass(head_dim, is_causal, has_mask)
+ && sequence_length > 1024
{
AttentionKernelVariant::TwoPass
} else {
@@ -289,10 +312,11 @@ impl AttentionKernel {
args: AttentionSinglePassArguments,
) -> Result<(), AttentionError> {
let has_sinks = args.sinks_buffer.is_some();
+ let has_mask = args.mask_buffer.is_some();
let pipeline = self
.pipelines
.single_pass
- .get(&(args.head_dim, has_sinks, args.is_causal))
+ .get(&(args.head_dim, has_sinks, args.is_causal, has_mask))
.ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;
compute_encoder.set_compute_pipeline_state(pipeline);
@@ -386,10 +410,11 @@ impl AttentionKernel {
args: AttentionTwoPassArguments,
) -> Result<(), AttentionError> {
let has_sinks = args.sinks_buffer.is_some();
+ let has_mask = args.mask_buffer.is_some();
let pass1_pipeline = self
.pipelines
.two_pass_1
- .get(&(args.head_dim, has_sinks, args.is_causal))
+ .get(&(args.head_dim, has_sinks, args.is_causal, has_mask))
.ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;
let pass2_pipeline =
@@ -567,12 +592,14 @@ impl AttentionKernel {
depth: args.head_dim as u64,
};
+ let threadgroup_depth = std::cmp::min(args.head_dim.max(1), 64) as u64;
+
compute_encoder.dispatch_threads(
threads_per_grid,
MTLSize {
width: 1,
height: 1,
- depth: 1,
+ depth: threadgroup_depth,
},
);
Ok(())