Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

dslify-attention (#193)

Aleksandr Golokoz2500d54acf50
diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rs
index 532afecfc..bac0a64ec 100644
--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs
@@ -1,15 +1,25 @@
//! Attention kernel encodable.
+use std::collections::HashMap;
+
use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};
use objc2::{rc::Retained, runtime::ProtocolObject};
use crate::{
DataType,
- backends::metal::{
- Metal, MetalContext,
- kernel::attention::{
- AttentionError, AttentionGemmArguments, AttentionKernel, AttentionKernelVariant,
- AttentionSinglePassArguments, AttentionTwoPassArguments, KVCacheUpdateArguments,
+ backends::{
+ common::kernel::{
+ AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,
+ },
+ metal::{
+ Metal, MetalContext, MetalError,
+ kernel::{
+ attention::{AttentionGemmArguments, AttentionGemmBlock},
+ dsl::{
+ AttentionSinglePassMetalKernel, AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel,
+ AttentionUpdateKVCacheMetalKernel,
+ },
+ },
},
},
encodable_block::{EncodableBlock, EncodingParameters},
@@ -33,7 +43,11 @@ fn env_gemm_attention_enabled() -> bool {
}
pub struct Attention {
- kernel: AttentionKernel,
+ single_pass_kernels: HashMap<KernelKey, AttentionSinglePassMetalKernel>,
+ two_pass_1_kernels: HashMap<KernelKey, AttentionTwoPass1MetalKernel>,
+ two_pass_2_kernels: HashMap<u32, AttentionTwoPass2MetalKernel>,
+ update_kv_cache_kernel: AttentionUpdateKVCacheMetalKernel,
+ gemm_block: AttentionGemmBlock,
layer_index: usize,
attention_scale: Option<f32>,
has_sinks: bool,
@@ -50,10 +64,44 @@ impl Attention {
has_sinks: bool,
is_causal: bool,
sliding_window_size: Option<usize>,
- ) -> Result<Self, AttentionError> {
- let kernel = AttentionKernel::new(context, data_type)?;
+ ) -> Result<Self, MetalError> {
+ let mut single_pass_kernels = HashMap::new();
+ let mut two_pass_1_kernels = HashMap::new();
+ let mut two_pass_2_kernels = HashMap::new();
+
+ let supported_head_dims = [64u32, 128u32, 256u32];
+ for head_dim in supported_head_dims {
+ for has_mask in [false, true] {
+ let key = KernelKey {
+ head_dim,
+ has_mask,
+ };
+ let float_mask = has_mask;
+
+ let sp_kernel = AttentionSinglePassMetalKernel::new(
+ context, data_type, head_dim, float_mask, has_mask, has_sinks, is_causal,
+ )?;
+ single_pass_kernels.insert(key, sp_kernel);
+
+ let tp1_kernel = AttentionTwoPass1MetalKernel::new(
+ context, data_type, head_dim, float_mask, has_mask, has_sinks, is_causal,
+ )?;
+ two_pass_1_kernels.insert(key, tp1_kernel);
+
+ let tp2_kernel = AttentionTwoPass2MetalKernel::new(context, data_type, head_dim)?;
+ two_pass_2_kernels.insert(head_dim, tp2_kernel);
+ }
+ }
+
+ let update_kv_cache_kernel = AttentionUpdateKVCacheMetalKernel::new(context, data_type)?;
+ let gemm_block = AttentionGemmBlock::new(data_type);
+
Ok(Self {
- kernel,
+ single_pass_kernels,
+ two_pass_1_kernels,
+ two_pass_2_kernels,
+ update_kv_cache_kernel,
+ gemm_block,
layer_index,
attention_scale,
has_sinks,
@@ -61,6 +109,33 @@ impl Attention {
sliding_window_size,
})
}
+
+ fn select_variant(
+ &self,
+ gemm_enabled: bool,
+ suffix_length: usize,
+ head_dim: usize,
+ sequence_length: usize,
+ use_mask: bool,
+ ) -> KernelVariant {
+ let use_gemm = gemm_enabled && suffix_length > 8 && matches!(head_dim, 64 | 128 | 256);
+ if use_gemm {
+ return KernelVariant::Gemm;
+ }
+
+ let kernel_key = KernelKey {
+ head_dim: head_dim as u32,
+ has_mask: use_mask,
+ };
+ if sequence_length > 1024
+ && self.two_pass_1_kernels.contains_key(&kernel_key)
+ && self.two_pass_2_kernels.contains_key(&(head_dim as u32))
+ {
+ return KernelVariant::TwoPass;
+ }
+
+ KernelVariant::SinglePass
+ }
}
impl EncodableBlock<Metal> for Attention {
@@ -136,18 +211,13 @@ impl EncodableBlock<Metal> for Attention {
let scale = self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());
let gemm_enabled = env_gemm_attention_enabled();
- let use_gemm = gemm_enabled && suffix_length > 8 && matches!(head_dim, 64 | 128 | 256);
if !gemm_enabled {
static PRINT_ONCE: std::sync::Once = std::sync::Once::new();
PRINT_ONCE.call_once(|| {
eprintln!("[uzu] Gemm attention disabled via UZU_USE_GEMM_ATTENTION");
});
}
- let variant = if use_gemm {
- AttentionKernelVariant::SinglePass
- } else {
- self.kernel.choose_variant(sequence_length, head_dim, self.is_causal, use_mask)
- };
+ let variant = self.select_variant(gemm_enabled, suffix_length, head_dim, sequence_length, use_mask);
let rotated_queries_binding = state.arrays(&[ArrayId::RotatedQueries]);
let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
@@ -191,23 +261,20 @@ impl EncodableBlock<Metal> for Attention {
let extracted_values_buf = extracted_values_array.buffer().clone();
// Reuse the KV cache update kernel to write values into extracted_values_buf.
- if let Err(e) = self.kernel.encode_kv_cache_update(
+ self.update_kv_cache_kernel.encode(
+ rotated_keys_buffer,
+ qkv_buffer,
+ // keys already in desired layout; harmless overwrite
+ rotated_keys_buffer,
+ &extracted_values_buf,
+ num_groups as u32,
+ num_heads as u32,
+ head_dim as u32,
+ suffix_length as u32,
+ 0u32,
+ max_sequence_length as u32,
&compute_encoder,
- KVCacheUpdateArguments {
- rotated_keys_buffer: &rotated_keys_buffer,
- qkv_buffer: &qkv_buffer,
- key_cache_buffer: &rotated_keys_buffer, // keys already in desired layout; harmless overwrite
- value_cache_buffer: &extracted_values_buf,
- num_groups,
- num_heads,
- head_dim,
- suffix_length,
- segment_prefix_length: 0,
- max_sequence_length,
- },
- ) {
- eprintln!("Failed to prepare rotated values buffer: {:?}", e);
- }
+ );
(rotated_keys_buffer.clone(), extracted_values_buf)
};
@@ -252,24 +319,19 @@ impl EncodableBlock<Metal> for Attention {
// Only update KV cache for LLM mode (not for classifiers)
if has_kv_cache {
- if let Err(e) = self.kernel.encode_kv_cache_update(
- compute_encoder,
- KVCacheUpdateArguments {
- rotated_keys_buffer: &rotated_keys_buffer,
- qkv_buffer: &qkv_buffer,
- key_cache_buffer: &key_cache_buffer,
- value_cache_buffer: &value_cache_buffer,
- num_groups,
- num_heads,
- head_dim,
- suffix_length,
- segment_prefix_length,
- max_sequence_length,
- },
- ) {
- eprintln!("Failed to encode KV cache update: {:?}", e);
- return;
- }
+ self.update_kv_cache_kernel.encode(
+ rotated_keys_buffer,
+ qkv_buffer,
+ &key_cache_buffer,
+ &value_cache_buffer,
+ num_groups as u32,
+ num_heads as u32,
+ head_dim as u32,
+ suffix_length as u32,
+ segment_prefix_length as u32,
+ max_sequence_length as u32,
+ &compute_encoder,
+ );
}
let k_head_stride = (max_sequence_length * head_dim) as i32;
@@ -277,93 +339,129 @@ impl EncodableBlock<Metal> for Attention {
let v_head_stride = (max_sequence_length * head_dim) as i32;
let v_seq_stride = head_dim as i32;
+ let kernel_key = KernelKey {
+ head_dim: head_dim as u32,
+ has_mask: use_mask,
+ };
+
+ let mut mask_kv_seq_stride_opt: Option<u32> = None;
+ let mut mask_q_seq_stride_opt: Option<u32> = None;
+ let mut mask_head_stride_opt: Option<u32> = None;
+ if attention_bias_buffer.is_some() {
+ mask_kv_seq_stride_opt = Some(mask_kv_seq_stride as u32);
+ mask_q_seq_stride_opt = Some(mask_q_seq_stride as u32);
+ mask_head_stride_opt = Some(mask_head_stride as u32);
+ }
+
match variant {
- AttentionKernelVariant::SinglePass => {
- if use_gemm {
- let mtl = state.mtl_context();
- if let Err(e) = self.kernel.encode_gemm(
- mtl,
- compute_encoder,
- AttentionGemmArguments {
- queries_buffer: &queries_buffer,
- keys_buffer: &key_cache_buffer,
- values_buffer: &value_cache_buffer,
- output_buffer: &attention_output_buffer,
- mask_buffer: attention_bias_buffer.as_deref(),
- sinks_buffer: sinks_buffer.as_deref(),
- num_heads,
- num_groups,
- suffix_length,
- sequence_length,
- segment_prefix_length,
- max_sequence_length,
- head_dim,
- is_causal: self.is_causal,
- scale,
- },
- ) {
- eprintln!("Failed to encode gemm attention: {:?}", e);
- }
- } else if let Err(e) = self.kernel.encode_single_pass(
- compute_encoder,
- AttentionSinglePassArguments {
- queries_buffer: &queries_buffer,
- keys_buffer: &key_cache_buffer,
- values_buffer: &value_cache_buffer,
- output_buffer: &attention_output_buffer,
- gqa_factor: gqa_factor as i32,
- sequence_length: sequence_length as i32,
- k_head_stride,
- k_seq_stride,
- v_head_stride,
- v_seq_stride,
- scale,
- mask_buffer: attention_bias_buffer.as_deref(),
- mask_kv_seq_stride,
- mask_q_seq_stride,
- mask_head_stride,
- sinks_buffer: sinks_buffer.as_deref(),
- num_heads,
- suffix_length,
- head_dim,
- is_causal: self.is_causal,
- },
- ) {
- eprintln!("Failed to encode single-pass attention: {:?}", e);
- }
+ KernelVariant::Gemm => {
+ let args = AttentionGemmArguments {
+ queries_buffer: &queries_buffer,
+ keys_buffer: &key_cache_buffer,
+ values_buffer: &value_cache_buffer,
+ output_buffer: &attention_output_buffer,
+ mask_buffer: attention_bias_buffer.as_deref(),
+ sinks_buffer: sinks_buffer.as_deref(),
+ num_heads,
+ num_groups,
+ suffix_length,
+ sequence_length,
+ segment_prefix_length,
+ max_sequence_length,
+ head_dim,
+ is_causal: self.is_causal,
+ scale,
+ };
+ self.gemm_block
+ .encode(state.mtl_context(), compute_encoder, &args)
+ .expect("Failed to encode AttentionGemmBlock");
+ },
+ KernelVariant::SinglePass => {
+ let kernel = match self.single_pass_kernels.get(&kernel_key) {
+ Some(k) => k,
+ None => panic!("Can not find AttentionSinglePassMetalKernel for key {:?}", kernel_key),
+ };
+ let mask_buffer_opt = attention_bias_buffer.as_ref();
+ let sinks_buffer_opt = sinks_buffer.as_ref();
+ kernel.encode(
+ queries_buffer,
+ &key_cache_buffer,
+ &value_cache_buffer,
+ attention_output_buffer,
+ gqa_factor as u32,
+ sequence_length as u32,
+ k_head_stride as u32,
+ k_seq_stride as u32,
+ v_head_stride as u32,
+ v_seq_stride as u32,
+ scale,
+ mask_buffer_opt,
+ mask_kv_seq_stride_opt,
+ mask_q_seq_stride_opt,
+ mask_head_stride_opt,
+ sinks_buffer_opt,
+ num_heads as u32,
+ suffix_length as u32,
+ &compute_encoder,
+ )
},
- AttentionKernelVariant::TwoPass => {
- if let Err(e) = self.kernel.encode_two_pass(
- compute_encoder,
- AttentionTwoPassArguments {
- queries_buffer: &queries_buffer,
- keys_buffer: &key_cache_buffer,
- values_buffer: &value_cache_buffer,
- partials_buffer: &partials_buffer,
- sums_buffer: &sums_buffer,
- maxs_buffer: &maxs_buffer,
- output_buffer: &attention_output_buffer,
- gqa_factor: gqa_factor as i32,
- sequence_length: sequence_length as i32,
- k_head_stride,
- k_seq_stride,
- v_head_stride,
- v_seq_stride,
- scale,
- mask_buffer: attention_bias_buffer.as_deref(),
- mask_kv_seq_stride,
- mask_q_seq_stride,
- mask_head_stride,
- sinks_buffer: sinks_buffer.as_deref(),
- num_heads,
- suffix_length,
- head_dim,
- is_causal: self.is_causal,
- },
- ) {
- eprintln!("Failed to encode two-pass attention: {:?}", e);
- }
+ KernelVariant::TwoPass => {
+ let kernel_pass1 = match self.two_pass_1_kernels.get(&kernel_key) {
+ Some(k) => k,
+ None => panic!("Can not find AttentionTwoPass1MetalKernel for key {:?}", kernel_key),
+ };
+ let kernel_pass2 = match self.two_pass_2_kernels.get(&(head_dim as u32)) {
+ Some(k) => k,
+ None => panic!("Can not find AttentionTwoPass2MetalKernel for key {:?}", kernel_key),
+ };
+
+ let mask_buffer_opt = attention_bias_buffer.as_ref().map(|b| b);
+ let sinks_buffer_opt = sinks_buffer.as_ref().map(|b| b);
+ kernel_pass1.encode(
+ queries_buffer,
+ &key_cache_buffer,
+ &value_cache_buffer,
+ partials_buffer,
+ sums_buffer,
+ maxs_buffer,
+ gqa_factor as u32,
+ sequence_length as u32,
+ k_head_stride as u32,
+ k_seq_stride as u32,
+ v_head_stride as u32,
+ v_seq_stride as u32,
+ scale,
+ num_heads as u32,
+ suffix_length as u32,
+ mask_buffer_opt,
+ mask_kv_seq_stride_opt,
+ mask_q_seq_stride_opt,
+ mask_head_stride_opt,
+ sinks_buffer_opt,
+ &compute_encoder,
+ );
+ kernel_pass2.encode(
+ partials_buffer,
+ sums_buffer,
+ maxs_buffer,
+ attention_output_buffer,
+ num_heads as u32,
+ suffix_length as u32,
+ &compute_encoder,
+ );
},
}
}
}
+
+enum KernelVariant {
+ Gemm,
+ SinglePass,
+ TwoPass,
+}
+
+#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)]
+struct KernelKey {
+ pub head_dim: u32,
+ pub has_mask: bool,
+}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention.metal
deleted file mode 100644
index 115926208..000000000
--- a/crates/uzu/src/backends/metal/kernel/attention/attention.metal
+++ /dev/null
@@ -1,872 +0,0 @@
-#include <metal_stdlib>
-#include <metal_simdgroup>
-#include "../definitions.metal"
-
-using namespace metal;
-
-constant bool has_mask [[function_constant(20)]];
-constant bool query_transposed [[function_constant(21)]];
-constant bool do_causal [[function_constant(22)]];
-constant bool bool_mask [[function_constant(23)]];
-constant bool float_mask [[function_constant(24)]];
-constant bool has_sinks [[function_constant(25)]];
-
-template <typename T, int head_dim, int value_dim = head_dim>
-void attention_single_pass_impl(
- const device T* queries,
- const device T* keys,
- const device T* values,
- device T* out,
- const constant int& gqa_factor,
- const constant int& sequence_length,
- const constant int& k_head_stride,
- const constant int& k_seq_stride,
- const constant int& v_head_stride,
- const constant int& v_seq_stride,
- const constant float& scale,
- const device bool* bmask,
- const device T* fmask,
- const constant int& mask_kv_seq_stride,
- const constant int& mask_q_seq_stride,
- const constant int& mask_head_stride,
- const device float* sinks,
- uint3 tid, // threadgroup position in grid
- uint3 tpg, // threadgroups per grid
- uint simd_gid, // simdgroup index in threadgroup
- uint simd_lid, // thread index in simdgroup
- threadgroup float* shared_max_scores,
- threadgroup float* shared_sum_exp_scores,
- threadgroup float* shared_outputs
-) {
- constexpr int sequence_block_size = 32;
- constexpr int head_block_size = 32;
- constexpr int qk_elements_per_thread = head_dim / head_block_size;
- constexpr int value_elements_per_thread = value_dim / head_block_size;
- int inner_k_stride = sequence_block_size * int(k_seq_stride);
- int inner_v_stride = sequence_block_size * int(v_seq_stride);
-
- typedef float U;
-
- thread U q[qk_elements_per_thread];
- thread U k[qk_elements_per_thread];
- thread U o[value_elements_per_thread];
-
- const int head_idx = tid.x;
- const int q_seq_idx = tid.y;
- const int kv_head_idx = head_idx / gqa_factor;
- const int o_offset = q_seq_idx * tpg.x + head_idx;
- const int q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx
- : head_idx * tpg.y + q_seq_idx;
-
- queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;
- keys += kv_head_idx * k_head_stride + simd_gid * k_seq_stride +
- simd_lid * qk_elements_per_thread;
- values += kv_head_idx * v_head_stride + simd_gid * v_seq_stride +
- simd_lid * value_elements_per_thread;
-
- if (bool_mask) {
- bmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +
- q_seq_idx * mask_q_seq_stride;
- }
- if (float_mask) {
- fmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +
- q_seq_idx * mask_q_seq_stride;
- }
-
- out += o_offset * value_dim + simd_gid * value_elements_per_thread;
-
- // Read the query and 0 the output accumulator
- for (int i = 0; i < qk_elements_per_thread; i++) {
- q[i] = static_cast<U>(scale) * queries[i];
- }
- for (int i = 0; i < value_elements_per_thread; i++) {
- o[i] = 0;
- }
-
- U max_score = -INFINITY;
- U sum_exp_score = 0;
- if (has_sinks && simd_gid == 0) {
- const int num_q_heads = static_cast<int>(tpg.x);
- int q_head_idx = head_idx % num_q_heads;
- max_score = static_cast<U>(sinks[q_head_idx]);
- sum_exp_score = 1;
- }
-
- // For each key
- for (int i = simd_gid; i < sequence_length; i += sequence_block_size) {
- bool use_key = true;
- if (do_causal) {
- use_key = i <= (sequence_length - int(tpg.y) + int(q_seq_idx));
- } else if (bool_mask) {
- use_key = bmask[0];
- }
-
- if (use_key) {
- // Read the key
- for (int j = 0; j < qk_elements_per_thread; j++) {
- k[j] = keys[j];
- }
-
- // Compute the i-th score
- U score = 0;
- for (int j = 0; j < qk_elements_per_thread; j++) {
- score += q[j] * k[j];
- }
- score = simd_sum(score);
- if (float_mask) {
- score += max(-1e9f, static_cast<U>(fmask[0]));
- }
-
- // Update the accumulators
- U new_max = max(max_score, score);
- U factor = fast::exp(max_score - new_max);
- U exp_score = fast::exp(score - new_max);
-
- max_score = new_max;
- sum_exp_score = sum_exp_score * factor + exp_score;
-
- // Update the output accumulator
- for (int j = 0; j < value_elements_per_thread; j++) {
- o[j] = o[j] * factor + exp_score * values[j];
- }
- }
-
- // Move the pointers to the next kv
- keys += inner_k_stride;
- values += inner_v_stride;
- if (bool_mask) {
- bmask += sequence_block_size * mask_kv_seq_stride;
- }
- if (float_mask) {
- fmask += sequence_block_size * mask_kv_seq_stride;
- }
- }
-
- // Each thread has a partial part of the output so we need to combine them.
- if (simd_lid == 0) {
- shared_max_scores[simd_gid] = max_score;
- shared_sum_exp_scores[simd_gid] = sum_exp_score;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- max_score = shared_max_scores[simd_lid];
- U new_max = simd_max(max_score);
- U factor = fast::exp(max_score - new_max);
- sum_exp_score = simd_sum(shared_sum_exp_scores[simd_lid] * factor);
-
- // Now we need to aggregate all the outputs
- for (int i = 0; i < value_elements_per_thread; i++) {
- shared_outputs[simd_lid * head_block_size + simd_gid] = o[i];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- o[i] = simd_sum(
- shared_outputs[simd_gid * head_block_size + simd_lid] * factor
- ) /
- sum_exp_score;
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- // And write the output
- if (simd_lid == 0) {
- for (int i = 0; i < value_elements_per_thread; i++) {
- out[i] = static_cast<T>(o[i]);
- }
- }
-}
-
-template <typename T, int head_dim, int value_dim = head_dim>
-void attention_2pass_1_impl(
- const device T* queries,
- const device T* keys,
- const device T* values,
- device float* out,
- device float* sums,
- device float* maxs,
- const constant int& gqa_factor,
- const constant int& sequence_length,
- const constant int& k_head_stride,
- const constant int& k_seq_stride,
- const constant int& v_head_stride,
- const constant int& v_seq_stride,
- const constant float& scale,
- const device bool* bmask,
- const device T* fmask,
- const constant int& mask_kv_seq_stride,
- const constant int& mask_q_seq_stride,
- const constant int& mask_head_stride,
- const device float* sinks,
- uint3 tid, // threadgroup position in grid
- uint3 tpg, // threadgroups per grid
- uint simd_gid, // simdgroup index in threadgroup
- uint simd_lid, // thread index in simdgroup
- threadgroup float* shared_max_scores,
- threadgroup float* shared_sum_exp_scores,
- threadgroup float* shared_outputs
-) {
- constexpr int sequence_block_size = 8;
- constexpr int head_block_size = 32;
- constexpr int qk_elements_per_thread = head_dim / head_block_size;
- constexpr int value_elements_per_thread = value_dim / head_block_size;
- int inner_k_stride = sequence_block_size * int(k_seq_stride);
- int inner_v_stride = sequence_block_size * int(v_seq_stride);
- constexpr int total_blocks_count = 32;
-
- typedef float U;
-
- thread U q[qk_elements_per_thread];
- thread U k[qk_elements_per_thread];
- thread U o[value_elements_per_thread];
-
- const int block_idx = tid.z;
- const int head_idx = tid.x;
- const int q_seq_idx = tid.y;
- const int o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
- const int q_offset =
- query_transposed
- ? tpg.x * q_seq_idx + head_idx
- : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass
- const int kv_head_idx = head_idx / gqa_factor;
-
- queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;
- keys += kv_head_idx * k_head_stride +
- (block_idx * sequence_block_size + simd_gid) * k_seq_stride +
- simd_lid * qk_elements_per_thread;
- values += kv_head_idx * v_head_stride +
- (block_idx * sequence_block_size + simd_gid) * v_seq_stride +
- simd_lid * value_elements_per_thread;
- out += o_offset * total_blocks_count * value_dim + block_idx * value_dim +
- simd_lid * value_elements_per_thread;
-
- if (bool_mask) {
- bmask += head_idx * mask_head_stride +
- (block_idx * sequence_block_size + simd_gid) * mask_kv_seq_stride +
- q_seq_idx * mask_q_seq_stride;
- }
- if (float_mask) {
- fmask += head_idx * mask_head_stride +
- (block_idx * sequence_block_size + simd_gid) * mask_kv_seq_stride +
- q_seq_idx * mask_q_seq_stride;
- }
- sums += o_offset * total_blocks_count + block_idx;
- maxs += o_offset * total_blocks_count + block_idx;
-
- // Read the query and 0 the output accumulator
- for (int i = 0; i < qk_elements_per_thread; i++) {
- q[i] = static_cast<U>(scale) * queries[i];
- }
- for (int i = 0; i < value_elements_per_thread; i++) {
- o[i] = 0;
- }
-
- U max_score = -1e9;
- U sum_exp_score = 0;
- if (has_sinks && block_idx == 0 && simd_gid == 0) {
- const int num_q_heads = static_cast<int>(tpg.x);
- int q_head_idx = head_idx % num_q_heads;
- max_score = static_cast<U>(sinks[q_head_idx]);
- sum_exp_score = 1;
- }
-
- // For each key
- for (int i = block_idx * sequence_block_size + simd_gid; i < sequence_length;
- i += total_blocks_count * sequence_block_size) {
- bool use_key = true;
- if (do_causal) {
- use_key = i <= (sequence_length - int(tpg.y) + int(q_seq_idx));
- } else if (bool_mask) {
- use_key = bmask[0];
- }
-
- if (use_key) {
- // Read the key
- for (int j = 0; j < qk_elements_per_thread; j++) {
- k[j] = keys[j];
- }
-
- // Compute the i-th score
- U score = 0;
- for (int j = 0; j < qk_elements_per_thread; j++) {
- score += q[j] * k[j];
- }
- score = simd_sum(score);
- if (float_mask) {
- score += max(-1e9f, static_cast<U>(fmask[0]));
- }
-
- // Update the accumulators
- U new_max = max(max_score, score);
- U factor = fast::exp(max_score - new_max);
- U exp_score = fast::exp(score - new_max);
-
- max_score = new_max;
- sum_exp_score = sum_exp_score * factor + exp_score;
-
- // Update the output accumulator
- for (int j = 0; j < value_elements_per_thread; j++) {
- o[j] = o[j] * factor + exp_score * values[j];
- }
- }
-
- // Move the pointers to the next kv
- keys += total_blocks_count * inner_k_stride;
- values += total_blocks_count * inner_v_stride;
- if (bool_mask) {
- bmask += sequence_block_size * total_blocks_count * mask_kv_seq_stride;
- }
- if (float_mask) {
- fmask += sequence_block_size * total_blocks_count * mask_kv_seq_stride;
- }
- }
-
- // Each thread has a partial part of the output so we need to combine them.
- if (simd_lid == 0) {
- shared_max_scores[simd_gid] = max_score;
- shared_sum_exp_scores[simd_gid] = sum_exp_score;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- max_score =
- (simd_lid < sequence_block_size) ? shared_max_scores[simd_lid] : -1e9;
- U new_max = simd_max(max_score);
- U factor = fast::exp(max_score - new_max);
- sum_exp_score =
- (simd_lid < sequence_block_size) ? shared_sum_exp_scores[simd_lid] : 0;
- sum_exp_score = simd_sum(sum_exp_score * factor);
-
- // Write the sum and new max
- if (simd_gid == 0) {
- sums[0] = sum_exp_score;
- maxs[0] = new_max;
- }
-
- // Now we need to aggregate all the outputs
- for (int i = 0; i < value_elements_per_thread; i++) {
- shared_outputs[simd_lid * sequence_block_size + simd_gid] =
- o[i] * fast::exp(shared_max_scores[simd_gid] - new_max);
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // And write the output
- if (simd_gid == 0) {
- U output = shared_outputs[simd_lid * sequence_block_size];
- for (int j = 1; j < sequence_block_size; j++) {
- output += shared_outputs[simd_lid * sequence_block_size + j];
- }
- out[i] = static_cast<T>(output);
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-}
-
-template <typename T, int head_dim>
-void attention_2pass_2_impl(
- const device float* partials,
- const device float* sums,
- const device float* maxs,
- device T* out,
- uint3 tid,
- uint3 tpg,
- uint simd_gid,
- uint simd_lid,
- threadgroup float* shared_outputs
-) {
- constexpr int sequence_block_size = 32;
- constexpr int head_block_size = 32;
- constexpr int elements_per_thread = head_dim / head_block_size;
- constexpr int total_blocks_count = 32;
-
- typedef float U;
-
- thread U o[elements_per_thread];
-
- const int head_idx = tid.x;
- const int q_seq_idx = tid.y;
- const int o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
- const int q_offset =
- query_transposed
- ? tpg.x * q_seq_idx + head_idx
- : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass
-
- partials += o_offset * total_blocks_count * head_dim + simd_gid * head_dim +
- simd_lid * elements_per_thread;
- sums += o_offset * total_blocks_count;
- maxs += o_offset * total_blocks_count;
- out += o_offset * head_dim +
- simd_gid * elements_per_thread; // Our custom output layout
-
- // First everybody reads the max and sum_exp
- U max_score = maxs[simd_lid];
- U new_max = simd_max(max_score);
- U factor = fast::exp(max_score - new_max);
- U sum_exp_score = simd_sum(sums[simd_lid] * factor);
-
- // Now read the block into registers and then use shared memory to transpose
- // it
- for (int i = 0; i < elements_per_thread; i++) {
- o[i] = partials[i];
- }
- for (int i = 0; i < elements_per_thread; i++) {
- shared_outputs[simd_lid * head_block_size + simd_gid] = o[i];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- o[i] = simd_sum(
- shared_outputs[simd_gid * head_block_size + simd_lid] * factor
- ) /
- sum_exp_score;
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- // And write the output
- if (simd_lid == 0) {
- for (int i = 0; i < elements_per_thread; i++) {
- out[i] = static_cast<T>(o[i]);
- }
- }
-}
-
-template <typename T, int head_dim, int value_dim = head_dim>
-void attention_single_pass(
- const device T* queries,
- const device T* keys,
- const device T* values,
- device T* out,
- const constant int& gqa_factor,
- const constant int& sequence_length,
- const constant int& k_head_stride,
- const constant int& k_seq_stride,
- const constant int& v_head_stride,
- const constant int& v_seq_stride,
- const constant float& scale,
- const device bool* bmask,
- const device T* fmask,
- const constant int& mask_kv_seq_stride,
- const constant int& mask_q_seq_stride,
- const constant int& mask_head_stride,
- const device float* sinks,
- uint3 tid,
- uint3 tpg,
- uint simd_gid,
- uint simd_lid,
- threadgroup float* shared_max_scores,
- threadgroup float* shared_sum_exp_scores,
- threadgroup float* shared_outputs
-) {
- attention_single_pass_impl<T, head_dim, value_dim>(
- queries,
- keys,
- values,
- out,
- gqa_factor,
- sequence_length,
- k_head_stride,
- k_seq_stride,
- v_head_stride,
- v_seq_stride,
- scale,
- bmask,
- fmask,
- mask_kv_seq_stride,
- mask_q_seq_stride,
- mask_head_stride,
- sinks,
- tid,
- tpg,
- simd_gid,
- simd_lid,
- shared_max_scores,
- shared_sum_exp_scores,
- shared_outputs
- );
-}
-
-template <typename T, int head_dim, int value_dim = head_dim>
-void attention_2pass_1(
- const device T* queries,
- const device T* keys,
- const device T* values,
- device float* out,
- device float* sums,
- device float* maxs,
- const constant int& gqa_factor,
- const constant int& sequence_length,
- const constant int& k_head_stride,
- const constant int& k_seq_stride,
- const constant int& v_head_stride,
- const constant int& v_seq_stride,
- const constant float& scale,
- const device bool* bmask,
- const device T* fmask,
- const constant int& mask_kv_seq_stride,
- const constant int& mask_q_seq_stride,
- const constant int& mask_head_stride,
- const device float* sinks,
- uint3 tid,
- uint3 tpg,
- uint simd_gid,
- uint simd_lid,
- threadgroup float* shared_max_scores,
- threadgroup float* shared_sum_exp_scores,
- threadgroup float* shared_outputs
-) {
- attention_2pass_1_impl<T, head_dim, value_dim>(
- queries,
- keys,
- values,
- out,
- sums,
- maxs,
- gqa_factor,
- sequence_length,
- k_head_stride,
- k_seq_stride,
- v_head_stride,
- v_seq_stride,
- scale,
- bmask,
- fmask,
- mask_kv_seq_stride,
- mask_q_seq_stride,
- mask_head_stride,
- sinks,
- tid,
- tpg,
- simd_gid,
- simd_lid,
- shared_max_scores,
- shared_sum_exp_scores,
- shared_outputs
- );
-}
-
-template <typename T, int head_dim>
-void attention_2pass_2(
- const device float* partials,
- const device float* sums,
- const device float* maxs,
- device T* out,
- uint3 tid,
- uint3 tpg,
- uint simd_gid,
- uint simd_lid,
- threadgroup float* shared_outputs
-) {
- attention_2pass_2_impl<T, head_dim>(
- partials,
- sums,
- maxs,
- out,
- tid,
- tpg,
- simd_gid,
- simd_lid,
- shared_outputs
- );
-}
-
-#define outerArguments(T) \
- (const device T* queries [[buffer(0)]], \
- const device T* keys [[buffer(1)]], \
- const device T* values [[buffer(2)]], \
- device T* out [[buffer(3)]], \
- const constant int& gqa_factor [[buffer(4)]], \
- const constant int& sequence_length [[buffer(5)]], \
- const constant int& k_head_stride [[buffer(6)]], \
- const constant int& k_seq_stride [[buffer(7)]], \
- const constant int& v_head_stride [[buffer(8)]], \
- const constant int& v_seq_stride [[buffer(9)]], \
- const constant float& scale [[buffer(10)]], \
- const device bool* bmask [[buffer(11), function_constant(bool_mask)]], \
- const device T* fmask [[buffer(12), function_constant(float_mask)]], \
- const constant int& mask_kv_seq_stride \
- [[buffer(13), function_constant(has_mask)]], \
- const constant int& mask_q_seq_stride \
- [[buffer(14), function_constant(has_mask)]], \
- const constant int& mask_head_stride \
- [[buffer(15), function_constant(has_mask)]], \
- const device float* sinks [[buffer(16), function_constant(has_sinks)]], \
- uint3 tid [[threadgroup_position_in_grid]], \
- uint3 tpg [[threadgroups_per_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]])
-
-#define innerArguments \
- (queries, \
- keys, \
- values, \
- out, \
- gqa_factor, \
- sequence_length, \
- k_head_stride, \
- k_seq_stride, \
- v_head_stride, \
- v_seq_stride, \
- scale, \
- bmask, \
- fmask, \
- mask_kv_seq_stride, \
- mask_q_seq_stride, \
- mask_head_stride, \
- sinks, \
- tid, \
- tpg, \
- simd_gid, \
- simd_lid, \
- shared_max_scores, \
- shared_sum_exp_scores, \
- shared_outputs)
-
-// Generate single-pass kernels for different head dimensions
-#define GENERATE_SINGLE_PASS_KERNELS(head_dim_value) \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void attention_single_pass_half_##head_dim_value \
- outerArguments(half) { \
- constexpr int sequence_block_size = 32; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_max_scores[sequence_block_size]; \
- threadgroup float shared_sum_exp_scores[sequence_block_size]; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_single_pass<half, head_dim_value> innerArguments; \
- } \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void attention_single_pass_bfloat_##head_dim_value \
- outerArguments(bfloat) { \
- constexpr int sequence_block_size = 32; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_max_scores[sequence_block_size]; \
- threadgroup float shared_sum_exp_scores[sequence_block_size]; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_single_pass<bfloat, head_dim_value> innerArguments; \
- } \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void attention_single_pass_float_##head_dim_value \
- outerArguments(float) { \
- constexpr int sequence_block_size = 32; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_max_scores[sequence_block_size]; \
- threadgroup float shared_sum_exp_scores[sequence_block_size]; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_single_pass<float, head_dim_value> innerArguments; \
- }
-
-GENERATE_SINGLE_PASS_KERNELS(64)
-GENERATE_SINGLE_PASS_KERNELS(128)
-GENERATE_SINGLE_PASS_KERNELS(256)
-
-#undef outerArguments
-#undef innerArguments
-
-#define outerArguments(T) \
- (const device T* queries [[buffer(0)]], \
- const device T* keys [[buffer(1)]], \
- const device T* values [[buffer(2)]], \
- device float* out [[buffer(3)]], \
- device float* sums [[buffer(4)]], \
- device float* maxs [[buffer(5)]], \
- const constant int& gqa_factor [[buffer(6)]], \
- const constant int& sequence_length [[buffer(7)]], \
- const constant int& k_head_stride [[buffer(8)]], \
- const constant int& k_seq_stride [[buffer(9)]], \
- const constant int& v_head_stride [[buffer(10)]], \
- const constant int& v_seq_stride [[buffer(11)]], \
- const constant float& scale [[buffer(12)]], \
- const device bool* bmask [[buffer(13), function_constant(bool_mask)]], \
- const device T* fmask [[buffer(14), function_constant(float_mask)]], \
- const constant int& mask_kv_seq_stride \
- [[buffer(15), function_constant(has_mask)]], \
- const constant int& mask_q_seq_stride \
- [[buffer(16), function_constant(has_mask)]], \
- const constant int& mask_head_stride \
- [[buffer(17), function_constant(has_mask)]], \
- const device float* sinks [[buffer(18), function_constant(has_sinks)]], \
- uint3 tid [[threadgroup_position_in_grid]], \
- uint3 tpg [[threadgroups_per_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]])
-
-#define innerArguments \
- (queries, \
- keys, \
- values, \
- out, \
- sums, \
- maxs, \
- gqa_factor, \
- sequence_length, \
- k_head_stride, \
- k_seq_stride, \
- v_head_stride, \
- v_seq_stride, \
- scale, \
- bmask, \
- fmask, \
- mask_kv_seq_stride, \
- mask_q_seq_stride, \
- mask_head_stride, \
- sinks, \
- tid, \
- tpg, \
- simd_gid, \
- simd_lid, \
- shared_max_scores, \
- shared_sum_exp_scores, \
- shared_outputs)
-
-// Generate 2-pass Pass 1 kernels for different head dimensions
-#define GENERATE_2PASS_1_KERNELS(head_dim_value) \
- [[max_total_threads_per_threadgroup( \
- 256 \
- )]] kernel void attention_2pass_1_half_##head_dim_value \
- outerArguments(half) { \
- constexpr int sequence_block_size = 8; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_max_scores[sequence_block_size]; \
- threadgroup float shared_sum_exp_scores[sequence_block_size]; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_2pass_1<half, head_dim_value> innerArguments; \
- } \
- [[max_total_threads_per_threadgroup( \
- 256 \
- )]] kernel void attention_2pass_1_bfloat_##head_dim_value \
- outerArguments(bfloat) { \
- constexpr int sequence_block_size = 8; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_max_scores[sequence_block_size]; \
- threadgroup float shared_sum_exp_scores[sequence_block_size]; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_2pass_1<bfloat, head_dim_value> innerArguments; \
- } \
- [[max_total_threads_per_threadgroup( \
- 256 \
- )]] kernel void attention_2pass_1_float_##head_dim_value \
- outerArguments(float) { \
- constexpr int sequence_block_size = 8; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_max_scores[sequence_block_size]; \
- threadgroup float shared_sum_exp_scores[sequence_block_size]; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_2pass_1<float, head_dim_value> innerArguments; \
- }
-
-GENERATE_2PASS_1_KERNELS(64)
-GENERATE_2PASS_1_KERNELS(128)
-GENERATE_2PASS_1_KERNELS(256)
-
-#undef outerArguments
-#undef innerArguments
-
-#define outerArguments(T) \
- (const device float* partials [[buffer(0)]], \
- const device float* sums [[buffer(1)]], \
- const device float* maxs [[buffer(2)]], \
- device T* out [[buffer(3)]], \
- uint3 tid [[threadgroup_position_in_grid]], \
- uint3 tpg [[threadgroups_per_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]])
-
-#define innerArguments \
- (partials, sums, maxs, out, tid, tpg, simd_gid, simd_lid, shared_outputs)
-
-// Generate 2-pass Pass 2 kernels for different head dimensions
-#define GENERATE_2PASS_2_KERNELS(head_dim_value) \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void attention_2pass_2_half_##head_dim_value \
- outerArguments(half) { \
- constexpr int sequence_block_size = 32; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_2pass_2<half, head_dim_value> innerArguments; \
- } \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void attention_2pass_2_float_##head_dim_value \
- outerArguments(float) { \
- constexpr int sequence_block_size = 32; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_2pass_2<float, head_dim_value> innerArguments; \
- }
-
-GENERATE_2PASS_2_KERNELS(64)
-GENERATE_2PASS_2_KERNELS(128)
-GENERATE_2PASS_2_KERNELS(256)
-
-#undef outerArguments
-#undef innerArguments
-
-template <typename T>
-void update_kv_cache(
- const device T* rotated_keys, // [num_groups, suffix_length, head_dim]
- const device T*
- qkv, // [suffix_length, (num_heads + 2*num_groups) * head_dim]
- device T* key_cache, // [num_groups, max_sequence_length, head_dim]
- device T* value_cache, // [num_groups, max_sequence_length, head_dim]
- const constant int& num_groups,
- const constant int& num_heads,
- const constant int& head_dim,
- const constant int& suffix_length,
- const constant int& prefix_segment_length,
- const constant int& max_sequence_length,
- uint3 position // x: group_idx, y: token_idx, z: dim_idx
-) {
- const uint groupIndex = position.x;
- const uint tokenIndex = position.y;
- const uint dimIndex = position.z;
-
- if (groupIndex >= num_groups || tokenIndex >= suffix_length ||
- dimIndex >= head_dim) {
- return;
- }
-
- const uint cacheTokenIndex = prefix_segment_length + tokenIndex;
-
- // Copy rotated key to cache
- const uint rotatedKeyOffset =
- (groupIndex * suffix_length + tokenIndex) * head_dim + dimIndex;
- const uint keyCacheOffset =
- (groupIndex * max_sequence_length + cacheTokenIndex) * head_dim +
- dimIndex;
- key_cache[keyCacheOffset] = rotated_keys[rotatedKeyOffset];
-
- // Copy value to cache
- // qkv layout: [suffix_length, (num_heads + 2*num_groups) * head_dim]
- // Values start at offset: (num_heads * head_dim) + (num_groups * head_dim)
- const uint qkvStride = (num_heads + 2 * num_groups) * head_dim;
- const uint valueBaseOffset = (num_heads + num_groups) * head_dim;
- const uint valueOffset = tokenIndex * qkvStride + valueBaseOffset +
- groupIndex * head_dim + dimIndex;
- const uint valueCacheOffset =
- (groupIndex * max_sequence_length + cacheTokenIndex) * head_dim +
- dimIndex;
- value_cache[valueCacheOffset] = qkv[valueOffset];
-}
-
-#define outerArguments(T) \
- (const device T* rotated_keys [[buffer(0)]], \
- const device T* qkv [[buffer(1)]], \
- device T* key_cache [[buffer(2)]], \
- device T* value_cache [[buffer(3)]], \
- const constant int& num_groups [[buffer(4)]], \
- const constant int& num_heads [[buffer(5)]], \
- const constant int& head_dim [[buffer(6)]], \
- const constant int& suffix_length [[buffer(7)]], \
- const constant int& prefix_segment_length [[buffer(8)]], \
- const constant int& max_sequence_length [[buffer(9)]], \
- uint3 position [[thread_position_in_grid]])
-
-#define innerArguments \
- (rotated_keys, \
- qkv, \
- key_cache, \
- value_cache, \
- num_groups, \
- num_heads, \
- head_dim, \
- suffix_length, \
- prefix_segment_length, \
- max_sequence_length, \
- position)
-
-generateKernels(64, update_kv_cache)
-
-#undef outerArguments
-#undef innerArguments
diff --git a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
similarity index 64%
rename from crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
rename to crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index 8efd95c71..7f9dd6c63 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -1,36 +1,17 @@
#include <metal_stdlib>
-
+#include "../definitions.metal"
#include "../matmul/common/loader.h"
#include "../matmul/common/mma.h"
-#include "../definitions.metal"
#include "attention.h"
using namespace metal;
using namespace uzu::matmul;
using namespace uzu::attention;
-///////////////////////////////////////////////////////////////////////////////
-// Function constants for compile-time specialization
-///////////////////////////////////////////////////////////////////////////////
-
-// Alignment flags (match Gemm convention / our matmul indices)
-constant bool align_Q [[function_constant(200)]];
-constant bool align_K [[function_constant(201)]];
-
-// Feature flags
-constant bool has_mask [[function_constant(300)]];
-constant bool do_causal [[function_constant(301)]];
-constant bool has_sinks [[function_constant(302)]];
-
-///////////////////////////////////////////////////////////////////////////////
-// Helpers
-///////////////////////////////////////////////////////////////////////////////
-
template <typename T>
struct TransformScale {
T scale;
METAL_FUNC TransformScale(T scale_) : scale(scale_) {}
-
METAL_FUNC T apply(T x) const { return scale * x; }
};
@@ -50,55 +31,63 @@ METAL_FUNC T row_reduce_sum(T v) {
return v;
}
-///////////////////////////////////////////////////////////////////////////////
-// Gemm attention kernel
-///////////////////////////////////////////////////////////////////////////////
-
-// clang-format off
-template <
- typename T,
- int BQ,
- int BK,
- int BD,
- int WM,
- int WN,
- typename AccumType = float>
-[[kernel, max_total_threads_per_threadgroup(WM * WN * 32)]] void attention_gemm(
- const device T* Q [[buffer(0)]],
- const device T* K [[buffer(1)]],
- const device T* V [[buffer(2)]],
- device T* O [[buffer(3)]],
- const constant AttnParams* params [[buffer(4)]],
- const constant AttnMaskParams* mask_params [[buffer(5), function_constant(has_mask)]],
- const device T* mask [[buffer(6), function_constant(has_mask)]],
- const device float* sinks [[buffer(7), function_constant(has_sinks)]],
- uint simd_lane_id [[thread_index_in_simdgroup]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]]) { // clang-format on
+#define BQ 32
+#define SIMD_WIDTH 32
+#define WM 4
+#define WN 1
+
+template <typename T, uint BK, uint BD>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(BK, 16, 32)
+VARIANTS(BD, 64, 128, 256)
+KERNEL(AttentionGemm)(
+ const device T* q,
+ const device T* k,
+ const device T* v,
+ device T* o,
+ const constant AttnParams* params,
+ const constant AttnMaskParams* mask_params OPTIONAL(has_mask),
+ const device T* mask OPTIONAL(has_mask),
+ const device float* sinks OPTIONAL(has_sinks),
+ const constant uint& num_heads,
+ const constant uint& suffix_length,
+ const bool align_q SPECIALIZE,
+ const bool align_k SPECIALIZE,
+ const bool do_causal SPECIALIZE,
+ const bool has_mask SPECIALIZE,
+ const bool has_sinks SPECIALIZE,
+ threadgroup T q_smem[BQ * (BD + 16 / sizeof(T))],
+ threadgroup T kv_smem[BK * (BD + 16 / sizeof(T))],
+ const uint tgid_x GROUPS(suffix_length.div_ceil(BQ)),
+ const uint tgid_y GROUPS(num_heads),
+ const uint tgid_z GROUPS(1),
+ const uint lid THREADS(128)
+) {
+ const uint simd_group_id = lid / SIMD_WIDTH;
+ const uint simd_lane_id = lid % SIMD_WIDTH;
// Pacifying compiler
(void)lid;
// -------------------------------------------------------------------------
// Pointer setup (all strides are in elements)
- // tid.x: query tile index (BQ rows)
- // tid.y: query head index
- // tid.z: batch index (currently 1 in uzu, but kept for completeness)
- const int64_t batch_idx = int64_t(tid.z);
- const int64_t head_idx = int64_t(tid.y);
- const int64_t q_tile_idx = int64_t(tid.x);
-
- Q += batch_idx * params->q_strides[0] + head_idx * params->q_strides[1] +
+ // tgid_x: query tile index (BQ rows)
+ // tgid_y: query head index
+ // tgid_z: batch index (currently 1 in uzu, but kept for completeness)
+ const uint batch_idx = tgid_z;
+ const uint head_idx = tgid_y;
+ const uint q_tile_idx = tgid_x;
+
+ q += batch_idx * params->q_strides[0] + head_idx * params->q_strides[1] +
q_tile_idx * int64_t(BQ) * params->q_strides[2];
- const int kv_head_idx = int(tid.y) / params->gqa_factor;
- K += batch_idx * params->k_strides[0] +
+ const int kv_head_idx = int(tgid_y) / params->gqa_factor;
+ k += batch_idx * params->k_strides[0] +
int64_t(kv_head_idx) * params->k_strides[1];
- V += batch_idx * params->v_strides[0] +
+ v += batch_idx * params->v_strides[0] +
int64_t(kv_head_idx) * params->v_strides[1];
- O += batch_idx * params->o_strides[0] + head_idx * params->o_strides[1] +
+ o += batch_idx * params->o_strides[0] + head_idx * params->o_strides[1] +
q_tile_idx * int64_t(BQ) * params->o_strides[2];
if (has_mask) {
@@ -120,13 +109,11 @@ template <
constexpr short tgp_mem_v = BK * (BD + padV);
constexpr short tgp_mem_s = tgp_mem_k > tgp_mem_v ? tgp_mem_k : tgp_mem_v;
- threadgroup T Q_smem[BQ * (BD + padQ)];
- threadgroup T KV_smem[tgp_mem_s];
-
- threadgroup T* Qs = Q_smem;
- threadgroup T* Ks = KV_smem;
- threadgroup T* Vs = KV_smem;
+ threadgroup T* Qs = q_smem;
+ threadgroup T* Ks = kv_smem;
+ threadgroup T* Vs = kv_smem;
+ //
// -------------------------------------------------------------------------
// Block loaders
using QBlockLoader = BlockLoader<
@@ -157,15 +144,16 @@ template <
const int k_src_ld = int(params->k_strides[2]);
const int v_src_ld = int(params->v_strides[2]);
- thread QBlockLoader loader_q(Q, q_src_ld, Qs, simd_group_id, simd_lane_id);
- thread KBlockLoader loader_k(K, k_src_ld, Ks, simd_group_id, simd_lane_id);
- thread VBlockLoader loader_v(V, v_src_ld, Vs, simd_group_id, simd_lane_id);
+ thread QBlockLoader loader_q(q, q_src_ld, Qs, simd_group_id, simd_lane_id);
+ thread KBlockLoader loader_k(k, k_src_ld, Ks, simd_group_id, simd_lane_id);
+ thread VBlockLoader loader_v(v, v_src_ld, Vs, simd_group_id, simd_lane_id);
TransformScale<T> ts(static_cast<T>(params->scale * M_LOG2E_F));
// -------------------------------------------------------------------------
// MMA tiles
constexpr short kFragSize = 8;
+ using AccumType = float;
using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;
constexpr int kNWarps = WM * WN;
@@ -216,7 +204,7 @@ template <
// Load Q block once (and apply scaling)
threadgroup_barrier(mem_flags::mem_threadgroup);
- if (!align_Q && int(tid.x) == params->nq_aligned) {
+ if (!align_q && int(tgid_x) == params->nq_aligned) {
loader_q.load_safe(short2(BD, params->q_rem));
} else {
loader_q.load_unsafe();
@@ -230,26 +218,26 @@ template <
AccumType sum_score = AccumType(0);
if (has_sinks) {
- max_score = M_LOG2E_F * static_cast<AccumType>(sinks[tid.y]);
+ max_score = M_LOG2E_F * static_cast<AccumType>(sinks[tgid_y]);
sum_score = AccumType(1);
}
// Determine K block loop limit (causal can early-stop)
int kb_lim = params->nk;
if (do_causal) {
- const int q_max = (int(tid.x) + 1) * BQ + params->q_off;
+ const int q_max = (int(tgid_x) + 1) * BQ + params->q_off;
kb_lim = (q_max + BK - 1) / BK;
kb_lim = min(params->nk, kb_lim);
}
- const int q_rel = int(tid.x) * BQ + int(tm) + int(sm); // [0, q_len)
- const int q_abs = q_rel + params->q_off; // [0, k_len)
+ const int q_rel = int(tgid_x) * BQ + int(tm) + int(sm); // [0, q_len)
+ const int q_abs = q_rel + params->q_off; // [0, k_len)
// Loop over KV blocks
for (int kb = 0; kb < kb_lim; kb++) {
// Load K block
threadgroup_barrier(mem_flags::mem_threadgroup);
- if (!align_K && kb == params->nk_aligned) {
+ if (!align_k && kb == params->nk_aligned) {
loader_k.load_safe(short2(BD, params->k_rem));
} else {
loader_k.load_unsafe();
@@ -276,7 +264,7 @@ template <
}
// Mask out tail keys for the last (unaligned) K block
- if (!align_K && kb == params->nk_aligned) {
+ if (!align_k && kb == params->nk_aligned) {
const int k_rem = params->k_rem;
UZU_PRAGMA_UNROLL
for (short j = 0; j < TK; j++) {
@@ -293,7 +281,7 @@ template <
// Causal mask (only needed for the last few blocks near the diagonal)
if (do_causal) {
- const int tail_blocks = (BQ + BK - 1) / BK + int(!align_K);
+ const int tail_blocks = (BQ + BK - 1) / BK + int(!align_k);
const int tail_start = kb_lim - tail_blocks;
if (kb >= tail_start) {
UZU_PRAGMA_UNROLL
@@ -338,7 +326,7 @@ template <
// Load V block (overwriting K in shared memory)
threadgroup_barrier(mem_flags::mem_threadgroup);
- if (!align_K && kb == params->nk_aligned) {
+ if (!align_k && kb == params->nk_aligned) {
loader_v.load_safe(short2(BD, params->k_rem));
} else {
loader_v.load_unsafe();
@@ -428,9 +416,9 @@ template <
threadgroup_barrier(mem_flags::mem_none);
// Store results (O is row-major with row-stride params->o_strides[2])
- O += int64_t(tm + sm) * params->o_strides[2] + int64_t(sn);
+ o += int64_t(tm + sm) * params->o_strides[2] + int64_t(sn);
- if (!align_Q && int(tid.x) == params->nq_aligned) {
+ if (!align_q && int(tgid_x) == params->nq_aligned) {
const short2 dst_tile_dims = short2(BD - sn, params->q_rem - (tm + sm));
if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0) {
@@ -438,64 +426,11 @@ template <
}
Otile.template store_safe<T, 1, 1>(
- O,
+ o,
int(params->o_strides[2]),
dst_tile_dims
);
} else {
- Otile.template store<T, 1, 1>(O, int(params->o_strides[2]));
+ Otile.template store<T, 1, 1>(o, int(params->o_strides[2]));
}
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Kernel instantiations (BQ=32, BK in {32,16}, WM=2, WN=2)
-///////////////////////////////////////////////////////////////////////////////
-
-#define instantiate_attention_gemm( \
- type_name, \
- element_type, \
- head_dim_value, \
- bk_value \
-) \
- template [[host_name( \
- "attention_gemm_" #type_name "_" #head_dim_value "_bk" #bk_value \
- )]] [[kernel]] void \
- attention_gemm<element_type, 32, bk_value, head_dim_value, 4, 1, float>( \
- const device element_type* Q [[buffer(0)]], \
- const device element_type* K [[buffer(1)]], \
- const device element_type* V [[buffer(2)]], \
- device element_type* O [[buffer(3)]], \
- const constant AttnParams* params [[buffer(4)]], \
- const constant AttnMaskParams* mask_params \
- [[buffer(5), function_constant(has_mask)]], \
- const device element_type* mask \
- [[buffer(6), function_constant(has_mask)]], \
- const device float* sinks [[buffer(7), function_constant(has_sinks)]], \
- uint simd_lane_id [[thread_index_in_simdgroup]], \
- uint simd_group_id [[simdgroup_index_in_threadgroup]], \
- uint3 tid [[threadgroup_position_in_grid]], \
- uint3 lid [[thread_position_in_threadgroup]] \
- );
-
-instantiate_attention_gemm(f16, half, 64, 32) instantiate_attention_gemm(
- f16,
- half,
- 128,
- 16
-) instantiate_attention_gemm(f16, half, 256, 16)
-
- instantiate_attention_gemm(bf16, bfloat, 64, 32) instantiate_attention_gemm(
- bf16,
- bfloat,
- 128,
- 16
- ) instantiate_attention_gemm(bf16, bfloat, 256, 16)
-
- instantiate_attention_gemm(f32, float, 64, 32) instantiate_attention_gemm(
- f32,
- float,
- 128,
- 16
- ) instantiate_attention_gemm(f32, float, 256, 16)
-
-#undef instantiate_attention_gemm
+}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
new file mode 100644
index 000000000..b3ef95a54
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -0,0 +1,177 @@
+#include <metal_stdlib>
+#include <metal_simdgroup>
+#include "../definitions.metal"
+
+#define SIMD_WIDTH 32
+#define SEQUENCE_BLOCK_SIZE 32
+#define HEAD_BLOCK_SIZE 32
+
+template <typename T, uint head_dim>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(head_dim, 64, 128, 256)
+KERNEL(AttentionSinglePass)(
+ const device T* queries,
+ const device T* keys,
+ const device T* values,
+ device T* out,
+ const constant uint& gqa_factor,
+ const constant uint& sequence_length,
+ const constant uint& k_head_stride,
+ const constant uint& k_seq_stride,
+ const constant uint& v_head_stride,
+ const constant uint& v_seq_stride,
+ const constant float& scale,
+ const device T* fmask OPTIONAL(float_mask),
+ const constant uint& mask_kv_seq_stride OPTIONAL(has_mask),
+ const constant uint& mask_q_seq_stride OPTIONAL(has_mask),
+ const constant uint& mask_head_stride OPTIONAL(has_mask),
+ const device float* sinks OPTIONAL(has_sinks),
+ const constant uint& num_heads,
+ const constant uint& suffix_length,
+ threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],
+ threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],
+ threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],
+ const bool float_mask SPECIALIZE,
+ const bool has_mask SPECIALIZE,
+ const bool has_sinks SPECIALIZE,
+ const bool do_causal SPECIALIZE,
+ const uint head_idx GROUPS(num_heads),
+ const uint q_seq_idx GROUPS(suffix_length),
+ const uint tid THREADS(1024)
+) {
+ const uint3 tpg = {num_heads, suffix_length, 1};
+ const uint simd_gid = tid / SIMD_WIDTH;
+ const uint simd_lid = tid % SIMD_WIDTH;
+ constexpr bool bool_mask = false;
+ constexpr bool query_transposed = false;
+
+ constexpr uint value_dim = head_dim;
+ constexpr uint qk_elements_per_thread = head_dim / HEAD_BLOCK_SIZE;
+ constexpr uint value_elements_per_thread = value_dim / HEAD_BLOCK_SIZE;
+ uint inner_k_stride = SEQUENCE_BLOCK_SIZE * int(k_seq_stride);
+ uint inner_v_stride = SEQUENCE_BLOCK_SIZE * int(v_seq_stride);
+
+ typedef float U;
+
+ thread U q[qk_elements_per_thread];
+ thread U k[qk_elements_per_thread];
+ thread U o[value_elements_per_thread];
+
+ const uint kv_head_idx = head_idx / gqa_factor;
+ const uint o_offset = q_seq_idx * tpg.x + head_idx;
+ const uint q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx
+ : head_idx * tpg.y + q_seq_idx;
+
+ queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;
+ keys += kv_head_idx * k_head_stride + simd_gid * k_seq_stride +
+ simd_lid * qk_elements_per_thread;
+ values += kv_head_idx * v_head_stride + simd_gid * v_seq_stride +
+ simd_lid * value_elements_per_thread;
+
+ if (bool_mask) {
+ // bmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +
+ // q_seq_idx * mask_q_seq_stride;
+ }
+ if (float_mask) {
+ fmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +
+ q_seq_idx * mask_q_seq_stride;
+ }
+
+ out += o_offset * value_dim + simd_gid * value_elements_per_thread;
+
+ // Read the query and 0 the output accumulator
+ for (int i = 0; i < qk_elements_per_thread; i++) {
+ q[i] = static_cast<U>(scale) * queries[i];
+ }
+ for (int i = 0; i < value_elements_per_thread; i++) {
+ o[i] = 0;
+ }
+
+ U max_score = -INFINITY;
+ U sum_exp_score = 0;
+ if (has_sinks && simd_gid == 0) {
+ const int num_q_heads = static_cast<int>(tpg.x);
+ int q_head_idx = head_idx % num_q_heads;
+ max_score = static_cast<U>(sinks[q_head_idx]);
+ sum_exp_score = 1;
+ }
+
+ // For each key
+ for (uint i = simd_gid; i < sequence_length; i += SEQUENCE_BLOCK_SIZE) {
+ bool use_key = true;
+ if (do_causal) {
+ use_key = i <= (sequence_length - tpg.y + q_seq_idx);
+ } else if (bool_mask) {
+ // use_key = bmask[0];
+ }
+
+ if (use_key) {
+ // Read the key
+ for (uint j = 0; j < qk_elements_per_thread; j++) {
+ k[j] = keys[j];
+ }
+
+ // Compute the i-th score
+ U score = 0;
+ for (uint j = 0; j < qk_elements_per_thread; j++) {
+ score += q[j] * k[j];
+ }
+ score = simd_sum(score);
+ if (float_mask) {
+ score += max(-1e9f, static_cast<U>(fmask[0]));
+ }
+
+ // Update the accumulators
+ U new_max = max(max_score, score);
+ U factor = fast::exp(max_score - new_max);
+ U exp_score = fast::exp(score - new_max);
+
+ max_score = new_max;
+ sum_exp_score = sum_exp_score * factor + exp_score;
+
+ // Update the output accumulator
+ for (int j = 0; j < value_elements_per_thread; j++) {
+ o[j] = o[j] * factor + exp_score * values[j];
+ }
+ }
+
+ // Move the pointers to the next kv
+ keys += inner_k_stride;
+ values += inner_v_stride;
+ if (bool_mask) {
+ // bmask += SEQUENCE_BLOCK_SIZE * mask_kv_seq_stride;
+ }
+ if (float_mask) {
+ fmask += SEQUENCE_BLOCK_SIZE * mask_kv_seq_stride;
+ }
+ }
+
+ // Each thread has a partial part of the output so we need to combine them.
+ if (simd_lid == 0) {
+ shared_max_scores[simd_gid] = max_score;
+ shared_sum_exp_scores[simd_gid] = sum_exp_score;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ max_score = shared_max_scores[simd_lid];
+ U new_max = simd_max(max_score);
+ U factor = fast::exp(max_score - new_max);
+ sum_exp_score = simd_sum(shared_sum_exp_scores[simd_lid] * factor);
+
+ // Now we need to aggregate all the outputs
+ for (int i = 0; i < value_elements_per_thread; i++) {
+ shared_outputs[simd_lid * HEAD_BLOCK_SIZE + simd_gid] = o[i];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ o[i] = simd_sum(
+ shared_outputs[simd_gid * HEAD_BLOCK_SIZE + simd_lid] * factor
+ ) /
+ sum_exp_score;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ // And write the output
+ if (simd_lid == 0) {
+ for (int i = 0; i < value_elements_per_thread; i++) {
+ out[i] = static_cast<T>(o[i]);
+ }
+ }
+}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
new file mode 100644
index 000000000..3c0295233
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -0,0 +1,251 @@
+#include <metal_stdlib>
+#include <metal_simdgroup>
+#include "../definitions.metal"
+
+#define HEAD_BLOCK_SIZE 32
+#define SEQUENCE_BLOCK_SIZE_1 8
+#define SEQUENCE_BLOCK_SIZE_2 32
+#define SIMD_WIDTH 32
+#define TOTAL_BLOCKS_COUNT 32
+
+template <typename T, uint head_dim>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(head_dim, 64, 128, 256)
+KERNEL(AttentionTwoPass1)(
+ const device T* queries,
+ const device T* keys,
+ const device T* values,
+ device float* out,
+ device float* sums,
+ device float* maxs,
+ const constant uint& gqa_factor,
+ const constant uint& sequence_length,
+ const constant uint& k_head_stride,
+ const constant uint& k_seq_stride,
+ const constant uint& v_head_stride,
+ const constant uint& v_seq_stride,
+ const constant float& scale,
+ const constant uint& num_heads,
+ const constant uint& suffix_length,
+ const device T* fmask OPTIONAL(float_mask),
+ const constant uint& mask_kv_seq_stride OPTIONAL(has_mask),
+ const constant uint& mask_q_seq_stride OPTIONAL(has_mask),
+ const constant uint& mask_head_stride OPTIONAL(has_mask),
+ const device float* sinks OPTIONAL(has_sinks),
+ const bool float_mask SPECIALIZE,
+ const bool has_mask SPECIALIZE,
+ const bool has_sinks SPECIALIZE,
+ const bool do_causal SPECIALIZE,
+ threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE_1],
+ threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE_1],
+ threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_1 * HEAD_BLOCK_SIZE],
+ const uint head_idx GROUPS(num_heads),
+ const uint q_seq_idx GROUPS(suffix_length),
+ const uint block_idx GROUPS(TOTAL_BLOCKS_COUNT),
+ const uint tid THREADS(256)
+) {
+ const uint3 tpg = {num_heads, suffix_length, TOTAL_BLOCKS_COUNT};
+ const uint simd_gid = tid / SIMD_WIDTH;
+ const uint simd_lid = tid % SIMD_WIDTH;
+
+ constexpr bool query_transposed = false;
+ constexpr uint value_dim = head_dim;
+ constexpr uint qk_elements_per_thread = head_dim / HEAD_BLOCK_SIZE;
+ constexpr uint value_elements_per_thread = value_dim / HEAD_BLOCK_SIZE;
+ uint inner_k_stride = SEQUENCE_BLOCK_SIZE_1 * k_seq_stride;
+ uint inner_v_stride = SEQUENCE_BLOCK_SIZE_1 * v_seq_stride;
+
+ typedef float U;
+
+ thread U q[qk_elements_per_thread];
+ thread U k[qk_elements_per_thread];
+ thread U o[value_elements_per_thread];
+
+ const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
+ const uint q_offset =
+ query_transposed
+ ? tpg.x * q_seq_idx + head_idx
+ : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass
+ const uint kv_head_idx = head_idx / gqa_factor;
+
+ queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;
+ keys += kv_head_idx * k_head_stride +
+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid) * k_seq_stride +
+ simd_lid * qk_elements_per_thread;
+ values += kv_head_idx * v_head_stride +
+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid) * v_seq_stride +
+ simd_lid * value_elements_per_thread;
+ out += o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim +
+ simd_lid * value_elements_per_thread;
+ if (float_mask) {
+ fmask +=
+ head_idx * mask_head_stride +
+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid) * mask_kv_seq_stride +
+ q_seq_idx * mask_q_seq_stride;
+ }
+ sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
+ maxs += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
+
+ // Read the query and 0 the output accumulator
+ for (uint i = 0; i < qk_elements_per_thread; i++) {
+ q[i] = static_cast<U>(scale) * queries[i];
+ }
+ for (uint i = 0; i < value_elements_per_thread; i++) {
+ o[i] = 0;
+ }
+
+ U max_score = -1e9;
+ U sum_exp_score = 0;
+ if (has_sinks && block_idx == 0 && simd_gid == 0) {
+ const uint num_q_heads = tpg.x;
+ int q_head_idx = head_idx % num_q_heads;
+ max_score = static_cast<U>(sinks[q_head_idx]);
+ sum_exp_score = 1;
+ }
+
+ // For each key
+ for (uint i = block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid;
+ i < sequence_length;
+ i += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1) {
+ bool use_key = true;
+ if (do_causal) {
+ use_key = i <= (sequence_length - tpg.y + q_seq_idx);
+ }
+
+ if (use_key) {
+ // Read the key
+ for (uint j = 0; j < qk_elements_per_thread; j++) {
+ k[j] = keys[j];
+ }
+
+ // Compute the i-th score
+ U score = 0;
+ for (uint j = 0; j < qk_elements_per_thread; j++) {
+ score += q[j] * k[j];
+ }
+ score = simd_sum(score);
+ if (float_mask) {
+ score += max(-1e9f, static_cast<U>(fmask[0]));
+ }
+
+ // Update the accumulators
+ U new_max = max(max_score, score);
+ U factor = fast::exp(max_score - new_max);
+ U exp_score = fast::exp(score - new_max);
+
+ max_score = new_max;
+ sum_exp_score = sum_exp_score * factor + exp_score;
+
+ // Update the output accumulator
+ for (uint j = 0; j < value_elements_per_thread; j++) {
+ o[j] = o[j] * factor + exp_score * values[j];
+ }
+ }
+
+ // Move the pointers to the next kv
+ keys += TOTAL_BLOCKS_COUNT * inner_k_stride;
+ values += TOTAL_BLOCKS_COUNT * inner_v_stride;
+ if (float_mask) {
+ fmask += SEQUENCE_BLOCK_SIZE_1 * TOTAL_BLOCKS_COUNT * mask_kv_seq_stride;
+ }
+ }
+
+ // Each thread has a partial part of the output so we need to combine them.
+ if (simd_lid == 0) {
+ shared_max_scores[simd_gid] = max_score;
+ shared_sum_exp_scores[simd_gid] = sum_exp_score;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ max_score =
+ (simd_lid < SEQUENCE_BLOCK_SIZE_1) ? shared_max_scores[simd_lid] : -1e9;
+ U new_max = simd_max(max_score);
+ U factor = fast::exp(max_score - new_max);
+ sum_exp_score =
+ (simd_lid < SEQUENCE_BLOCK_SIZE_1) ? shared_sum_exp_scores[simd_lid] : 0;
+ sum_exp_score = simd_sum(sum_exp_score * factor);
+
+ // Write the sum and new max
+ if (simd_gid == 0) {
+ sums[0] = sum_exp_score;
+ maxs[0] = new_max;
+ }
+
+ // Now we need to aggregate all the outputs
+ for (uint i = 0; i < value_elements_per_thread; i++) {
+ shared_outputs[simd_lid * SEQUENCE_BLOCK_SIZE_1 + simd_gid] =
+ o[i] * fast::exp(shared_max_scores[simd_gid] - new_max);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // And write the output
+ if (simd_gid == 0) {
+ U output = shared_outputs[simd_lid * SEQUENCE_BLOCK_SIZE_1];
+ for (uint j = 1; j < SEQUENCE_BLOCK_SIZE_1; j++) {
+ output += shared_outputs[simd_lid * SEQUENCE_BLOCK_SIZE_1 + j];
+ }
+ out[i] = static_cast<T>(output);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+}
+
+template <typename T, uint head_dim>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(head_dim, 64, 128, 256)
+KERNEL(AttentionTwoPass2)(
+ const device float* partials,
+ const device float* sums,
+ const device float* maxs,
+ device T* out,
+ const constant uint& num_heads,
+ const constant uint& suffix_length,
+ threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_2 * HEAD_BLOCK_SIZE],
+ const uint head_idx GROUPS(num_heads),
+ const uint q_seq_idx GROUPS(suffix_length),
+ const uint tid THREADS(1024)
+) {
+ const uint3 tpg = {num_heads, suffix_length, 1};
+ const uint simd_gid = tid / SIMD_WIDTH;
+ const uint simd_lid = tid % SIMD_WIDTH;
+
+ constexpr uint elements_per_thread = head_dim / HEAD_BLOCK_SIZE;
+
+ typedef float U;
+ thread U o[elements_per_thread];
+
+ const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
+
+ partials += o_offset * TOTAL_BLOCKS_COUNT * head_dim + simd_gid * head_dim +
+ simd_lid * elements_per_thread;
+ sums += o_offset * TOTAL_BLOCKS_COUNT;
+ maxs += o_offset * TOTAL_BLOCKS_COUNT;
+ out += o_offset * head_dim +
+ simd_gid * elements_per_thread; // Our custom output layout
+
+ // First everybody reads the max and sum_exp
+ U max_score = maxs[simd_lid];
+ U new_max = simd_max(max_score);
+ U factor = fast::exp(max_score - new_max);
+ U sum_exp_score = simd_sum(sums[simd_lid] * factor);
+
+ // Now read the block into registers and then use shared memory to transpose
+ // it
+ for (uint i = 0; i < elements_per_thread; i++) {
+ o[i] = partials[i];
+ }
+ for (uint i = 0; i < elements_per_thread; i++) {
+ shared_outputs[simd_lid * HEAD_BLOCK_SIZE + simd_gid] = o[i];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ o[i] = simd_sum(
+ shared_outputs[simd_gid * HEAD_BLOCK_SIZE + simd_lid] * factor
+ ) /
+ sum_exp_score;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ // And write the output
+ if (simd_lid == 0) {
+ for (uint i = 0; i < elements_per_thread; i++) {
+ out[i] = static_cast<T>(o[i]);
+ }
+ }
+}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal
new file mode 100644
index 000000000..fc76a792d
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal
@@ -0,0 +1,42 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(AttentionUpdateKVCache)(
+ const device T* rotated_keys,
+ const device T* qkv,
+ device T* key_cache,
+ device T* value_cache,
+ const constant uint& num_groups,
+ const constant uint& num_heads,
+ const constant uint& head_dim,
+ const constant uint& suffix_length,
+ const constant uint& prefix_segment_length,
+ const constant uint& max_sequence_length,
+ const uint group_index AXIS(num_groups, 1),
+ const uint token_index AXIS(suffix_length, 1),
+ const uint dim_index AXIS(head_dim, 64)
+) {
+ const uint cacheTokenIndex = prefix_segment_length + token_index;
+
+ // Copy rotated key to cache
+ const uint rotatedKeyOffset =
+ (group_index * suffix_length + token_index) * head_dim + dim_index;
+ const uint keyCacheOffset =
+ (group_index * max_sequence_length + cacheTokenIndex) * head_dim +
+ dim_index;
+ key_cache[keyCacheOffset] = rotated_keys[rotatedKeyOffset];
+
+ // Copy value to cache
+ // qkv layout: [suffix_length, (num_heads + 2*num_groups) * head_dim]
+ // Values start at offset: (num_heads * head_dim) + (num_groups * head_dim)
+ const uint qkvStride = (num_heads + 2 * num_groups) * head_dim;
+ const uint valueBaseOffset = (num_heads + num_groups) * head_dim;
+ const uint valueOffset = token_index * qkvStride + valueBaseOffset +
+ group_index * head_dim + dim_index;
+ const uint valueCacheOffset =
+ (group_index * max_sequence_length + cacheTokenIndex) * head_dim +
+ dim_index;
+ value_cache[valueCacheOffset] = qkv[valueOffset];
+}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
index 241747e35..2f2f5cc67 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
@@ -1,111 +1,23 @@
-use std::{collections::HashMap, mem::size_of, ptr::NonNull};
+use std::{
+ cell::RefCell,
+ collections::{HashMap, hash_map::Entry},
+};
-use metal::{MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLFunctionConstantValues, MTLSize};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-use thiserror::Error;
+use metal::{MTLBuffer, MTLComputeCommandEncoder};
+use objc2::{Message, runtime::ProtocolObject};
use crate::{
DataType,
backends::{
- common::gpu_types::{AttnMaskParams, AttnParams},
- metal::{
- FunctionConstantValuesSetValue, MetalContext, MetalError, data_type::MetalDataTypeExt,
- kernel::moe::dtype_suffix,
+ common::{
+ gpu_types::{AttnMaskParams, AttnParams},
+ kernel::AttentionGemmKernel,
},
+ metal::{MetalContext, MetalError, kernel::dsl::AttentionGemmMetalKernel},
},
};
-#[derive(Debug, Clone, Copy)]
-pub enum AttentionKernelVariant {
- SinglePass,
- TwoPass,
-}
-
-type PipelineKey = (usize, bool, bool, bool); // (head_dim, has_sinks, is_causal, has_mask)
-
-pub struct AttentionKernelPipelines {
- single_pass: HashMap<PipelineKey, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
- two_pass_1: HashMap<PipelineKey, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
- two_pass_2: HashMap<usize, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
- kv_cache_update: Option<Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
-}
-
-pub struct AttentionKernel {
- data_type: DataType,
- pipelines: AttentionKernelPipelines,
-}
-
-#[derive(Debug, Error)]
-pub enum AttentionError {
- #[error("Metal error: {0}")]
- MetalError(#[from] MetalError),
- #[error("Function not found: {0}")]
- FunctionNotFound(String),
- #[error("Unsupported head dimension: {0}")]
- UnsupportedHeadDim(usize),
-}
-
-pub struct AttentionSinglePassArguments<'a> {
- pub queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)
- pub keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)
- pub values_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)
- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3)
- pub gqa_factor: i32, // buffer(4)
- pub sequence_length: i32, // buffer(5) - sequence_length
- pub k_head_stride: i32, // buffer(6)
- pub k_seq_stride: i32, // buffer(7)
- pub v_head_stride: i32, // buffer(8)
- pub v_seq_stride: i32, // buffer(9)
- pub scale: f32, // buffer(10)
- pub mask_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(11/12)
- pub mask_kv_seq_stride: i32, // buffer(13)
- pub mask_q_seq_stride: i32, // buffer(14)
- pub mask_head_stride: i32, // buffer(15)
- pub sinks_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(16)
- pub num_heads: usize,
- pub suffix_length: usize,
- pub head_dim: usize,
- pub is_causal: bool,
-}
-
-pub struct AttentionTwoPassArguments<'a> {
- pub queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)
- pub keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)
- pub values_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)
- pub partials_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) - pass 1 output
- pub sums_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4) - pass 1 output
- pub maxs_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5) - pass 1 output
- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) - pass 2 output
- pub gqa_factor: i32, // buffer(6)
- pub sequence_length: i32, // buffer(7) - sequence_length
- pub k_head_stride: i32, // buffer(8)
- pub k_seq_stride: i32, // buffer(9)
- pub v_head_stride: i32, // buffer(10)
- pub v_seq_stride: i32, // buffer(11)
- pub scale: f32, // buffer(12)
- pub mask_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(13/14)
- pub mask_kv_seq_stride: i32, // buffer(15)
- pub mask_q_seq_stride: i32, // buffer(16)
- pub mask_head_stride: i32, // buffer(17)
- pub sinks_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(18)
- pub num_heads: usize,
- pub suffix_length: usize,
- pub head_dim: usize,
- pub is_causal: bool,
-}
-
-pub struct KVCacheUpdateArguments<'a> {
- pub rotated_keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)
- pub qkv_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)
- pub key_cache_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)
- pub value_cache_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3)
- pub num_groups: usize,
- pub num_heads: usize,
- pub head_dim: usize,
- pub suffix_length: usize,
- pub segment_prefix_length: usize,
- pub max_sequence_length: usize,
-}
+const BQ: usize = 32;
pub struct AttentionGemmArguments<'a> {
pub queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)
@@ -125,484 +37,63 @@ pub struct AttentionGemmArguments<'a> {
pub scale: f32,
}
-fn make_function_constants(
- has_mask_value: bool,
- has_sinks_value: bool,
- is_causal_value: bool,
-) -> Retained<MTLFunctionConstantValues> {
- let function_constants = MTLFunctionConstantValues::new();
-
- let query_transposed_value = false;
- let bool_mask_value = false;
- let float_mask_value = has_mask_value;
-
- function_constants.set_value(&has_mask_value, 20); // has_mask
- function_constants.set_value(&query_transposed_value, 21); // query_transposed
- function_constants.set_value(&bool_mask_value, 23); // bool_mask
- function_constants.set_value(&float_mask_value, 24); // float_mask
- function_constants.set_value(&is_causal_value, 22); // do_causal
- function_constants.set_value(&has_sinks_value, 25); // has_sinks
-
- function_constants
+pub struct AttentionGemmBlock {
+ data_type: DataType,
+ cache: RefCell<HashMap<KernelKey, AttentionGemmMetalKernel>>,
}
-impl AttentionKernel {
- pub fn new(
- context: &MetalContext,
- data_type: DataType,
- ) -> Result<Self, AttentionError> {
- let data_suffix = data_type.metal_type();
-
- let supported_head_dims = [64, 128, 256];
- let mut single_pass = HashMap::new();
- let mut two_pass_1 = HashMap::new();
- let mut two_pass_2 = HashMap::new();
-
- // Pre-generate all supported variants for sinks and causal configurations
- for &has_sinks_value in &[false, true] {
- for &is_causal_value in &[false, true] {
- for &has_mask_value in &[false, true] {
- let function_constants = make_function_constants(has_mask_value, has_sinks_value, is_causal_value);
-
- for &head_dim in &supported_head_dims {
- if let Ok(pipeline) = context.compute_pipeline_state(
- &format!("attention_single_pass_{}_{}", data_suffix, head_dim),
- Some(&function_constants),
- ) {
- single_pass.insert((head_dim, has_sinks_value, is_causal_value, has_mask_value), pipeline);
- }
-
- if let Ok(pipeline) = context.compute_pipeline_state(
- &format!("attention_2pass_1_{}_{}", data_suffix, head_dim),
- Some(&function_constants),
- ) {
- two_pass_1.insert((head_dim, has_sinks_value, is_causal_value, has_mask_value), pipeline);
- }
-
- if !two_pass_2.contains_key(&head_dim) {
- if let Ok(pipeline) = context.compute_pipeline_state(
- &format!("attention_2pass_2_{}_{}", data_suffix, head_dim),
- Some(&function_constants),
- ) {
- two_pass_2.insert(head_dim, pipeline);
- }
- }
- }
- }
- }
- }
-
- let kv_cache_update = context.compute_pipeline_state(&format!("update_kv_cache_{}", data_suffix), None).ok();
-
- Ok(Self {
+impl AttentionGemmBlock {
+ pub fn new(data_type: DataType) -> Self {
+ let cache = RefCell::new(HashMap::new());
+ Self {
data_type,
- pipelines: AttentionKernelPipelines {
- single_pass,
- two_pass_1,
- two_pass_2,
- kv_cache_update,
- },
- })
- }
-
- pub fn supports_single_pass(
- &self,
- head_dim: usize,
- is_causal: bool,
- has_mask: bool,
- ) -> bool {
- self.pipelines.single_pass.contains_key(&(head_dim, false, is_causal, has_mask))
- }
-
- pub fn supports_two_pass(
- &self,
- head_dim: usize,
- is_causal: bool,
- has_mask: bool,
- ) -> bool {
- self.pipelines.two_pass_1.contains_key(&(head_dim, false, is_causal, has_mask))
- && self.pipelines.two_pass_2.contains_key(&head_dim)
- }
-
- pub fn choose_variant(
- &self,
- sequence_length: usize,
- head_dim: usize,
- is_causal: bool,
- has_mask: bool,
- ) -> AttentionKernelVariant {
- if self.supports_two_pass(head_dim, is_causal, has_mask) && sequence_length > 1024 {
- AttentionKernelVariant::TwoPass
- } else {
- AttentionKernelVariant::SinglePass
+ cache,
}
}
- pub fn encode_single_pass(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: AttentionSinglePassArguments,
- ) -> Result<(), AttentionError> {
- let has_sinks = args.sinks_buffer.is_some();
- let has_mask = args.mask_buffer.is_some();
- let pipeline = self
- .pipelines
- .single_pass
- .get(&(args.head_dim, has_sinks, args.is_causal, has_mask))
- .ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;
-
- MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pipeline);
-
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.queries_buffer), 0, 0);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.keys_buffer), 0, 1);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.values_buffer), 0, 2);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.output_buffer), 0, 3);
-
- unsafe {
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.gqa_factor as *const i32 as *mut _),
- size_of::<i32>(),
- 4,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.sequence_length as *const i32 as *mut _),
- size_of::<i32>(),
- 5,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.k_head_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 6,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.k_seq_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 7,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.v_head_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 8,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.v_seq_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 9,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.scale as *const f32 as *mut _),
- size_of::<f32>(),
- 10,
- );
- }
-
- if let Some(mask_buffer) = args.mask_buffer {
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(mask_buffer), 0, 12); // float_mask
- unsafe {
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.mask_kv_seq_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 13,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.mask_q_seq_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 14,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.mask_head_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 15,
- );
- }
- }
-
- if let Some(sinks_buffer) = args.sinks_buffer {
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(sinks_buffer), 0, 16);
- }
-
- let threads_per_threadgroup = MTLSize {
- width: 32 * 32, // sequence_block_size * head_block_size
- height: 1,
- depth: 1,
- };
-
- let threadgroups_per_grid = MTLSize::new(args.num_heads, args.suffix_length, 1);
-
- compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);
- Ok(())
- }
-
- pub fn encode_two_pass(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: AttentionTwoPassArguments,
- ) -> Result<(), AttentionError> {
- let has_sinks = args.sinks_buffer.is_some();
- let has_mask = args.mask_buffer.is_some();
- let pass1_pipeline = self
- .pipelines
- .two_pass_1
- .get(&(args.head_dim, has_sinks, args.is_causal, has_mask))
- .ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;
-
- let pass2_pipeline = self
- .pipelines
- .two_pass_2
- .get(&args.head_dim)
- .ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;
-
- MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pass1_pipeline);
-
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.queries_buffer), 0, 0);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.keys_buffer), 0, 1);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.values_buffer), 0, 2);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.partials_buffer), 0, 3);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.sums_buffer), 0, 4);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.maxs_buffer), 0, 5);
-
- unsafe {
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.gqa_factor as *const i32 as *mut _),
- size_of::<i32>(),
- 6,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.sequence_length as *const i32 as *mut _),
- size_of::<i32>(),
- 7,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.k_head_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 8,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.k_seq_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 9,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.v_head_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 10,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.v_seq_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 11,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.scale as *const f32 as *mut _),
- size_of::<f32>(),
- 12,
- );
- }
-
- // Set mask buffer if present
- if let Some(mask_buffer) = args.mask_buffer {
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(mask_buffer), 0, 14); // float_mask
- unsafe {
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.mask_kv_seq_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 15,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.mask_q_seq_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 16,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&args.mask_head_stride as *const i32 as *mut _),
- size_of::<i32>(),
- 17,
- );
- }
- }
-
- if let Some(sinks_buffer) = args.sinks_buffer {
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(sinks_buffer), 0, 18);
- }
-
- let total_blocks_count = 32u64;
- let pass1_threads_per_threadgroup = MTLSize {
- width: 8 * 32, // sequence_block_size * head_block_size
- height: 1,
- depth: 1,
- };
- let pass1_threadgroups_per_grid = MTLSize::new(args.num_heads, args.suffix_length, total_blocks_count as usize);
-
- compute_encoder.dispatch_threadgroups(pass1_threadgroups_per_grid, pass1_threads_per_threadgroup);
-
- MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pass2_pipeline);
-
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.partials_buffer), 0, 0);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.sums_buffer), 0, 1);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.maxs_buffer), 0, 2);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.output_buffer), 0, 3);
-
- let pass2_threads_per_threadgroup = MTLSize {
- width: 32 * 32,
- height: 1,
- depth: 1,
- };
- let pass2_threadgroups_per_grid = MTLSize::new(args.num_heads, args.suffix_length, 1);
-
- compute_encoder.dispatch_threadgroups(pass2_threadgroups_per_grid, pass2_threads_per_threadgroup);
-
- Ok(())
- }
-
- pub fn encode_kv_cache_update(
- &self,
- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: KVCacheUpdateArguments,
- ) -> Result<(), AttentionError> {
- let pipeline = self
- .pipelines
- .kv_cache_update
- .as_ref()
- .ok_or_else(|| AttentionError::FunctionNotFound("KV cache update kernel".to_string()))?;
-
- MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pipeline);
-
- // Set buffers
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.rotated_keys_buffer), 0, 0);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.qkv_buffer), 0, 1);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.key_cache_buffer), 0, 2);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.value_cache_buffer), 0, 3);
-
- // Set constants
- unsafe {
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&(args.num_groups as i32) as *const i32 as *mut _),
- size_of::<i32>(),
- 4,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&(args.num_heads as i32) as *const i32 as *mut _),
- size_of::<i32>(),
- 5,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&(args.head_dim as i32) as *const i32 as *mut _),
- size_of::<i32>(),
- 6,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&(args.suffix_length as i32) as *const i32 as *mut _),
- size_of::<i32>(),
- 7,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&(args.segment_prefix_length as i32) as *const i32 as *mut _),
- size_of::<i32>(),
- 8,
- );
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&(args.max_sequence_length as i32) as *const i32 as *mut _),
- size_of::<i32>(),
- 9,
- );
- }
-
- let threads_per_grid = MTLSize::new(args.num_groups, args.suffix_length, args.head_dim as usize);
-
- let threadgroup_depth = std::cmp::min(args.head_dim.max(1), 64) as usize;
-
- MTLComputeCommandEncoder::dispatch_threads(
- compute_encoder,
- threads_per_grid,
- MTLSize {
- width: 1,
- height: 1,
- depth: threadgroup_depth,
- },
- );
- Ok(())
- }
-
- pub fn encode_gemm(
+ pub fn encode(
&self,
context: &MetalContext,
compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: AttentionGemmArguments,
- ) -> Result<(), AttentionError> {
- const BQ: usize = 32;
- const WM: u64 = 4;
- const WN: u64 = 1;
-
- if !matches!(args.head_dim, 64 | 128 | 256) {
- return Err(AttentionError::UnsupportedHeadDim(args.head_dim));
- }
-
+ args: &AttentionGemmArguments,
+ ) -> Result<(), MetalError> {
let bk: usize = if args.head_dim < 128 {
32
} else {
16
};
-
let align_q = (args.suffix_length % BQ) == 0;
let align_k = (args.sequence_length % bk) == 0;
let has_mask = args.mask_buffer.is_some();
let has_sinks = args.sinks_buffer.is_some();
+ let key = KernelKey {
+ bk,
+ head_dim: args.head_dim,
+ align_q,
+ align_k,
+ is_causal: args.is_causal,
+ has_mask,
+ has_sinks,
+ };
- let fcv = MTLFunctionConstantValues::new();
- fcv.set_value(&align_q, 200);
- fcv.set_value(&align_k, 201);
- fcv.set_value(&has_mask, 300);
- fcv.set_value(&args.is_causal, 301);
- fcv.set_value(&has_sinks, 302);
-
- // Kernel name matches gemm_attention.metal instantiations:
- // attention_gemm_{f16|bf16|f32}_{head_dim}_bk{bk}
- let type_name = dtype_suffix(self.data_type);
-
- let function_name = format!("attention_gemm_{}_{}_bk{}", type_name, args.head_dim, bk);
-
- let cache_key = format!(
- "{}_aq{}_ak{}_m{}_c{}_s{}",
- function_name, align_q as u8, align_k as u8, has_mask as u8, args.is_causal as u8, has_sinks as u8
- );
-
- let pipeline = context
- .compute_pipeline_state_cached(&cache_key, &function_name, Some(&fcv))
- .map_err(AttentionError::MetalError)?;
-
- MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, &pipeline);
-
- // Buffers
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.queries_buffer), 0, 0);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.keys_buffer), 0, 1);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.values_buffer), 0, 2);
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.output_buffer), 0, 3);
+ let mut map = self.cache.borrow_mut();
+ let kernel = match map.entry(key) {
+ Entry::Occupied(entry) => entry.into_mut(),
+ Entry::Vacant(entry) => {
+ let kernel = AttentionGemmMetalKernel::new(
+ context,
+ self.data_type,
+ bk as u32,
+ args.head_dim as u32,
+ align_q,
+ align_k,
+ args.is_causal,
+ has_mask,
+ has_sinks,
+ )?;
+ entry.insert(kernel)
+ },
+ };
// Params (all strides in elements)
let q_head_stride = (args.suffix_length * args.head_dim) as i64;
@@ -614,7 +105,6 @@ impl AttentionKernel {
let o_head_stride = args.head_dim as i64;
let o_seq_stride = (args.num_heads * args.head_dim) as i64;
- let nq = (args.suffix_length + BQ - 1) / BQ;
let nk = (args.sequence_length + bk - 1) / bk;
let nq_aligned = args.suffix_length / BQ;
let nk_aligned = args.sequence_length / bk;
@@ -636,45 +126,42 @@ impl AttentionKernel {
k_rem: (args.sequence_length - nk_aligned * bk) as i32,
};
- unsafe {
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&params as *const AttnParams as *mut _),
- size_of::<AttnParams>(),
- 4,
- );
- }
-
- if let Some(mask_buffer) = args.mask_buffer {
- let mask_params = AttnMaskParams {
- // We use a shared bias matrix for all heads/batches.
- m_strides: [0, 0, args.sequence_length as i64],
- };
- unsafe {
- MTLComputeCommandEncoder::set_bytes(
- compute_encoder,
- NonNull::new_unchecked(&mask_params as *const AttnMaskParams as *mut _),
- size_of::<AttnMaskParams>(),
- 5,
- );
- }
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(mask_buffer), 0, 6);
- }
-
- if let Some(sinks_buffer) = args.sinks_buffer {
- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(sinks_buffer), 0, 16);
- }
-
- // Dispatch
- let threadgroups_per_grid = MTLSize::new(nq as usize, args.num_heads, 1);
- let threads_per_threadgroup = MTLSize {
- width: 32,
- height: WM as usize,
- depth: WN as usize,
+ let attn_mask_params = AttnMaskParams {
+ // We use a shared bias matrix for all heads/batches.
+ m_strides: [0, 0, args.sequence_length as i64],
};
-
- compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);
+ let attn_mask_params_opt = args.mask_buffer.map(|_| std::slice::from_ref(&attn_mask_params));
+
+ let mask_buffer_retain = args.mask_buffer.map(|b| b.retain());
+ let mask_buffer_opt = mask_buffer_retain.as_ref().map(|b| b);
+ let sinks_buffer_retain = args.sinks_buffer.map(|b| b.retain());
+ let sinks_buffer_opt = sinks_buffer_retain.as_ref().map(|b| b);
+
+ kernel.encode(
+ &args.queries_buffer.retain(),
+ &args.keys_buffer.retain(),
+ &args.values_buffer.retain(),
+ &args.output_buffer.retain(),
+ std::slice::from_ref(&params),
+ attn_mask_params_opt,
+ mask_buffer_opt,
+ sinks_buffer_opt,
+ args.num_heads as u32,
+ args.suffix_length as u32,
+ &compute_encoder,
+ );
Ok(())
}
}
+
+#[derive(PartialEq, Eq, Hash, Clone, Copy)]
+struct KernelKey {
+ bk: usize,
+ head_dim: usize,
+ align_q: bool,
+ align_k: bool,
+ is_causal: bool,
+ has_mask: bool,
+ has_sinks: bool,
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 828bf05c5..e1400f79a 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -15,10 +15,6 @@ pub mod moe;
pub mod quant_matmul;
pub mod ssm;
-pub use attention::{
- AttentionError, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments, AttentionTwoPassArguments,
- KVCacheUpdateArguments,
-};
pub use matmul::{MatmulArguments, MatmulKernel};
pub use moe::{
MoeBlockBasesArguments, MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,
diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rs
index e4085aa52..3cec1dbd3 100644
--- a/crates/uzu/tests/attention_kernel_test.rs
+++ b/crates/uzu/tests/attention_kernel_test.rs
@@ -11,12 +11,15 @@ use objc2::{rc::Retained, runtime::ProtocolObject};
use uzu::{
DataType,
backends::{
- common::Context,
+ common::{
+ Context,
+ kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},
+ },
metal::{
MetalContext,
- kernel::attention::{
- AttentionGemmArguments, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments,
- AttentionTwoPassArguments,
+ kernel::{
+ attention::{AttentionGemmArguments, AttentionGemmBlock},
+ dsl::{AttentionSinglePassMetalKernel, AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel},
},
metal_extensions::CommandBufferTimingExt,
},
@@ -293,7 +296,7 @@ fn convert_kernel_output(
}
fn run_single_pass_attention(
- kernel: &AttentionKernel,
+ kernel: &AttentionSinglePassMetalKernel,
context: &MetalContext,
queries: &Array4<f32>,
keys: &Array4<f32>,
@@ -320,32 +323,37 @@ fn run_single_pass_attention(
let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- let args = AttentionSinglePassArguments {
- queries_buffer: &query_buffer,
- keys_buffer: &key_cache_buffer,
- values_buffer: &value_cache_buffer,
- output_buffer: &output_buffer,
- gqa_factor: (num_heads / num_kv_heads) as i32,
- sequence_length: seq_len as i32,
- k_head_stride: (seq_len * head_dim) as i32,
- k_seq_stride: head_dim as i32,
- v_head_stride: (seq_len * head_dim) as i32,
- v_seq_stride: head_dim as i32,
- scale,
- mask_buffer: mask_buffer.as_deref(),
- mask_kv_seq_stride: 1,
- mask_q_seq_stride: seq_len as i32,
- mask_head_stride: 0,
- sinks_buffer: sinks_buffer.as_deref(),
- num_heads,
- suffix_length: seq_len,
- head_dim,
- is_causal: false,
- };
+ let mut mask_kv_seq_stride: Option<u32> = None;
+ let mut mask_q_seq_stride: Option<u32> = None;
+ let mut mask_head_stride: Option<u32> = None;
+ if mask_buffer.is_some() {
+ mask_kv_seq_stride = Some(1);
+ mask_q_seq_stride = Some(seq_len as u32);
+ mask_head_stride = Some(0);
+ }
- let encode_result = kernel.encode_single_pass(&compute_encoder, args);
+ kernel.encode(
+ &query_buffer,
+ &key_cache_buffer,
+ &value_cache_buffer,
+ &output_buffer,
+ (num_heads / num_kv_heads) as u32,
+ seq_len as u32,
+ (seq_len * head_dim) as u32,
+ head_dim as u32,
+ (seq_len * head_dim) as u32,
+ head_dim as u32,
+ scale,
+ mask_buffer.as_ref().map(|b| b),
+ mask_kv_seq_stride,
+ mask_q_seq_stride,
+ mask_head_stride,
+ sinks_buffer.as_ref().map(|b| b),
+ num_heads as u32,
+ seq_len as u32,
+ &compute_encoder,
+ );
compute_encoder.end_encoding();
- encode_result?;
command_buffer.commit();
command_buffer.wait_until_completed();
@@ -359,7 +367,7 @@ fn run_single_pass_attention(
}
fn run_single_pass_attention_with_is_causal(
- kernel: &AttentionKernel,
+ kernel: &AttentionSinglePassMetalKernel,
context: &MetalContext,
queries: &Array4<f32>,
keys: &Array4<f32>,
@@ -367,7 +375,7 @@ fn run_single_pass_attention_with_is_causal(
mask: Option<&Array3<f32>>,
sinks: Option<&[f32]>,
scale: f32,
- is_causal: bool,
+ _is_causal: bool,
) -> Result<Array4<f32>, Box<dyn std::error::Error>> {
let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();
@@ -376,6 +384,7 @@ fn run_single_pass_attention_with_is_causal(
let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);
let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);
+ let has_mask = mask.is_some();
let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));
let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
@@ -387,32 +396,36 @@ fn run_single_pass_attention_with_is_causal(
let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- let args = AttentionSinglePassArguments {
- queries_buffer: &query_buffer,
- keys_buffer: &key_cache_buffer,
- values_buffer: &value_cache_buffer,
- output_buffer: &output_buffer,
- gqa_factor: (num_heads / num_kv_heads) as i32,
- sequence_length: seq_len as i32,
- k_head_stride: (seq_len * head_dim) as i32,
- k_seq_stride: head_dim as i32,
- v_head_stride: (seq_len * head_dim) as i32,
- v_seq_stride: head_dim as i32,
+ let mut mask_kv_seq_stride: Option<u32> = None;
+ let mut mask_q_seq_stride: Option<u32> = None;
+ let mut mask_head_stride: Option<u32> = None;
+ if has_mask {
+ mask_kv_seq_stride = Some(1);
+ mask_q_seq_stride = Some(seq_len as u32);
+ mask_head_stride = Some(0);
+ }
+ kernel.encode(
+ &query_buffer,
+ &key_cache_buffer,
+ &value_cache_buffer,
+ &output_buffer,
+ (num_heads / num_kv_heads) as u32,
+ seq_len as u32,
+ (seq_len * head_dim) as u32,
+ head_dim as u32,
+ (seq_len * head_dim) as u32,
+ head_dim as u32,
scale,
- mask_buffer: mask_buffer.as_deref(),
- mask_kv_seq_stride: 1,
- mask_q_seq_stride: seq_len as i32,
- mask_head_stride: 0,
- sinks_buffer: sinks_buffer.as_deref(),
- num_heads,
- suffix_length: seq_len,
- head_dim,
- is_causal,
- };
-
- let encode_result = kernel.encode_single_pass(&compute_encoder, args);
+ mask_buffer.as_ref().map(|b| b),
+ mask_kv_seq_stride,
+ mask_q_seq_stride,
+ mask_head_stride,
+ sinks_buffer.as_ref().map(|b| b),
+ num_heads as u32,
+ seq_len as u32,
+ &compute_encoder,
+ );
compute_encoder.end_encoding();
- encode_result?;
command_buffer.commit();
command_buffer.wait_until_completed();
@@ -426,7 +439,7 @@ fn run_single_pass_attention_with_is_causal(
}
fn run_gemm_attention(
- kernel: &AttentionKernel,
+ kernel: &AttentionGemmBlock,
context: &MetalContext,
queries: &Array4<f32>,
keys: &Array4<f32>,
@@ -472,7 +485,7 @@ fn run_gemm_attention(
scale,
};
- let encode_result = kernel.encode_gemm(context, &compute_encoder, args);
+ let encode_result = kernel.encode(context, &compute_encoder, &args);
compute_encoder.end_encoding();
encode_result?;
@@ -536,7 +549,6 @@ fn compare_results(
}
#[test]
-#[ignore]
fn test_single_pass_attention_basic() {
let context = MetalContext::new().expect("Failed to create MetalContext");
@@ -557,26 +569,10 @@ fn test_single_pass_attention_basic() {
println!("Reference output range (no mask): [{}, {}]", ref_min, ref_max);
println!("Reference sample values: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());
- let kernel = match AttentionKernel::new(&context, DataType::F32) {
- Ok(k) => k,
- Err(e) => {
- panic!("Failed to create AttentionKernel: {:?}", e);
- },
- };
-
let is_causal = false; // Non-causal attention for this test
- let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
- println!("Using kernel variant: {:?}", variant);
- println!(
- "Supports single-pass for head_dim={}: {}",
- head_dim,
- kernel.supports_single_pass(head_dim, is_causal, false)
- );
- println!("Supports two-pass for head_dim={}: {}", head_dim, kernel.supports_two_pass(head_dim, is_causal, false));
-
- if !kernel.supports_single_pass(head_dim, false, false) {
- panic!("Single-pass kernel not supported for head_dim={}", head_dim);
- }
+ let kernel =
+ AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, false, is_causal)
+ .expect("Failed to create attention single pass metal");
let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale)
{
@@ -593,7 +589,6 @@ fn test_single_pass_attention_basic() {
}
#[test]
-#[ignore]
fn test_gemm_attention_basic() {
let context = MetalContext::new().expect("Failed to create MetalContext");
@@ -606,7 +601,7 @@ fn test_gemm_attention_basic() {
let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
- let kernel = AttentionKernel::new(&context, DataType::F32).expect("Failed to create AttentionKernel");
+ let kernel = AttentionGemmBlock::new(DataType::F32);
let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
@@ -621,7 +616,6 @@ fn test_gemm_attention_basic() {
}
#[test]
-#[ignore]
fn test_gemm_attention_f32_head_dim_128() {
let context = MetalContext::new().expect("Failed to create MetalContext");
@@ -634,7 +628,7 @@ fn test_gemm_attention_f32_head_dim_128() {
let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);
- let kernel = AttentionKernel::new(&context, DataType::F32).expect("Failed to create AttentionKernel");
+ let kernel = AttentionGemmBlock::new(DataType::F32);
let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
@@ -649,7 +643,6 @@ fn test_gemm_attention_f32_head_dim_128() {
}
#[test]
-#[ignore]
fn test_matrix_attention_matches_vector_and_cpu_seq256() {
let context = MetalContext::new().expect("Failed to create MetalContext");
@@ -663,12 +656,13 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);
- let kernel = AttentionKernel::new(&context, DataType::F32).expect("Failed to create AttentionKernel");
-
// CPU reference (uses the same additive mask)
let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);
// vector attention path (single-pass)
+ let kernel =
+ AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, true, true, false, is_causal)
+ .expect("Failed to create attention single pass metal");
let vector_output = run_single_pass_attention_with_is_causal(
&kernel,
&context,
@@ -683,8 +677,9 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
.expect("run vector attention");
// matrix attention path (gemm)
+ let gemm_kernel = AttentionGemmBlock::new(DataType::F32);
let matrix_output =
- run_gemm_attention(&kernel, &context, &queries, &keys, &values, Some(&mask), None, scale, is_causal)
+ run_gemm_attention(&gemm_kernel, &context, &queries, &keys, &values, Some(&mask), None, scale, is_causal)
.expect("run matrix attention");
// Compare both to CPU
@@ -705,7 +700,6 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
}
#[test]
-#[ignore]
fn test_single_pass_attention_with_mask() {
let context = MetalContext::new().expect("Failed to create MetalContext");
@@ -718,17 +712,9 @@ fn test_single_pass_attention_with_mask() {
let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- let kernel = match AttentionKernel::new(&context, DataType::F32) {
- Ok(k) => k,
- Err(e) => {
- panic!("Failed to create AttentionKernel: {:?}", e);
- },
- };
-
- if !kernel.supports_single_pass(head_dim, false, true) {
- panic!("Single-pass kernel not supported for head_dim={}", head_dim);
- }
-
+ let kernel =
+ AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, true, true, false, false)
+ .expect("Failed to create AttentionSinglePassMetalKernel");
let kernel_output =
match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&mask), None, scale) {
Ok(output) => output,
@@ -754,7 +740,6 @@ fn test_single_pass_attention_with_mask() {
}
#[test]
-#[ignore]
fn test_single_pass_attention_with_sinks() {
let context = MetalContext::new().expect("Failed to create MetalContext");
@@ -770,16 +755,9 @@ fn test_single_pass_attention_with_sinks() {
let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();
println!("Using sinks: {:?}", sinks);
- let kernel = match AttentionKernel::new(&context, DataType::F32) {
- Ok(k) => k,
- Err(e) => {
- panic!("Failed to create AttentionKernel: {:?}", e);
- },
- };
-
- if !kernel.supports_single_pass(head_dim, false, false) {
- panic!("Single-pass kernel not supported for head_dim={}", head_dim);
- }
+ let kernel =
+ AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, true, false)
+ .expect("Failed to create AttentionSinglePassMetalKernel");
let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);
@@ -798,7 +776,6 @@ fn test_single_pass_attention_with_sinks() {
}
#[test]
-#[ignore]
fn test_single_pass_attention_with_sinks_long_sequence() {
let context = MetalContext::new().expect("Failed to create MetalContext");
@@ -814,16 +791,9 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();
println!("Long sequence sinks: {:?}", sinks);
- let kernel = match AttentionKernel::new(&context, DataType::F32) {
- Ok(k) => k,
- Err(e) => {
- panic!("Failed to create AttentionKernel: {:?}", e);
- },
- };
-
- if !kernel.supports_single_pass(head_dim, false, false) {
- panic!("Single-pass kernel not supported for head_dim={}", head_dim);
- }
+ let kernel =
+ AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, true, false)
+ .expect("Failed to create AttentionSinglePassMetalKernel");
let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);
@@ -847,7 +817,6 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
}
#[test]
-#[ignore]
fn test_single_pass_attention_gqa() {
let context = MetalContext::new().expect("Failed to create MetalContext");
@@ -860,20 +829,10 @@ fn test_single_pass_attention_gqa() {
let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- let kernel = match AttentionKernel::new(&context, DataType::F32) {
- Ok(k) => k,
- Err(e) => {
- panic!("Failed to create AttentionKernel: {:?}", e);
- },
- };
-
let is_causal = false; // Non-causal attention for this test
- let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
- println!("Using kernel variant for GQA: {:?}", variant);
-
- if !kernel.supports_single_pass(head_dim, false, false) {
- panic!("Single-pass kernel not supported for head_dim={}", head_dim);
- }
+ let kernel =
+ AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, false, is_causal)
+ .expect("Failed to create AttentionSinglePassMetalKernel");
let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale)
{
@@ -892,7 +851,8 @@ fn test_single_pass_attention_gqa() {
}
fn run_two_pass_attention(
- kernel: &AttentionKernel,
+ kernel_pass1: &AttentionTwoPass1MetalKernel,
+ kernel_pass2: &AttentionTwoPass2MetalKernel,
context: &MetalContext,
queries: &Array4<f32>,
keys: &Array4<f32>,
@@ -940,35 +900,47 @@ fn run_two_pass_attention(
let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- let args = AttentionTwoPassArguments {
- queries_buffer: &queries_buffer,
- keys_buffer: &keys_buffer,
- values_buffer: &values_buffer,
- partials_buffer: &partials_buffer,
- sums_buffer: &sums_buffer,
- maxs_buffer: &maxs_buffer,
- output_buffer: &output_buffer,
- gqa_factor: (num_heads / num_kv_heads) as i32,
- sequence_length: seq_len as i32,
- k_head_stride: (seq_len * head_dim) as i32,
- k_seq_stride: head_dim as i32,
- v_head_stride: (seq_len * head_dim) as i32,
- v_seq_stride: head_dim as i32,
+ let mut mask_kv_seq_stride: Option<u32> = None;
+ let mut mask_q_seq_stride: Option<u32> = None;
+ let mut mask_head_stride: Option<u32> = None;
+ if mask_buffer.is_some() {
+ mask_kv_seq_stride = Some(1);
+ mask_q_seq_stride = Some(seq_len as u32);
+ mask_head_stride = Some(0);
+ }
+ kernel_pass1.encode(
+ &queries_buffer,
+ &keys_buffer,
+ &values_buffer,
+ &partials_buffer,
+ &sums_buffer,
+ &maxs_buffer,
+ (num_heads / num_kv_heads) as u32,
+ seq_len as u32,
+ (seq_len * head_dim) as u32,
+ head_dim as u32,
+ (seq_len * head_dim) as u32,
+ head_dim as u32,
scale,
- mask_buffer: mask_buffer.as_deref(),
- mask_kv_seq_stride: 1,
- mask_q_seq_stride: seq_len as i32,
- mask_head_stride: 0,
- sinks_buffer: sinks_buffer.as_deref(),
- num_heads,
- suffix_length: seq_len,
- head_dim,
- is_causal: false,
- };
-
- let encode_result = kernel.encode_two_pass(&compute_encoder, args);
+ num_heads as u32,
+ seq_len as u32,
+ mask_buffer.as_ref().map(|b| b),
+ mask_kv_seq_stride,
+ mask_q_seq_stride,
+ mask_head_stride,
+ sinks_buffer.as_ref().map(|b| b),
+ &compute_encoder,
+ );
+ kernel_pass2.encode(
+ &partials_buffer,
+ &sums_buffer,
+ &maxs_buffer,
+ &output_buffer,
+ num_heads as u32,
+ seq_len as u32,
+ &compute_encoder,
+ );
compute_encoder.end_encoding();
- encode_result?;
command_buffer.commit();
command_buffer.wait_until_completed();
@@ -982,17 +954,9 @@ fn run_two_pass_attention(
}
#[test]
-#[ignore]
fn test_two_pass_attention() {
let context = MetalContext::new().expect("Failed to create MetalContext");
- let kernel = match AttentionKernel::new(&context, DataType::F32) {
- Ok(k) => k,
- Err(e) => {
- panic!("Failed to create AttentionKernel: {:?}", e);
- },
- };
-
let batch_size = 1;
let num_heads = 8;
let num_kv_heads: usize = 8;
@@ -1001,20 +965,26 @@ fn test_two_pass_attention() {
let scale = 1.0 / (head_dim as f32).sqrt();
let is_causal = false; // Non-causal attention for this test
- if !kernel.supports_two_pass(head_dim, is_causal, false) {
- panic!("Two-pass kernel not supported for head_dim={}", head_dim);
- }
-
- let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
- if !matches!(variant, AttentionKernelVariant::TwoPass) {
- panic!("Two-pass not selected for seq_len={}. Got {:?}", seq_len, variant);
- }
-
let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
- let kernel_output = match run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale) {
+ let kernel_pass1 =
+ AttentionTwoPass1MetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, false, is_causal)
+ .expect("Failed to create AttentionTwoPass1MetalKernel");
+ let kernel_pass2 = AttentionTwoPass2MetalKernel::new(&context, DataType::F32, head_dim as u32)
+ .expect("Failed to create AttentionTwoPass2MetalKernel");
+ let kernel_output = match run_two_pass_attention(
+ &kernel_pass1,
+ &kernel_pass2,
+ &context,
+ &queries,
+ &keys,
+ &values,
+ None,
+ None,
+ scale,
+ ) {
Ok(output) => output,
Err(e) => {
panic!("Failed to run two-pass attention: {:?}", e);
@@ -1028,17 +998,9 @@ fn test_two_pass_attention() {
}
#[test]
-#[ignore]
fn test_two_pass_attention_gqa() {
let context = MetalContext::new().expect("Failed to create MetalContext");
- let kernel = match AttentionKernel::new(&context, DataType::F32) {
- Ok(k) => k,
- Err(e) => {
- panic!("Failed to create AttentionKernel: {:?}", e);
- },
- };
-
let batch_size = 1;
let num_heads = 8;
let num_kv_heads = 2;
@@ -1047,20 +1009,26 @@ fn test_two_pass_attention_gqa() {
let scale = 1.0 / (head_dim as f32).sqrt();
let is_causal = false; // Non-causal attention for this test
- if !kernel.supports_two_pass(head_dim, is_causal, false) {
- panic!("Two-pass kernel not supported for head_dim={}", head_dim);
- }
-
- let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
- if !matches!(variant, AttentionKernelVariant::TwoPass) {
- panic!("Two-pass not selected for GQA seq_len={}. Got {:?}", seq_len, variant);
- }
-
let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
- let kernel_output = match run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale) {
+ let kernel_pass1 =
+ AttentionTwoPass1MetalKernel::new(&context, DataType::F32, head_dim as u32, is_causal, false, false, false)
+ .expect("Failed to create AttentionTwoPass1MetalKernel");
+ let kernel_pass2 = AttentionTwoPass2MetalKernel::new(&context, DataType::F32, head_dim as u32)
+ .expect("Failed to create AttentionTwoPass2MetalKernel");
+ let kernel_output = match run_two_pass_attention(
+ &kernel_pass1,
+ &kernel_pass2,
+ &context,
+ &queries,
+ &keys,
+ &values,
+ None,
+ None,
+ scale,
+ ) {
Ok(output) => output,
Err(e) => {
panic!("Failed to run two-pass attention GQA: {:?}", e);
@@ -1074,19 +1042,11 @@ fn test_two_pass_attention_gqa() {
}
#[test]
-#[ignore]
fn perf_two_pass_attention() {
use std::time::Instant;
let context = MetalContext::new().expect("Failed to create MetalContext");
- let kernel = match AttentionKernel::new(&context, DataType::F32) {
- Ok(k) => k,
- Err(e) => {
- panic!("Failed to create AttentionKernel: {:?}", e);
- },
- };
-
// ---- Problem sizes requiring two-pass ----
let batch_size = 1;
let num_heads = 32;
@@ -1097,17 +1057,6 @@ fn perf_two_pass_attention() {
let scale = 1.0 / (head_dim as f32).sqrt();
let is_causal = false;
- if !kernel.supports_two_pass(head_dim, is_causal, false) {
- println!("Skipping two-pass perf test: not supported for head_dim={}", head_dim);
- return;
- }
-
- let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);
- if !matches!(variant, AttentionKernelVariant::TwoPass) {
- println!("Skipping two-pass perf test: variant {:?} selected instead", variant);
- return;
- }
-
println!(
"Creating test data for two-pass performance test (prefix={}, suffix={})...",
seq_len - suffix_length,
@@ -1115,6 +1064,12 @@ fn perf_two_pass_attention() {
);
let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
+ let kernel_pass1 =
+ AttentionTwoPass1MetalKernel::new(&context, DataType::F32, head_dim as u32, is_causal, false, false, false)
+ .expect("Failed to create AttentionTwoPass1MetalKernel");
+ let kernel_pass2 = AttentionTwoPass2MetalKernel::new(&context, DataType::F32, head_dim as u32)
+ .expect("Failed to create AttentionTwoPass2MetalKernel");
+
// ---- Create buffers ----
// For realistic inference, we only process queries for the suffix (new tokens)
let queries_suffix = queries.slice(s![.., .., (seq_len - suffix_length).., ..]).to_owned();
@@ -1150,35 +1105,41 @@ fn perf_two_pass_attention() {
let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- let args = AttentionTwoPassArguments {
- queries_buffer: &queries_buffer,
- keys_buffer: &keys_buffer,
- values_buffer: &values_buffer,
- partials_buffer: &partials_buffer,
- sums_buffer: &sums_buffer,
- maxs_buffer: &maxs_buffer,
- output_buffer: &output_buffer,
- gqa_factor: (num_heads / num_kv_heads) as i32,
- sequence_length: seq_len as i32,
- k_head_stride: (seq_len * head_dim) as i32,
- k_seq_stride: head_dim as i32,
- v_head_stride: (seq_len * head_dim) as i32,
- v_seq_stride: head_dim as i32,
+ let mask_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;
+ let sinks_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;
+ kernel_pass1.encode(
+ &queries_buffer,
+ &keys_buffer,
+ &values_buffer,
+ &partials_buffer,
+ &sums_buffer,
+ &maxs_buffer,
+ (num_heads / num_kv_heads) as u32,
+ seq_len as u32,
+ (seq_len * head_dim) as u32,
+ head_dim as u32,
+ (seq_len * head_dim) as u32,
+ head_dim as u32,
scale,
- mask_buffer: None,
- mask_kv_seq_stride: 1,
- mask_q_seq_stride: suffix_length as i32,
- mask_head_stride: 0,
- sinks_buffer: None,
- num_heads,
- suffix_length, // Use actual suffix_length, not seq_len
- head_dim,
- is_causal: false,
- };
-
- let encode_result = kernel.encode_two_pass(&compute_encoder, args);
+ num_heads as u32,
+ suffix_length as u32,
+ mask_buffer.as_ref().map(|b| b),
+ None,
+ None,
+ None,
+ sinks_buffer.as_ref().map(|b| b),
+ &compute_encoder,
+ );
+ kernel_pass2.encode(
+ &partials_buffer,
+ &sums_buffer,
+ &maxs_buffer,
+ &output_buffer,
+ num_heads as u32,
+ suffix_length as u32,
+ &compute_encoder,
+ );
compute_encoder.end_encoding();
- encode_result.expect("encode");
// Time both host-side and GPU execution
let host_timer = Instant::now();