A high-performance inference engine for AI models
dslify-attention (#193)
Aleksandr Golokoz2500d54acf50
diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rsindex 532afecfc..bac0a64ec 100644--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs@@ -1,15 +1,25 @@//! Attention kernel encodable.+use std::collections::HashMap;+use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};use objc2::{rc::Retained, runtime::ProtocolObject};use crate::{DataType,- backends::metal::{- Metal, MetalContext,- kernel::attention::{- AttentionError, AttentionGemmArguments, AttentionKernel, AttentionKernelVariant,- AttentionSinglePassArguments, AttentionTwoPassArguments, KVCacheUpdateArguments,+ backends::{+ common::kernel::{+ AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,+ },+ metal::{+ Metal, MetalContext, MetalError,+ kernel::{+ attention::{AttentionGemmArguments, AttentionGemmBlock},+ dsl::{+ AttentionSinglePassMetalKernel, AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel,+ AttentionUpdateKVCacheMetalKernel,+ },+ },},},encodable_block::{EncodableBlock, EncodingParameters},@@ -33,7 +43,11 @@ fn env_gemm_attention_enabled() -> bool {}pub struct Attention {- kernel: AttentionKernel,+ single_pass_kernels: HashMap<KernelKey, AttentionSinglePassMetalKernel>,+ two_pass_1_kernels: HashMap<KernelKey, AttentionTwoPass1MetalKernel>,+ two_pass_2_kernels: HashMap<u32, AttentionTwoPass2MetalKernel>,+ update_kv_cache_kernel: AttentionUpdateKVCacheMetalKernel,+ gemm_block: AttentionGemmBlock,layer_index: usize,attention_scale: Option<f32>,has_sinks: bool,@@ -50,10 +64,44 @@ impl Attention {has_sinks: bool,is_causal: bool,sliding_window_size: Option<usize>,- ) -> Result<Self, AttentionError> {- let kernel = AttentionKernel::new(context, data_type)?;+ ) -> Result<Self, MetalError> {+ let mut single_pass_kernels = HashMap::new();+ let mut two_pass_1_kernels = HashMap::new();+ let mut two_pass_2_kernels = HashMap::new();++ let supported_head_dims = [64u32, 128u32, 256u32];+ for head_dim in supported_head_dims {+ for has_mask in [false, true] {+ let key = KernelKey {+ head_dim,+ has_mask,+ };+ let float_mask = has_mask;++ let sp_kernel = AttentionSinglePassMetalKernel::new(+ context, data_type, head_dim, float_mask, has_mask, has_sinks, is_causal,+ )?;+ single_pass_kernels.insert(key, sp_kernel);++ let tp1_kernel = AttentionTwoPass1MetalKernel::new(+ context, data_type, head_dim, float_mask, has_mask, has_sinks, is_causal,+ )?;+ two_pass_1_kernels.insert(key, tp1_kernel);++ let tp2_kernel = AttentionTwoPass2MetalKernel::new(context, data_type, head_dim)?;+ two_pass_2_kernels.insert(head_dim, tp2_kernel);+ }+ }++ let update_kv_cache_kernel = AttentionUpdateKVCacheMetalKernel::new(context, data_type)?;+ let gemm_block = AttentionGemmBlock::new(data_type);+Ok(Self {- kernel,+ single_pass_kernels,+ two_pass_1_kernels,+ two_pass_2_kernels,+ update_kv_cache_kernel,+ gemm_block,layer_index,attention_scale,has_sinks,@@ -61,6 +109,33 @@ impl Attention {sliding_window_size,})}++ fn select_variant(+ &self,+ gemm_enabled: bool,+ suffix_length: usize,+ head_dim: usize,+ sequence_length: usize,+ use_mask: bool,+ ) -> KernelVariant {+ let use_gemm = gemm_enabled && suffix_length > 8 && matches!(head_dim, 64 | 128 | 256);+ if use_gemm {+ return KernelVariant::Gemm;+ }++ let kernel_key = KernelKey {+ head_dim: head_dim as u32,+ has_mask: use_mask,+ };+ if sequence_length > 1024+ && self.two_pass_1_kernels.contains_key(&kernel_key)+ && self.two_pass_2_kernels.contains_key(&(head_dim as u32))+ {+ return KernelVariant::TwoPass;+ }++ KernelVariant::SinglePass+ }}impl EncodableBlock<Metal> for Attention {@@ -136,18 +211,13 @@ impl EncodableBlock<Metal> for Attention {let scale = self.attention_scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt());let gemm_enabled = env_gemm_attention_enabled();- let use_gemm = gemm_enabled && suffix_length > 8 && matches!(head_dim, 64 | 128 | 256);if !gemm_enabled {static PRINT_ONCE: std::sync::Once = std::sync::Once::new();PRINT_ONCE.call_once(|| {eprintln!("[uzu] Gemm attention disabled via UZU_USE_GEMM_ATTENTION");});}- let variant = if use_gemm {- AttentionKernelVariant::SinglePass- } else {- self.kernel.choose_variant(sequence_length, head_dim, self.is_causal, use_mask)- };+ let variant = self.select_variant(gemm_enabled, suffix_length, head_dim, sequence_length, use_mask);let rotated_queries_binding = state.arrays(&[ArrayId::RotatedQueries]);let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);@@ -191,23 +261,20 @@ impl EncodableBlock<Metal> for Attention {let extracted_values_buf = extracted_values_array.buffer().clone();// Reuse the KV cache update kernel to write values into extracted_values_buf.- if let Err(e) = self.kernel.encode_kv_cache_update(+ self.update_kv_cache_kernel.encode(+ rotated_keys_buffer,+ qkv_buffer,+ // keys already in desired layout; harmless overwrite+ rotated_keys_buffer,+ &extracted_values_buf,+ num_groups as u32,+ num_heads as u32,+ head_dim as u32,+ suffix_length as u32,+ 0u32,+ max_sequence_length as u32,&compute_encoder,- KVCacheUpdateArguments {- rotated_keys_buffer: &rotated_keys_buffer,- qkv_buffer: &qkv_buffer,- key_cache_buffer: &rotated_keys_buffer, // keys already in desired layout; harmless overwrite- value_cache_buffer: &extracted_values_buf,- num_groups,- num_heads,- head_dim,- suffix_length,- segment_prefix_length: 0,- max_sequence_length,- },- ) {- eprintln!("Failed to prepare rotated values buffer: {:?}", e);- }+ );(rotated_keys_buffer.clone(), extracted_values_buf)};@@ -252,24 +319,19 @@ impl EncodableBlock<Metal> for Attention {// Only update KV cache for LLM mode (not for classifiers)if has_kv_cache {- if let Err(e) = self.kernel.encode_kv_cache_update(- compute_encoder,- KVCacheUpdateArguments {- rotated_keys_buffer: &rotated_keys_buffer,- qkv_buffer: &qkv_buffer,- key_cache_buffer: &key_cache_buffer,- value_cache_buffer: &value_cache_buffer,- num_groups,- num_heads,- head_dim,- suffix_length,- segment_prefix_length,- max_sequence_length,- },- ) {- eprintln!("Failed to encode KV cache update: {:?}", e);- return;- }+ self.update_kv_cache_kernel.encode(+ rotated_keys_buffer,+ qkv_buffer,+ &key_cache_buffer,+ &value_cache_buffer,+ num_groups as u32,+ num_heads as u32,+ head_dim as u32,+ suffix_length as u32,+ segment_prefix_length as u32,+ max_sequence_length as u32,+ &compute_encoder,+ );}let k_head_stride = (max_sequence_length * head_dim) as i32;@@ -277,93 +339,129 @@ impl EncodableBlock<Metal> for Attention {let v_head_stride = (max_sequence_length * head_dim) as i32;let v_seq_stride = head_dim as i32;+ let kernel_key = KernelKey {+ head_dim: head_dim as u32,+ has_mask: use_mask,+ };++ let mut mask_kv_seq_stride_opt: Option<u32> = None;+ let mut mask_q_seq_stride_opt: Option<u32> = None;+ let mut mask_head_stride_opt: Option<u32> = None;+ if attention_bias_buffer.is_some() {+ mask_kv_seq_stride_opt = Some(mask_kv_seq_stride as u32);+ mask_q_seq_stride_opt = Some(mask_q_seq_stride as u32);+ mask_head_stride_opt = Some(mask_head_stride as u32);+ }+match variant {- AttentionKernelVariant::SinglePass => {- if use_gemm {- let mtl = state.mtl_context();- if let Err(e) = self.kernel.encode_gemm(- mtl,- compute_encoder,- AttentionGemmArguments {- queries_buffer: &queries_buffer,- keys_buffer: &key_cache_buffer,- values_buffer: &value_cache_buffer,- output_buffer: &attention_output_buffer,- mask_buffer: attention_bias_buffer.as_deref(),- sinks_buffer: sinks_buffer.as_deref(),- num_heads,- num_groups,- suffix_length,- sequence_length,- segment_prefix_length,- max_sequence_length,- head_dim,- is_causal: self.is_causal,- scale,- },- ) {- eprintln!("Failed to encode gemm attention: {:?}", e);- }- } else if let Err(e) = self.kernel.encode_single_pass(- compute_encoder,- AttentionSinglePassArguments {- queries_buffer: &queries_buffer,- keys_buffer: &key_cache_buffer,- values_buffer: &value_cache_buffer,- output_buffer: &attention_output_buffer,- gqa_factor: gqa_factor as i32,- sequence_length: sequence_length as i32,- k_head_stride,- k_seq_stride,- v_head_stride,- v_seq_stride,- scale,- mask_buffer: attention_bias_buffer.as_deref(),- mask_kv_seq_stride,- mask_q_seq_stride,- mask_head_stride,- sinks_buffer: sinks_buffer.as_deref(),- num_heads,- suffix_length,- head_dim,- is_causal: self.is_causal,- },- ) {- eprintln!("Failed to encode single-pass attention: {:?}", e);- }+ KernelVariant::Gemm => {+ let args = AttentionGemmArguments {+ queries_buffer: &queries_buffer,+ keys_buffer: &key_cache_buffer,+ values_buffer: &value_cache_buffer,+ output_buffer: &attention_output_buffer,+ mask_buffer: attention_bias_buffer.as_deref(),+ sinks_buffer: sinks_buffer.as_deref(),+ num_heads,+ num_groups,+ suffix_length,+ sequence_length,+ segment_prefix_length,+ max_sequence_length,+ head_dim,+ is_causal: self.is_causal,+ scale,+ };+ self.gemm_block+ .encode(state.mtl_context(), compute_encoder, &args)+ .expect("Failed to encode AttentionGemmBlock");+ },+ KernelVariant::SinglePass => {+ let kernel = match self.single_pass_kernels.get(&kernel_key) {+ Some(k) => k,+ None => panic!("Can not find AttentionSinglePassMetalKernel for key {:?}", kernel_key),+ };+ let mask_buffer_opt = attention_bias_buffer.as_ref();+ let sinks_buffer_opt = sinks_buffer.as_ref();+ kernel.encode(+ queries_buffer,+ &key_cache_buffer,+ &value_cache_buffer,+ attention_output_buffer,+ gqa_factor as u32,+ sequence_length as u32,+ k_head_stride as u32,+ k_seq_stride as u32,+ v_head_stride as u32,+ v_seq_stride as u32,+ scale,+ mask_buffer_opt,+ mask_kv_seq_stride_opt,+ mask_q_seq_stride_opt,+ mask_head_stride_opt,+ sinks_buffer_opt,+ num_heads as u32,+ suffix_length as u32,+ &compute_encoder,+ )},- AttentionKernelVariant::TwoPass => {- if let Err(e) = self.kernel.encode_two_pass(- compute_encoder,- AttentionTwoPassArguments {- queries_buffer: &queries_buffer,- keys_buffer: &key_cache_buffer,- values_buffer: &value_cache_buffer,- partials_buffer: &partials_buffer,- sums_buffer: &sums_buffer,- maxs_buffer: &maxs_buffer,- output_buffer: &attention_output_buffer,- gqa_factor: gqa_factor as i32,- sequence_length: sequence_length as i32,- k_head_stride,- k_seq_stride,- v_head_stride,- v_seq_stride,- scale,- mask_buffer: attention_bias_buffer.as_deref(),- mask_kv_seq_stride,- mask_q_seq_stride,- mask_head_stride,- sinks_buffer: sinks_buffer.as_deref(),- num_heads,- suffix_length,- head_dim,- is_causal: self.is_causal,- },- ) {- eprintln!("Failed to encode two-pass attention: {:?}", e);- }+ KernelVariant::TwoPass => {+ let kernel_pass1 = match self.two_pass_1_kernels.get(&kernel_key) {+ Some(k) => k,+ None => panic!("Can not find AttentionTwoPass1MetalKernel for key {:?}", kernel_key),+ };+ let kernel_pass2 = match self.two_pass_2_kernels.get(&(head_dim as u32)) {+ Some(k) => k,+ None => panic!("Can not find AttentionTwoPass2MetalKernel for key {:?}", kernel_key),+ };++ let mask_buffer_opt = attention_bias_buffer.as_ref().map(|b| b);+ let sinks_buffer_opt = sinks_buffer.as_ref().map(|b| b);+ kernel_pass1.encode(+ queries_buffer,+ &key_cache_buffer,+ &value_cache_buffer,+ partials_buffer,+ sums_buffer,+ maxs_buffer,+ gqa_factor as u32,+ sequence_length as u32,+ k_head_stride as u32,+ k_seq_stride as u32,+ v_head_stride as u32,+ v_seq_stride as u32,+ scale,+ num_heads as u32,+ suffix_length as u32,+ mask_buffer_opt,+ mask_kv_seq_stride_opt,+ mask_q_seq_stride_opt,+ mask_head_stride_opt,+ sinks_buffer_opt,+ &compute_encoder,+ );+ kernel_pass2.encode(+ partials_buffer,+ sums_buffer,+ maxs_buffer,+ attention_output_buffer,+ num_heads as u32,+ suffix_length as u32,+ &compute_encoder,+ );},}}}++enum KernelVariant {+ Gemm,+ SinglePass,+ TwoPass,+}++#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)]+struct KernelKey {+ pub head_dim: u32,+ pub has_mask: bool,+}diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention.metaldeleted file mode 100644index 115926208..000000000--- a/crates/uzu/src/backends/metal/kernel/attention/attention.metal+++ /dev/null@@ -1,872 +0,0 @@-#include <metal_stdlib>-#include <metal_simdgroup>-#include "../definitions.metal"--using namespace metal;--constant bool has_mask [[function_constant(20)]];-constant bool query_transposed [[function_constant(21)]];-constant bool do_causal [[function_constant(22)]];-constant bool bool_mask [[function_constant(23)]];-constant bool float_mask [[function_constant(24)]];-constant bool has_sinks [[function_constant(25)]];--template <typename T, int head_dim, int value_dim = head_dim>-void attention_single_pass_impl(- const device T* queries,- const device T* keys,- const device T* values,- device T* out,- const constant int& gqa_factor,- const constant int& sequence_length,- const constant int& k_head_stride,- const constant int& k_seq_stride,- const constant int& v_head_stride,- const constant int& v_seq_stride,- const constant float& scale,- const device bool* bmask,- const device T* fmask,- const constant int& mask_kv_seq_stride,- const constant int& mask_q_seq_stride,- const constant int& mask_head_stride,- const device float* sinks,- uint3 tid, // threadgroup position in grid- uint3 tpg, // threadgroups per grid- uint simd_gid, // simdgroup index in threadgroup- uint simd_lid, // thread index in simdgroup- threadgroup float* shared_max_scores,- threadgroup float* shared_sum_exp_scores,- threadgroup float* shared_outputs-) {- constexpr int sequence_block_size = 32;- constexpr int head_block_size = 32;- constexpr int qk_elements_per_thread = head_dim / head_block_size;- constexpr int value_elements_per_thread = value_dim / head_block_size;- int inner_k_stride = sequence_block_size * int(k_seq_stride);- int inner_v_stride = sequence_block_size * int(v_seq_stride);-- typedef float U;-- thread U q[qk_elements_per_thread];- thread U k[qk_elements_per_thread];- thread U o[value_elements_per_thread];-- const int head_idx = tid.x;- const int q_seq_idx = tid.y;- const int kv_head_idx = head_idx / gqa_factor;- const int o_offset = q_seq_idx * tpg.x + head_idx;- const int q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx- : head_idx * tpg.y + q_seq_idx;-- queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;- keys += kv_head_idx * k_head_stride + simd_gid * k_seq_stride +- simd_lid * qk_elements_per_thread;- values += kv_head_idx * v_head_stride + simd_gid * v_seq_stride +- simd_lid * value_elements_per_thread;-- if (bool_mask) {- bmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +- q_seq_idx * mask_q_seq_stride;- }- if (float_mask) {- fmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +- q_seq_idx * mask_q_seq_stride;- }-- out += o_offset * value_dim + simd_gid * value_elements_per_thread;-- // Read the query and 0 the output accumulator- for (int i = 0; i < qk_elements_per_thread; i++) {- q[i] = static_cast<U>(scale) * queries[i];- }- for (int i = 0; i < value_elements_per_thread; i++) {- o[i] = 0;- }-- U max_score = -INFINITY;- U sum_exp_score = 0;- if (has_sinks && simd_gid == 0) {- const int num_q_heads = static_cast<int>(tpg.x);- int q_head_idx = head_idx % num_q_heads;- max_score = static_cast<U>(sinks[q_head_idx]);- sum_exp_score = 1;- }-- // For each key- for (int i = simd_gid; i < sequence_length; i += sequence_block_size) {- bool use_key = true;- if (do_causal) {- use_key = i <= (sequence_length - int(tpg.y) + int(q_seq_idx));- } else if (bool_mask) {- use_key = bmask[0];- }-- if (use_key) {- // Read the key- for (int j = 0; j < qk_elements_per_thread; j++) {- k[j] = keys[j];- }-- // Compute the i-th score- U score = 0;- for (int j = 0; j < qk_elements_per_thread; j++) {- score += q[j] * k[j];- }- score = simd_sum(score);- if (float_mask) {- score += max(-1e9f, static_cast<U>(fmask[0]));- }-- // Update the accumulators- U new_max = max(max_score, score);- U factor = fast::exp(max_score - new_max);- U exp_score = fast::exp(score - new_max);-- max_score = new_max;- sum_exp_score = sum_exp_score * factor + exp_score;-- // Update the output accumulator- for (int j = 0; j < value_elements_per_thread; j++) {- o[j] = o[j] * factor + exp_score * values[j];- }- }-- // Move the pointers to the next kv- keys += inner_k_stride;- values += inner_v_stride;- if (bool_mask) {- bmask += sequence_block_size * mask_kv_seq_stride;- }- if (float_mask) {- fmask += sequence_block_size * mask_kv_seq_stride;- }- }-- // Each thread has a partial part of the output so we need to combine them.- if (simd_lid == 0) {- shared_max_scores[simd_gid] = max_score;- shared_sum_exp_scores[simd_gid] = sum_exp_score;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- max_score = shared_max_scores[simd_lid];- U new_max = simd_max(max_score);- U factor = fast::exp(max_score - new_max);- sum_exp_score = simd_sum(shared_sum_exp_scores[simd_lid] * factor);-- // Now we need to aggregate all the outputs- for (int i = 0; i < value_elements_per_thread; i++) {- shared_outputs[simd_lid * head_block_size + simd_gid] = o[i];- threadgroup_barrier(mem_flags::mem_threadgroup);- o[i] = simd_sum(- shared_outputs[simd_gid * head_block_size + simd_lid] * factor- ) /- sum_exp_score;- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- // And write the output- if (simd_lid == 0) {- for (int i = 0; i < value_elements_per_thread; i++) {- out[i] = static_cast<T>(o[i]);- }- }-}--template <typename T, int head_dim, int value_dim = head_dim>-void attention_2pass_1_impl(- const device T* queries,- const device T* keys,- const device T* values,- device float* out,- device float* sums,- device float* maxs,- const constant int& gqa_factor,- const constant int& sequence_length,- const constant int& k_head_stride,- const constant int& k_seq_stride,- const constant int& v_head_stride,- const constant int& v_seq_stride,- const constant float& scale,- const device bool* bmask,- const device T* fmask,- const constant int& mask_kv_seq_stride,- const constant int& mask_q_seq_stride,- const constant int& mask_head_stride,- const device float* sinks,- uint3 tid, // threadgroup position in grid- uint3 tpg, // threadgroups per grid- uint simd_gid, // simdgroup index in threadgroup- uint simd_lid, // thread index in simdgroup- threadgroup float* shared_max_scores,- threadgroup float* shared_sum_exp_scores,- threadgroup float* shared_outputs-) {- constexpr int sequence_block_size = 8;- constexpr int head_block_size = 32;- constexpr int qk_elements_per_thread = head_dim / head_block_size;- constexpr int value_elements_per_thread = value_dim / head_block_size;- int inner_k_stride = sequence_block_size * int(k_seq_stride);- int inner_v_stride = sequence_block_size * int(v_seq_stride);- constexpr int total_blocks_count = 32;-- typedef float U;-- thread U q[qk_elements_per_thread];- thread U k[qk_elements_per_thread];- thread U o[value_elements_per_thread];-- const int block_idx = tid.z;- const int head_idx = tid.x;- const int q_seq_idx = tid.y;- const int o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout- const int q_offset =- query_transposed- ? tpg.x * q_seq_idx + head_idx- : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass- const int kv_head_idx = head_idx / gqa_factor;-- queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;- keys += kv_head_idx * k_head_stride +- (block_idx * sequence_block_size + simd_gid) * k_seq_stride +- simd_lid * qk_elements_per_thread;- values += kv_head_idx * v_head_stride +- (block_idx * sequence_block_size + simd_gid) * v_seq_stride +- simd_lid * value_elements_per_thread;- out += o_offset * total_blocks_count * value_dim + block_idx * value_dim +- simd_lid * value_elements_per_thread;-- if (bool_mask) {- bmask += head_idx * mask_head_stride +- (block_idx * sequence_block_size + simd_gid) * mask_kv_seq_stride +- q_seq_idx * mask_q_seq_stride;- }- if (float_mask) {- fmask += head_idx * mask_head_stride +- (block_idx * sequence_block_size + simd_gid) * mask_kv_seq_stride +- q_seq_idx * mask_q_seq_stride;- }- sums += o_offset * total_blocks_count + block_idx;- maxs += o_offset * total_blocks_count + block_idx;-- // Read the query and 0 the output accumulator- for (int i = 0; i < qk_elements_per_thread; i++) {- q[i] = static_cast<U>(scale) * queries[i];- }- for (int i = 0; i < value_elements_per_thread; i++) {- o[i] = 0;- }-- U max_score = -1e9;- U sum_exp_score = 0;- if (has_sinks && block_idx == 0 && simd_gid == 0) {- const int num_q_heads = static_cast<int>(tpg.x);- int q_head_idx = head_idx % num_q_heads;- max_score = static_cast<U>(sinks[q_head_idx]);- sum_exp_score = 1;- }-- // For each key- for (int i = block_idx * sequence_block_size + simd_gid; i < sequence_length;- i += total_blocks_count * sequence_block_size) {- bool use_key = true;- if (do_causal) {- use_key = i <= (sequence_length - int(tpg.y) + int(q_seq_idx));- } else if (bool_mask) {- use_key = bmask[0];- }-- if (use_key) {- // Read the key- for (int j = 0; j < qk_elements_per_thread; j++) {- k[j] = keys[j];- }-- // Compute the i-th score- U score = 0;- for (int j = 0; j < qk_elements_per_thread; j++) {- score += q[j] * k[j];- }- score = simd_sum(score);- if (float_mask) {- score += max(-1e9f, static_cast<U>(fmask[0]));- }-- // Update the accumulators- U new_max = max(max_score, score);- U factor = fast::exp(max_score - new_max);- U exp_score = fast::exp(score - new_max);-- max_score = new_max;- sum_exp_score = sum_exp_score * factor + exp_score;-- // Update the output accumulator- for (int j = 0; j < value_elements_per_thread; j++) {- o[j] = o[j] * factor + exp_score * values[j];- }- }-- // Move the pointers to the next kv- keys += total_blocks_count * inner_k_stride;- values += total_blocks_count * inner_v_stride;- if (bool_mask) {- bmask += sequence_block_size * total_blocks_count * mask_kv_seq_stride;- }- if (float_mask) {- fmask += sequence_block_size * total_blocks_count * mask_kv_seq_stride;- }- }-- // Each thread has a partial part of the output so we need to combine them.- if (simd_lid == 0) {- shared_max_scores[simd_gid] = max_score;- shared_sum_exp_scores[simd_gid] = sum_exp_score;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- max_score =- (simd_lid < sequence_block_size) ? shared_max_scores[simd_lid] : -1e9;- U new_max = simd_max(max_score);- U factor = fast::exp(max_score - new_max);- sum_exp_score =- (simd_lid < sequence_block_size) ? shared_sum_exp_scores[simd_lid] : 0;- sum_exp_score = simd_sum(sum_exp_score * factor);-- // Write the sum and new max- if (simd_gid == 0) {- sums[0] = sum_exp_score;- maxs[0] = new_max;- }-- // Now we need to aggregate all the outputs- for (int i = 0; i < value_elements_per_thread; i++) {- shared_outputs[simd_lid * sequence_block_size + simd_gid] =- o[i] * fast::exp(shared_max_scores[simd_gid] - new_max);- threadgroup_barrier(mem_flags::mem_threadgroup);-- // And write the output- if (simd_gid == 0) {- U output = shared_outputs[simd_lid * sequence_block_size];- for (int j = 1; j < sequence_block_size; j++) {- output += shared_outputs[simd_lid * sequence_block_size + j];- }- out[i] = static_cast<T>(output);- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }-}--template <typename T, int head_dim>-void attention_2pass_2_impl(- const device float* partials,- const device float* sums,- const device float* maxs,- device T* out,- uint3 tid,- uint3 tpg,- uint simd_gid,- uint simd_lid,- threadgroup float* shared_outputs-) {- constexpr int sequence_block_size = 32;- constexpr int head_block_size = 32;- constexpr int elements_per_thread = head_dim / head_block_size;- constexpr int total_blocks_count = 32;-- typedef float U;-- thread U o[elements_per_thread];-- const int head_idx = tid.x;- const int q_seq_idx = tid.y;- const int o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout- const int q_offset =- query_transposed- ? tpg.x * q_seq_idx + head_idx- : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass-- partials += o_offset * total_blocks_count * head_dim + simd_gid * head_dim +- simd_lid * elements_per_thread;- sums += o_offset * total_blocks_count;- maxs += o_offset * total_blocks_count;- out += o_offset * head_dim +- simd_gid * elements_per_thread; // Our custom output layout-- // First everybody reads the max and sum_exp- U max_score = maxs[simd_lid];- U new_max = simd_max(max_score);- U factor = fast::exp(max_score - new_max);- U sum_exp_score = simd_sum(sums[simd_lid] * factor);-- // Now read the block into registers and then use shared memory to transpose- // it- for (int i = 0; i < elements_per_thread; i++) {- o[i] = partials[i];- }- for (int i = 0; i < elements_per_thread; i++) {- shared_outputs[simd_lid * head_block_size + simd_gid] = o[i];- threadgroup_barrier(mem_flags::mem_threadgroup);- o[i] = simd_sum(- shared_outputs[simd_gid * head_block_size + simd_lid] * factor- ) /- sum_exp_score;- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- // And write the output- if (simd_lid == 0) {- for (int i = 0; i < elements_per_thread; i++) {- out[i] = static_cast<T>(o[i]);- }- }-}--template <typename T, int head_dim, int value_dim = head_dim>-void attention_single_pass(- const device T* queries,- const device T* keys,- const device T* values,- device T* out,- const constant int& gqa_factor,- const constant int& sequence_length,- const constant int& k_head_stride,- const constant int& k_seq_stride,- const constant int& v_head_stride,- const constant int& v_seq_stride,- const constant float& scale,- const device bool* bmask,- const device T* fmask,- const constant int& mask_kv_seq_stride,- const constant int& mask_q_seq_stride,- const constant int& mask_head_stride,- const device float* sinks,- uint3 tid,- uint3 tpg,- uint simd_gid,- uint simd_lid,- threadgroup float* shared_max_scores,- threadgroup float* shared_sum_exp_scores,- threadgroup float* shared_outputs-) {- attention_single_pass_impl<T, head_dim, value_dim>(- queries,- keys,- values,- out,- gqa_factor,- sequence_length,- k_head_stride,- k_seq_stride,- v_head_stride,- v_seq_stride,- scale,- bmask,- fmask,- mask_kv_seq_stride,- mask_q_seq_stride,- mask_head_stride,- sinks,- tid,- tpg,- simd_gid,- simd_lid,- shared_max_scores,- shared_sum_exp_scores,- shared_outputs- );-}--template <typename T, int head_dim, int value_dim = head_dim>-void attention_2pass_1(- const device T* queries,- const device T* keys,- const device T* values,- device float* out,- device float* sums,- device float* maxs,- const constant int& gqa_factor,- const constant int& sequence_length,- const constant int& k_head_stride,- const constant int& k_seq_stride,- const constant int& v_head_stride,- const constant int& v_seq_stride,- const constant float& scale,- const device bool* bmask,- const device T* fmask,- const constant int& mask_kv_seq_stride,- const constant int& mask_q_seq_stride,- const constant int& mask_head_stride,- const device float* sinks,- uint3 tid,- uint3 tpg,- uint simd_gid,- uint simd_lid,- threadgroup float* shared_max_scores,- threadgroup float* shared_sum_exp_scores,- threadgroup float* shared_outputs-) {- attention_2pass_1_impl<T, head_dim, value_dim>(- queries,- keys,- values,- out,- sums,- maxs,- gqa_factor,- sequence_length,- k_head_stride,- k_seq_stride,- v_head_stride,- v_seq_stride,- scale,- bmask,- fmask,- mask_kv_seq_stride,- mask_q_seq_stride,- mask_head_stride,- sinks,- tid,- tpg,- simd_gid,- simd_lid,- shared_max_scores,- shared_sum_exp_scores,- shared_outputs- );-}--template <typename T, int head_dim>-void attention_2pass_2(- const device float* partials,- const device float* sums,- const device float* maxs,- device T* out,- uint3 tid,- uint3 tpg,- uint simd_gid,- uint simd_lid,- threadgroup float* shared_outputs-) {- attention_2pass_2_impl<T, head_dim>(- partials,- sums,- maxs,- out,- tid,- tpg,- simd_gid,- simd_lid,- shared_outputs- );-}--#define outerArguments(T) \- (const device T* queries [[buffer(0)]], \- const device T* keys [[buffer(1)]], \- const device T* values [[buffer(2)]], \- device T* out [[buffer(3)]], \- const constant int& gqa_factor [[buffer(4)]], \- const constant int& sequence_length [[buffer(5)]], \- const constant int& k_head_stride [[buffer(6)]], \- const constant int& k_seq_stride [[buffer(7)]], \- const constant int& v_head_stride [[buffer(8)]], \- const constant int& v_seq_stride [[buffer(9)]], \- const constant float& scale [[buffer(10)]], \- const device bool* bmask [[buffer(11), function_constant(bool_mask)]], \- const device T* fmask [[buffer(12), function_constant(float_mask)]], \- const constant int& mask_kv_seq_stride \- [[buffer(13), function_constant(has_mask)]], \- const constant int& mask_q_seq_stride \- [[buffer(14), function_constant(has_mask)]], \- const constant int& mask_head_stride \- [[buffer(15), function_constant(has_mask)]], \- const device float* sinks [[buffer(16), function_constant(has_sinks)]], \- uint3 tid [[threadgroup_position_in_grid]], \- uint3 tpg [[threadgroups_per_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]])--#define innerArguments \- (queries, \- keys, \- values, \- out, \- gqa_factor, \- sequence_length, \- k_head_stride, \- k_seq_stride, \- v_head_stride, \- v_seq_stride, \- scale, \- bmask, \- fmask, \- mask_kv_seq_stride, \- mask_q_seq_stride, \- mask_head_stride, \- sinks, \- tid, \- tpg, \- simd_gid, \- simd_lid, \- shared_max_scores, \- shared_sum_exp_scores, \- shared_outputs)--// Generate single-pass kernels for different head dimensions-#define GENERATE_SINGLE_PASS_KERNELS(head_dim_value) \- [[max_total_threads_per_threadgroup( \- 1024 \- )]] kernel void attention_single_pass_half_##head_dim_value \- outerArguments(half) { \- constexpr int sequence_block_size = 32; \- constexpr int head_block_size = 32; \- threadgroup float shared_max_scores[sequence_block_size]; \- threadgroup float shared_sum_exp_scores[sequence_block_size]; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_single_pass<half, head_dim_value> innerArguments; \- } \- [[max_total_threads_per_threadgroup( \- 1024 \- )]] kernel void attention_single_pass_bfloat_##head_dim_value \- outerArguments(bfloat) { \- constexpr int sequence_block_size = 32; \- constexpr int head_block_size = 32; \- threadgroup float shared_max_scores[sequence_block_size]; \- threadgroup float shared_sum_exp_scores[sequence_block_size]; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_single_pass<bfloat, head_dim_value> innerArguments; \- } \- [[max_total_threads_per_threadgroup( \- 1024 \- )]] kernel void attention_single_pass_float_##head_dim_value \- outerArguments(float) { \- constexpr int sequence_block_size = 32; \- constexpr int head_block_size = 32; \- threadgroup float shared_max_scores[sequence_block_size]; \- threadgroup float shared_sum_exp_scores[sequence_block_size]; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_single_pass<float, head_dim_value> innerArguments; \- }--GENERATE_SINGLE_PASS_KERNELS(64)-GENERATE_SINGLE_PASS_KERNELS(128)-GENERATE_SINGLE_PASS_KERNELS(256)--#undef outerArguments-#undef innerArguments--#define outerArguments(T) \- (const device T* queries [[buffer(0)]], \- const device T* keys [[buffer(1)]], \- const device T* values [[buffer(2)]], \- device float* out [[buffer(3)]], \- device float* sums [[buffer(4)]], \- device float* maxs [[buffer(5)]], \- const constant int& gqa_factor [[buffer(6)]], \- const constant int& sequence_length [[buffer(7)]], \- const constant int& k_head_stride [[buffer(8)]], \- const constant int& k_seq_stride [[buffer(9)]], \- const constant int& v_head_stride [[buffer(10)]], \- const constant int& v_seq_stride [[buffer(11)]], \- const constant float& scale [[buffer(12)]], \- const device bool* bmask [[buffer(13), function_constant(bool_mask)]], \- const device T* fmask [[buffer(14), function_constant(float_mask)]], \- const constant int& mask_kv_seq_stride \- [[buffer(15), function_constant(has_mask)]], \- const constant int& mask_q_seq_stride \- [[buffer(16), function_constant(has_mask)]], \- const constant int& mask_head_stride \- [[buffer(17), function_constant(has_mask)]], \- const device float* sinks [[buffer(18), function_constant(has_sinks)]], \- uint3 tid [[threadgroup_position_in_grid]], \- uint3 tpg [[threadgroups_per_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]])--#define innerArguments \- (queries, \- keys, \- values, \- out, \- sums, \- maxs, \- gqa_factor, \- sequence_length, \- k_head_stride, \- k_seq_stride, \- v_head_stride, \- v_seq_stride, \- scale, \- bmask, \- fmask, \- mask_kv_seq_stride, \- mask_q_seq_stride, \- mask_head_stride, \- sinks, \- tid, \- tpg, \- simd_gid, \- simd_lid, \- shared_max_scores, \- shared_sum_exp_scores, \- shared_outputs)--// Generate 2-pass Pass 1 kernels for different head dimensions-#define GENERATE_2PASS_1_KERNELS(head_dim_value) \- [[max_total_threads_per_threadgroup( \- 256 \- )]] kernel void attention_2pass_1_half_##head_dim_value \- outerArguments(half) { \- constexpr int sequence_block_size = 8; \- constexpr int head_block_size = 32; \- threadgroup float shared_max_scores[sequence_block_size]; \- threadgroup float shared_sum_exp_scores[sequence_block_size]; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_2pass_1<half, head_dim_value> innerArguments; \- } \- [[max_total_threads_per_threadgroup( \- 256 \- )]] kernel void attention_2pass_1_bfloat_##head_dim_value \- outerArguments(bfloat) { \- constexpr int sequence_block_size = 8; \- constexpr int head_block_size = 32; \- threadgroup float shared_max_scores[sequence_block_size]; \- threadgroup float shared_sum_exp_scores[sequence_block_size]; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_2pass_1<bfloat, head_dim_value> innerArguments; \- } \- [[max_total_threads_per_threadgroup( \- 256 \- )]] kernel void attention_2pass_1_float_##head_dim_value \- outerArguments(float) { \- constexpr int sequence_block_size = 8; \- constexpr int head_block_size = 32; \- threadgroup float shared_max_scores[sequence_block_size]; \- threadgroup float shared_sum_exp_scores[sequence_block_size]; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_2pass_1<float, head_dim_value> innerArguments; \- }--GENERATE_2PASS_1_KERNELS(64)-GENERATE_2PASS_1_KERNELS(128)-GENERATE_2PASS_1_KERNELS(256)--#undef outerArguments-#undef innerArguments--#define outerArguments(T) \- (const device float* partials [[buffer(0)]], \- const device float* sums [[buffer(1)]], \- const device float* maxs [[buffer(2)]], \- device T* out [[buffer(3)]], \- uint3 tid [[threadgroup_position_in_grid]], \- uint3 tpg [[threadgroups_per_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]])--#define innerArguments \- (partials, sums, maxs, out, tid, tpg, simd_gid, simd_lid, shared_outputs)--// Generate 2-pass Pass 2 kernels for different head dimensions-#define GENERATE_2PASS_2_KERNELS(head_dim_value) \- [[max_total_threads_per_threadgroup( \- 1024 \- )]] kernel void attention_2pass_2_half_##head_dim_value \- outerArguments(half) { \- constexpr int sequence_block_size = 32; \- constexpr int head_block_size = 32; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_2pass_2<half, head_dim_value> innerArguments; \- } \- [[max_total_threads_per_threadgroup( \- 1024 \- )]] kernel void attention_2pass_2_float_##head_dim_value \- outerArguments(float) { \- constexpr int sequence_block_size = 32; \- constexpr int head_block_size = 32; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_2pass_2<float, head_dim_value> innerArguments; \- }--GENERATE_2PASS_2_KERNELS(64)-GENERATE_2PASS_2_KERNELS(128)-GENERATE_2PASS_2_KERNELS(256)--#undef outerArguments-#undef innerArguments--template <typename T>-void update_kv_cache(- const device T* rotated_keys, // [num_groups, suffix_length, head_dim]- const device T*- qkv, // [suffix_length, (num_heads + 2*num_groups) * head_dim]- device T* key_cache, // [num_groups, max_sequence_length, head_dim]- device T* value_cache, // [num_groups, max_sequence_length, head_dim]- const constant int& num_groups,- const constant int& num_heads,- const constant int& head_dim,- const constant int& suffix_length,- const constant int& prefix_segment_length,- const constant int& max_sequence_length,- uint3 position // x: group_idx, y: token_idx, z: dim_idx-) {- const uint groupIndex = position.x;- const uint tokenIndex = position.y;- const uint dimIndex = position.z;-- if (groupIndex >= num_groups || tokenIndex >= suffix_length ||- dimIndex >= head_dim) {- return;- }-- const uint cacheTokenIndex = prefix_segment_length + tokenIndex;-- // Copy rotated key to cache- const uint rotatedKeyOffset =- (groupIndex * suffix_length + tokenIndex) * head_dim + dimIndex;- const uint keyCacheOffset =- (groupIndex * max_sequence_length + cacheTokenIndex) * head_dim +- dimIndex;- key_cache[keyCacheOffset] = rotated_keys[rotatedKeyOffset];-- // Copy value to cache- // qkv layout: [suffix_length, (num_heads + 2*num_groups) * head_dim]- // Values start at offset: (num_heads * head_dim) + (num_groups * head_dim)- const uint qkvStride = (num_heads + 2 * num_groups) * head_dim;- const uint valueBaseOffset = (num_heads + num_groups) * head_dim;- const uint valueOffset = tokenIndex * qkvStride + valueBaseOffset +- groupIndex * head_dim + dimIndex;- const uint valueCacheOffset =- (groupIndex * max_sequence_length + cacheTokenIndex) * head_dim +- dimIndex;- value_cache[valueCacheOffset] = qkv[valueOffset];-}--#define outerArguments(T) \- (const device T* rotated_keys [[buffer(0)]], \- const device T* qkv [[buffer(1)]], \- device T* key_cache [[buffer(2)]], \- device T* value_cache [[buffer(3)]], \- const constant int& num_groups [[buffer(4)]], \- const constant int& num_heads [[buffer(5)]], \- const constant int& head_dim [[buffer(6)]], \- const constant int& suffix_length [[buffer(7)]], \- const constant int& prefix_segment_length [[buffer(8)]], \- const constant int& max_sequence_length [[buffer(9)]], \- uint3 position [[thread_position_in_grid]])--#define innerArguments \- (rotated_keys, \- qkv, \- key_cache, \- value_cache, \- num_groups, \- num_heads, \- head_dim, \- suffix_length, \- prefix_segment_length, \- max_sequence_length, \- position)--generateKernels(64, update_kv_cache)--#undef outerArguments-#undef innerArgumentsdiff --git a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metalsimilarity index 64%rename from crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metalrename to crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex 8efd95c71..7f9dd6c63 100644--- a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -1,36 +1,17 @@#include <metal_stdlib>-+#include "../definitions.metal"#include "../matmul/common/loader.h"#include "../matmul/common/mma.h"-#include "../definitions.metal"#include "attention.h"using namespace metal;using namespace uzu::matmul;using namespace uzu::attention;-///////////////////////////////////////////////////////////////////////////////-// Function constants for compile-time specialization-///////////////////////////////////////////////////////////////////////////////--// Alignment flags (match Gemm convention / our matmul indices)-constant bool align_Q [[function_constant(200)]];-constant bool align_K [[function_constant(201)]];--// Feature flags-constant bool has_mask [[function_constant(300)]];-constant bool do_causal [[function_constant(301)]];-constant bool has_sinks [[function_constant(302)]];--///////////////////////////////////////////////////////////////////////////////-// Helpers-///////////////////////////////////////////////////////////////////////////////-template <typename T>struct TransformScale {T scale;METAL_FUNC TransformScale(T scale_) : scale(scale_) {}-METAL_FUNC T apply(T x) const { return scale * x; }};@@ -50,55 +31,63 @@ METAL_FUNC T row_reduce_sum(T v) {return v;}-///////////////////////////////////////////////////////////////////////////////-// Gemm attention kernel-///////////////////////////////////////////////////////////////////////////////--// clang-format off-template <- typename T,- int BQ,- int BK,- int BD,- int WM,- int WN,- typename AccumType = float>-[[kernel, max_total_threads_per_threadgroup(WM * WN * 32)]] void attention_gemm(- const device T* Q [[buffer(0)]],- const device T* K [[buffer(1)]],- const device T* V [[buffer(2)]],- device T* O [[buffer(3)]],- const constant AttnParams* params [[buffer(4)]],- const constant AttnMaskParams* mask_params [[buffer(5), function_constant(has_mask)]],- const device T* mask [[buffer(6), function_constant(has_mask)]],- const device float* sinks [[buffer(7), function_constant(has_sinks)]],- uint simd_lane_id [[thread_index_in_simdgroup]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]]) { // clang-format on+#define BQ 32+#define SIMD_WIDTH 32+#define WM 4+#define WN 1++template <typename T, uint BK, uint BD>+VARIANTS(T, float, half, bfloat)+VARIANTS(BK, 16, 32)+VARIANTS(BD, 64, 128, 256)+KERNEL(AttentionGemm)(+ const device T* q,+ const device T* k,+ const device T* v,+ device T* o,+ const constant AttnParams* params,+ const constant AttnMaskParams* mask_params OPTIONAL(has_mask),+ const device T* mask OPTIONAL(has_mask),+ const device float* sinks OPTIONAL(has_sinks),+ const constant uint& num_heads,+ const constant uint& suffix_length,+ const bool align_q SPECIALIZE,+ const bool align_k SPECIALIZE,+ const bool do_causal SPECIALIZE,+ const bool has_mask SPECIALIZE,+ const bool has_sinks SPECIALIZE,+ threadgroup T q_smem[BQ * (BD + 16 / sizeof(T))],+ threadgroup T kv_smem[BK * (BD + 16 / sizeof(T))],+ const uint tgid_x GROUPS(suffix_length.div_ceil(BQ)),+ const uint tgid_y GROUPS(num_heads),+ const uint tgid_z GROUPS(1),+ const uint lid THREADS(128)+) {+ const uint simd_group_id = lid / SIMD_WIDTH;+ const uint simd_lane_id = lid % SIMD_WIDTH;// Pacifying compiler(void)lid;// -------------------------------------------------------------------------// Pointer setup (all strides are in elements)- // tid.x: query tile index (BQ rows)- // tid.y: query head index- // tid.z: batch index (currently 1 in uzu, but kept for completeness)- const int64_t batch_idx = int64_t(tid.z);- const int64_t head_idx = int64_t(tid.y);- const int64_t q_tile_idx = int64_t(tid.x);-- Q += batch_idx * params->q_strides[0] + head_idx * params->q_strides[1] ++ // tgid_x: query tile index (BQ rows)+ // tgid_y: query head index+ // tgid_z: batch index (currently 1 in uzu, but kept for completeness)+ const uint batch_idx = tgid_z;+ const uint head_idx = tgid_y;+ const uint q_tile_idx = tgid_x;++ q += batch_idx * params->q_strides[0] + head_idx * params->q_strides[1] +q_tile_idx * int64_t(BQ) * params->q_strides[2];- const int kv_head_idx = int(tid.y) / params->gqa_factor;- K += batch_idx * params->k_strides[0] ++ const int kv_head_idx = int(tgid_y) / params->gqa_factor;+ k += batch_idx * params->k_strides[0] +int64_t(kv_head_idx) * params->k_strides[1];- V += batch_idx * params->v_strides[0] ++ v += batch_idx * params->v_strides[0] +int64_t(kv_head_idx) * params->v_strides[1];- O += batch_idx * params->o_strides[0] + head_idx * params->o_strides[1] ++ o += batch_idx * params->o_strides[0] + head_idx * params->o_strides[1] +q_tile_idx * int64_t(BQ) * params->o_strides[2];if (has_mask) {@@ -120,13 +109,11 @@ template <constexpr short tgp_mem_v = BK * (BD + padV);constexpr short tgp_mem_s = tgp_mem_k > tgp_mem_v ? tgp_mem_k : tgp_mem_v;- threadgroup T Q_smem[BQ * (BD + padQ)];- threadgroup T KV_smem[tgp_mem_s];-- threadgroup T* Qs = Q_smem;- threadgroup T* Ks = KV_smem;- threadgroup T* Vs = KV_smem;+ threadgroup T* Qs = q_smem;+ threadgroup T* Ks = kv_smem;+ threadgroup T* Vs = kv_smem;+ //// -------------------------------------------------------------------------// Block loadersusing QBlockLoader = BlockLoader<@@ -157,15 +144,16 @@ template <const int k_src_ld = int(params->k_strides[2]);const int v_src_ld = int(params->v_strides[2]);- thread QBlockLoader loader_q(Q, q_src_ld, Qs, simd_group_id, simd_lane_id);- thread KBlockLoader loader_k(K, k_src_ld, Ks, simd_group_id, simd_lane_id);- thread VBlockLoader loader_v(V, v_src_ld, Vs, simd_group_id, simd_lane_id);+ thread QBlockLoader loader_q(q, q_src_ld, Qs, simd_group_id, simd_lane_id);+ thread KBlockLoader loader_k(k, k_src_ld, Ks, simd_group_id, simd_lane_id);+ thread VBlockLoader loader_v(v, v_src_ld, Vs, simd_group_id, simd_lane_id);TransformScale<T> ts(static_cast<T>(params->scale * M_LOG2E_F));// -------------------------------------------------------------------------// MMA tilesconstexpr short kFragSize = 8;+ using AccumType = float;using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;constexpr int kNWarps = WM * WN;@@ -216,7 +204,7 @@ template <// Load Q block once (and apply scaling)threadgroup_barrier(mem_flags::mem_threadgroup);- if (!align_Q && int(tid.x) == params->nq_aligned) {+ if (!align_q && int(tgid_x) == params->nq_aligned) {loader_q.load_safe(short2(BD, params->q_rem));} else {loader_q.load_unsafe();@@ -230,26 +218,26 @@ template <AccumType sum_score = AccumType(0);if (has_sinks) {- max_score = M_LOG2E_F * static_cast<AccumType>(sinks[tid.y]);+ max_score = M_LOG2E_F * static_cast<AccumType>(sinks[tgid_y]);sum_score = AccumType(1);}// Determine K block loop limit (causal can early-stop)int kb_lim = params->nk;if (do_causal) {- const int q_max = (int(tid.x) + 1) * BQ + params->q_off;+ const int q_max = (int(tgid_x) + 1) * BQ + params->q_off;kb_lim = (q_max + BK - 1) / BK;kb_lim = min(params->nk, kb_lim);}- const int q_rel = int(tid.x) * BQ + int(tm) + int(sm); // [0, q_len)- const int q_abs = q_rel + params->q_off; // [0, k_len)+ const int q_rel = int(tgid_x) * BQ + int(tm) + int(sm); // [0, q_len)+ const int q_abs = q_rel + params->q_off; // [0, k_len)// Loop over KV blocksfor (int kb = 0; kb < kb_lim; kb++) {// Load K blockthreadgroup_barrier(mem_flags::mem_threadgroup);- if (!align_K && kb == params->nk_aligned) {+ if (!align_k && kb == params->nk_aligned) {loader_k.load_safe(short2(BD, params->k_rem));} else {loader_k.load_unsafe();@@ -276,7 +264,7 @@ template <}// Mask out tail keys for the last (unaligned) K block- if (!align_K && kb == params->nk_aligned) {+ if (!align_k && kb == params->nk_aligned) {const int k_rem = params->k_rem;UZU_PRAGMA_UNROLLfor (short j = 0; j < TK; j++) {@@ -293,7 +281,7 @@ template <// Causal mask (only needed for the last few blocks near the diagonal)if (do_causal) {- const int tail_blocks = (BQ + BK - 1) / BK + int(!align_K);+ const int tail_blocks = (BQ + BK - 1) / BK + int(!align_k);const int tail_start = kb_lim - tail_blocks;if (kb >= tail_start) {UZU_PRAGMA_UNROLL@@ -338,7 +326,7 @@ template <// Load V block (overwriting K in shared memory)threadgroup_barrier(mem_flags::mem_threadgroup);- if (!align_K && kb == params->nk_aligned) {+ if (!align_k && kb == params->nk_aligned) {loader_v.load_safe(short2(BD, params->k_rem));} else {loader_v.load_unsafe();@@ -428,9 +416,9 @@ template <threadgroup_barrier(mem_flags::mem_none);// Store results (O is row-major with row-stride params->o_strides[2])- O += int64_t(tm + sm) * params->o_strides[2] + int64_t(sn);+ o += int64_t(tm + sm) * params->o_strides[2] + int64_t(sn);- if (!align_Q && int(tid.x) == params->nq_aligned) {+ if (!align_q && int(tgid_x) == params->nq_aligned) {const short2 dst_tile_dims = short2(BD - sn, params->q_rem - (tm + sm));if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0) {@@ -438,64 +426,11 @@ template <}Otile.template store_safe<T, 1, 1>(- O,+ o,int(params->o_strides[2]),dst_tile_dims);} else {- Otile.template store<T, 1, 1>(O, int(params->o_strides[2]));+ Otile.template store<T, 1, 1>(o, int(params->o_strides[2]));}-}--///////////////////////////////////////////////////////////////////////////////-// Kernel instantiations (BQ=32, BK in {32,16}, WM=2, WN=2)-///////////////////////////////////////////////////////////////////////////////--#define instantiate_attention_gemm( \- type_name, \- element_type, \- head_dim_value, \- bk_value \-) \- template [[host_name( \- "attention_gemm_" #type_name "_" #head_dim_value "_bk" #bk_value \- )]] [[kernel]] void \- attention_gemm<element_type, 32, bk_value, head_dim_value, 4, 1, float>( \- const device element_type* Q [[buffer(0)]], \- const device element_type* K [[buffer(1)]], \- const device element_type* V [[buffer(2)]], \- device element_type* O [[buffer(3)]], \- const constant AttnParams* params [[buffer(4)]], \- const constant AttnMaskParams* mask_params \- [[buffer(5), function_constant(has_mask)]], \- const device element_type* mask \- [[buffer(6), function_constant(has_mask)]], \- const device float* sinks [[buffer(7), function_constant(has_sinks)]], \- uint simd_lane_id [[thread_index_in_simdgroup]], \- uint simd_group_id [[simdgroup_index_in_threadgroup]], \- uint3 tid [[threadgroup_position_in_grid]], \- uint3 lid [[thread_position_in_threadgroup]] \- );--instantiate_attention_gemm(f16, half, 64, 32) instantiate_attention_gemm(- f16,- half,- 128,- 16-) instantiate_attention_gemm(f16, half, 256, 16)-- instantiate_attention_gemm(bf16, bfloat, 64, 32) instantiate_attention_gemm(- bf16,- bfloat,- 128,- 16- ) instantiate_attention_gemm(bf16, bfloat, 256, 16)-- instantiate_attention_gemm(f32, float, 64, 32) instantiate_attention_gemm(- f32,- float,- 128,- 16- ) instantiate_attention_gemm(f32, float, 256, 16)--#undef instantiate_attention_gemm+}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metalnew file mode 100644index 000000000..b3ef95a54--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -0,0 +1,177 @@+#include <metal_stdlib>+#include <metal_simdgroup>+#include "../definitions.metal"++#define SIMD_WIDTH 32+#define SEQUENCE_BLOCK_SIZE 32+#define HEAD_BLOCK_SIZE 32++template <typename T, uint head_dim>+VARIANTS(T, float, half, bfloat)+VARIANTS(head_dim, 64, 128, 256)+KERNEL(AttentionSinglePass)(+ const device T* queries,+ const device T* keys,+ const device T* values,+ device T* out,+ const constant uint& gqa_factor,+ const constant uint& sequence_length,+ const constant uint& k_head_stride,+ const constant uint& k_seq_stride,+ const constant uint& v_head_stride,+ const constant uint& v_seq_stride,+ const constant float& scale,+ const device T* fmask OPTIONAL(float_mask),+ const constant uint& mask_kv_seq_stride OPTIONAL(has_mask),+ const constant uint& mask_q_seq_stride OPTIONAL(has_mask),+ const constant uint& mask_head_stride OPTIONAL(has_mask),+ const device float* sinks OPTIONAL(has_sinks),+ const constant uint& num_heads,+ const constant uint& suffix_length,+ threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],+ threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],+ threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],+ const bool float_mask SPECIALIZE,+ const bool has_mask SPECIALIZE,+ const bool has_sinks SPECIALIZE,+ const bool do_causal SPECIALIZE,+ const uint head_idx GROUPS(num_heads),+ const uint q_seq_idx GROUPS(suffix_length),+ const uint tid THREADS(1024)+) {+ const uint3 tpg = {num_heads, suffix_length, 1};+ const uint simd_gid = tid / SIMD_WIDTH;+ const uint simd_lid = tid % SIMD_WIDTH;+ constexpr bool bool_mask = false;+ constexpr bool query_transposed = false;++ constexpr uint value_dim = head_dim;+ constexpr uint qk_elements_per_thread = head_dim / HEAD_BLOCK_SIZE;+ constexpr uint value_elements_per_thread = value_dim / HEAD_BLOCK_SIZE;+ uint inner_k_stride = SEQUENCE_BLOCK_SIZE * int(k_seq_stride);+ uint inner_v_stride = SEQUENCE_BLOCK_SIZE * int(v_seq_stride);++ typedef float U;++ thread U q[qk_elements_per_thread];+ thread U k[qk_elements_per_thread];+ thread U o[value_elements_per_thread];++ const uint kv_head_idx = head_idx / gqa_factor;+ const uint o_offset = q_seq_idx * tpg.x + head_idx;+ const uint q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx+ : head_idx * tpg.y + q_seq_idx;++ queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;+ keys += kv_head_idx * k_head_stride + simd_gid * k_seq_stride ++ simd_lid * qk_elements_per_thread;+ values += kv_head_idx * v_head_stride + simd_gid * v_seq_stride ++ simd_lid * value_elements_per_thread;++ if (bool_mask) {+ // bmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride ++ // q_seq_idx * mask_q_seq_stride;+ }+ if (float_mask) {+ fmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride ++ q_seq_idx * mask_q_seq_stride;+ }++ out += o_offset * value_dim + simd_gid * value_elements_per_thread;++ // Read the query and 0 the output accumulator+ for (int i = 0; i < qk_elements_per_thread; i++) {+ q[i] = static_cast<U>(scale) * queries[i];+ }+ for (int i = 0; i < value_elements_per_thread; i++) {+ o[i] = 0;+ }++ U max_score = -INFINITY;+ U sum_exp_score = 0;+ if (has_sinks && simd_gid == 0) {+ const int num_q_heads = static_cast<int>(tpg.x);+ int q_head_idx = head_idx % num_q_heads;+ max_score = static_cast<U>(sinks[q_head_idx]);+ sum_exp_score = 1;+ }++ // For each key+ for (uint i = simd_gid; i < sequence_length; i += SEQUENCE_BLOCK_SIZE) {+ bool use_key = true;+ if (do_causal) {+ use_key = i <= (sequence_length - tpg.y + q_seq_idx);+ } else if (bool_mask) {+ // use_key = bmask[0];+ }++ if (use_key) {+ // Read the key+ for (uint j = 0; j < qk_elements_per_thread; j++) {+ k[j] = keys[j];+ }++ // Compute the i-th score+ U score = 0;+ for (uint j = 0; j < qk_elements_per_thread; j++) {+ score += q[j] * k[j];+ }+ score = simd_sum(score);+ if (float_mask) {+ score += max(-1e9f, static_cast<U>(fmask[0]));+ }++ // Update the accumulators+ U new_max = max(max_score, score);+ U factor = fast::exp(max_score - new_max);+ U exp_score = fast::exp(score - new_max);++ max_score = new_max;+ sum_exp_score = sum_exp_score * factor + exp_score;++ // Update the output accumulator+ for (int j = 0; j < value_elements_per_thread; j++) {+ o[j] = o[j] * factor + exp_score * values[j];+ }+ }++ // Move the pointers to the next kv+ keys += inner_k_stride;+ values += inner_v_stride;+ if (bool_mask) {+ // bmask += SEQUENCE_BLOCK_SIZE * mask_kv_seq_stride;+ }+ if (float_mask) {+ fmask += SEQUENCE_BLOCK_SIZE * mask_kv_seq_stride;+ }+ }++ // Each thread has a partial part of the output so we need to combine them.+ if (simd_lid == 0) {+ shared_max_scores[simd_gid] = max_score;+ shared_sum_exp_scores[simd_gid] = sum_exp_score;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ max_score = shared_max_scores[simd_lid];+ U new_max = simd_max(max_score);+ U factor = fast::exp(max_score - new_max);+ sum_exp_score = simd_sum(shared_sum_exp_scores[simd_lid] * factor);++ // Now we need to aggregate all the outputs+ for (int i = 0; i < value_elements_per_thread; i++) {+ shared_outputs[simd_lid * HEAD_BLOCK_SIZE + simd_gid] = o[i];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ o[i] = simd_sum(+ shared_outputs[simd_gid * HEAD_BLOCK_SIZE + simd_lid] * factor+ ) /+ sum_exp_score;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++ // And write the output+ if (simd_lid == 0) {+ for (int i = 0; i < value_elements_per_thread; i++) {+ out[i] = static_cast<T>(o[i]);+ }+ }+}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metalnew file mode 100644index 000000000..3c0295233--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -0,0 +1,251 @@+#include <metal_stdlib>+#include <metal_simdgroup>+#include "../definitions.metal"++#define HEAD_BLOCK_SIZE 32+#define SEQUENCE_BLOCK_SIZE_1 8+#define SEQUENCE_BLOCK_SIZE_2 32+#define SIMD_WIDTH 32+#define TOTAL_BLOCKS_COUNT 32++template <typename T, uint head_dim>+VARIANTS(T, float, half, bfloat)+VARIANTS(head_dim, 64, 128, 256)+KERNEL(AttentionTwoPass1)(+ const device T* queries,+ const device T* keys,+ const device T* values,+ device float* out,+ device float* sums,+ device float* maxs,+ const constant uint& gqa_factor,+ const constant uint& sequence_length,+ const constant uint& k_head_stride,+ const constant uint& k_seq_stride,+ const constant uint& v_head_stride,+ const constant uint& v_seq_stride,+ const constant float& scale,+ const constant uint& num_heads,+ const constant uint& suffix_length,+ const device T* fmask OPTIONAL(float_mask),+ const constant uint& mask_kv_seq_stride OPTIONAL(has_mask),+ const constant uint& mask_q_seq_stride OPTIONAL(has_mask),+ const constant uint& mask_head_stride OPTIONAL(has_mask),+ const device float* sinks OPTIONAL(has_sinks),+ const bool float_mask SPECIALIZE,+ const bool has_mask SPECIALIZE,+ const bool has_sinks SPECIALIZE,+ const bool do_causal SPECIALIZE,+ threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE_1],+ threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE_1],+ threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_1 * HEAD_BLOCK_SIZE],+ const uint head_idx GROUPS(num_heads),+ const uint q_seq_idx GROUPS(suffix_length),+ const uint block_idx GROUPS(TOTAL_BLOCKS_COUNT),+ const uint tid THREADS(256)+) {+ const uint3 tpg = {num_heads, suffix_length, TOTAL_BLOCKS_COUNT};+ const uint simd_gid = tid / SIMD_WIDTH;+ const uint simd_lid = tid % SIMD_WIDTH;++ constexpr bool query_transposed = false;+ constexpr uint value_dim = head_dim;+ constexpr uint qk_elements_per_thread = head_dim / HEAD_BLOCK_SIZE;+ constexpr uint value_elements_per_thread = value_dim / HEAD_BLOCK_SIZE;+ uint inner_k_stride = SEQUENCE_BLOCK_SIZE_1 * k_seq_stride;+ uint inner_v_stride = SEQUENCE_BLOCK_SIZE_1 * v_seq_stride;++ typedef float U;++ thread U q[qk_elements_per_thread];+ thread U k[qk_elements_per_thread];+ thread U o[value_elements_per_thread];++ const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout+ const uint q_offset =+ query_transposed+ ? tpg.x * q_seq_idx + head_idx+ : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass+ const uint kv_head_idx = head_idx / gqa_factor;++ queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;+ keys += kv_head_idx * k_head_stride ++ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid) * k_seq_stride ++ simd_lid * qk_elements_per_thread;+ values += kv_head_idx * v_head_stride ++ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid) * v_seq_stride ++ simd_lid * value_elements_per_thread;+ out += o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim ++ simd_lid * value_elements_per_thread;+ if (float_mask) {+ fmask +=+ head_idx * mask_head_stride ++ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid) * mask_kv_seq_stride ++ q_seq_idx * mask_q_seq_stride;+ }+ sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;+ maxs += o_offset * TOTAL_BLOCKS_COUNT + block_idx;++ // Read the query and 0 the output accumulator+ for (uint i = 0; i < qk_elements_per_thread; i++) {+ q[i] = static_cast<U>(scale) * queries[i];+ }+ for (uint i = 0; i < value_elements_per_thread; i++) {+ o[i] = 0;+ }++ U max_score = -1e9;+ U sum_exp_score = 0;+ if (has_sinks && block_idx == 0 && simd_gid == 0) {+ const uint num_q_heads = tpg.x;+ int q_head_idx = head_idx % num_q_heads;+ max_score = static_cast<U>(sinks[q_head_idx]);+ sum_exp_score = 1;+ }++ // For each key+ for (uint i = block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid;+ i < sequence_length;+ i += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1) {+ bool use_key = true;+ if (do_causal) {+ use_key = i <= (sequence_length - tpg.y + q_seq_idx);+ }++ if (use_key) {+ // Read the key+ for (uint j = 0; j < qk_elements_per_thread; j++) {+ k[j] = keys[j];+ }++ // Compute the i-th score+ U score = 0;+ for (uint j = 0; j < qk_elements_per_thread; j++) {+ score += q[j] * k[j];+ }+ score = simd_sum(score);+ if (float_mask) {+ score += max(-1e9f, static_cast<U>(fmask[0]));+ }++ // Update the accumulators+ U new_max = max(max_score, score);+ U factor = fast::exp(max_score - new_max);+ U exp_score = fast::exp(score - new_max);++ max_score = new_max;+ sum_exp_score = sum_exp_score * factor + exp_score;++ // Update the output accumulator+ for (uint j = 0; j < value_elements_per_thread; j++) {+ o[j] = o[j] * factor + exp_score * values[j];+ }+ }++ // Move the pointers to the next kv+ keys += TOTAL_BLOCKS_COUNT * inner_k_stride;+ values += TOTAL_BLOCKS_COUNT * inner_v_stride;+ if (float_mask) {+ fmask += SEQUENCE_BLOCK_SIZE_1 * TOTAL_BLOCKS_COUNT * mask_kv_seq_stride;+ }+ }++ // Each thread has a partial part of the output so we need to combine them.+ if (simd_lid == 0) {+ shared_max_scores[simd_gid] = max_score;+ shared_sum_exp_scores[simd_gid] = sum_exp_score;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ max_score =+ (simd_lid < SEQUENCE_BLOCK_SIZE_1) ? shared_max_scores[simd_lid] : -1e9;+ U new_max = simd_max(max_score);+ U factor = fast::exp(max_score - new_max);+ sum_exp_score =+ (simd_lid < SEQUENCE_BLOCK_SIZE_1) ? shared_sum_exp_scores[simd_lid] : 0;+ sum_exp_score = simd_sum(sum_exp_score * factor);++ // Write the sum and new max+ if (simd_gid == 0) {+ sums[0] = sum_exp_score;+ maxs[0] = new_max;+ }++ // Now we need to aggregate all the outputs+ for (uint i = 0; i < value_elements_per_thread; i++) {+ shared_outputs[simd_lid * SEQUENCE_BLOCK_SIZE_1 + simd_gid] =+ o[i] * fast::exp(shared_max_scores[simd_gid] - new_max);+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // And write the output+ if (simd_gid == 0) {+ U output = shared_outputs[simd_lid * SEQUENCE_BLOCK_SIZE_1];+ for (uint j = 1; j < SEQUENCE_BLOCK_SIZE_1; j++) {+ output += shared_outputs[simd_lid * SEQUENCE_BLOCK_SIZE_1 + j];+ }+ out[i] = static_cast<T>(output);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+}++template <typename T, uint head_dim>+VARIANTS(T, float, half, bfloat)+VARIANTS(head_dim, 64, 128, 256)+KERNEL(AttentionTwoPass2)(+ const device float* partials,+ const device float* sums,+ const device float* maxs,+ device T* out,+ const constant uint& num_heads,+ const constant uint& suffix_length,+ threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_2 * HEAD_BLOCK_SIZE],+ const uint head_idx GROUPS(num_heads),+ const uint q_seq_idx GROUPS(suffix_length),+ const uint tid THREADS(1024)+) {+ const uint3 tpg = {num_heads, suffix_length, 1};+ const uint simd_gid = tid / SIMD_WIDTH;+ const uint simd_lid = tid % SIMD_WIDTH;++ constexpr uint elements_per_thread = head_dim / HEAD_BLOCK_SIZE;++ typedef float U;+ thread U o[elements_per_thread];++ const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout++ partials += o_offset * TOTAL_BLOCKS_COUNT * head_dim + simd_gid * head_dim ++ simd_lid * elements_per_thread;+ sums += o_offset * TOTAL_BLOCKS_COUNT;+ maxs += o_offset * TOTAL_BLOCKS_COUNT;+ out += o_offset * head_dim ++ simd_gid * elements_per_thread; // Our custom output layout++ // First everybody reads the max and sum_exp+ U max_score = maxs[simd_lid];+ U new_max = simd_max(max_score);+ U factor = fast::exp(max_score - new_max);+ U sum_exp_score = simd_sum(sums[simd_lid] * factor);++ // Now read the block into registers and then use shared memory to transpose+ // it+ for (uint i = 0; i < elements_per_thread; i++) {+ o[i] = partials[i];+ }+ for (uint i = 0; i < elements_per_thread; i++) {+ shared_outputs[simd_lid * HEAD_BLOCK_SIZE + simd_gid] = o[i];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ o[i] = simd_sum(+ shared_outputs[simd_gid * HEAD_BLOCK_SIZE + simd_lid] * factor+ ) /+ sum_exp_score;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++ // And write the output+ if (simd_lid == 0) {+ for (uint i = 0; i < elements_per_thread; i++) {+ out[i] = static_cast<T>(o[i]);+ }+ }+}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metalnew file mode 100644index 000000000..fc76a792d--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal@@ -0,0 +1,42 @@+#include <metal_stdlib>+#include "../definitions.metal"++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(AttentionUpdateKVCache)(+ const device T* rotated_keys,+ const device T* qkv,+ device T* key_cache,+ device T* value_cache,+ const constant uint& num_groups,+ const constant uint& num_heads,+ const constant uint& head_dim,+ const constant uint& suffix_length,+ const constant uint& prefix_segment_length,+ const constant uint& max_sequence_length,+ const uint group_index AXIS(num_groups, 1),+ const uint token_index AXIS(suffix_length, 1),+ const uint dim_index AXIS(head_dim, 64)+) {+ const uint cacheTokenIndex = prefix_segment_length + token_index;++ // Copy rotated key to cache+ const uint rotatedKeyOffset =+ (group_index * suffix_length + token_index) * head_dim + dim_index;+ const uint keyCacheOffset =+ (group_index * max_sequence_length + cacheTokenIndex) * head_dim ++ dim_index;+ key_cache[keyCacheOffset] = rotated_keys[rotatedKeyOffset];++ // Copy value to cache+ // qkv layout: [suffix_length, (num_heads + 2*num_groups) * head_dim]+ // Values start at offset: (num_heads * head_dim) + (num_groups * head_dim)+ const uint qkvStride = (num_heads + 2 * num_groups) * head_dim;+ const uint valueBaseOffset = (num_heads + num_groups) * head_dim;+ const uint valueOffset = token_index * qkvStride + valueBaseOffset ++ group_index * head_dim + dim_index;+ const uint valueCacheOffset =+ (group_index * max_sequence_length + cacheTokenIndex) * head_dim ++ dim_index;+ value_cache[valueCacheOffset] = qkv[valueOffset];+}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rsindex 241747e35..2f2f5cc67 100644--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs@@ -1,111 +1,23 @@-use std::{collections::HashMap, mem::size_of, ptr::NonNull};+use std::{+ cell::RefCell,+ collections::{HashMap, hash_map::Entry},+};-use metal::{MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLFunctionConstantValues, MTLSize};-use objc2::{rc::Retained, runtime::ProtocolObject};-use thiserror::Error;+use metal::{MTLBuffer, MTLComputeCommandEncoder};+use objc2::{Message, runtime::ProtocolObject};use crate::{DataType,backends::{- common::gpu_types::{AttnMaskParams, AttnParams},- metal::{- FunctionConstantValuesSetValue, MetalContext, MetalError, data_type::MetalDataTypeExt,- kernel::moe::dtype_suffix,+ common::{+ gpu_types::{AttnMaskParams, AttnParams},+ kernel::AttentionGemmKernel,},+ metal::{MetalContext, MetalError, kernel::dsl::AttentionGemmMetalKernel},},};-#[derive(Debug, Clone, Copy)]-pub enum AttentionKernelVariant {- SinglePass,- TwoPass,-}--type PipelineKey = (usize, bool, bool, bool); // (head_dim, has_sinks, is_causal, has_mask)--pub struct AttentionKernelPipelines {- single_pass: HashMap<PipelineKey, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,- two_pass_1: HashMap<PipelineKey, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,- two_pass_2: HashMap<usize, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,- kv_cache_update: Option<Retained<ProtocolObject<dyn MTLComputePipelineState>>>,-}--pub struct AttentionKernel {- data_type: DataType,- pipelines: AttentionKernelPipelines,-}--#[derive(Debug, Error)]-pub enum AttentionError {- #[error("Metal error: {0}")]- MetalError(#[from] MetalError),- #[error("Function not found: {0}")]- FunctionNotFound(String),- #[error("Unsupported head dimension: {0}")]- UnsupportedHeadDim(usize),-}--pub struct AttentionSinglePassArguments<'a> {- pub queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)- pub keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)- pub values_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3)- pub gqa_factor: i32, // buffer(4)- pub sequence_length: i32, // buffer(5) - sequence_length- pub k_head_stride: i32, // buffer(6)- pub k_seq_stride: i32, // buffer(7)- pub v_head_stride: i32, // buffer(8)- pub v_seq_stride: i32, // buffer(9)- pub scale: f32, // buffer(10)- pub mask_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(11/12)- pub mask_kv_seq_stride: i32, // buffer(13)- pub mask_q_seq_stride: i32, // buffer(14)- pub mask_head_stride: i32, // buffer(15)- pub sinks_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(16)- pub num_heads: usize,- pub suffix_length: usize,- pub head_dim: usize,- pub is_causal: bool,-}--pub struct AttentionTwoPassArguments<'a> {- pub queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)- pub keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)- pub values_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)- pub partials_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) - pass 1 output- pub sums_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(4) - pass 1 output- pub maxs_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(5) - pass 1 output- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3) - pass 2 output- pub gqa_factor: i32, // buffer(6)- pub sequence_length: i32, // buffer(7) - sequence_length- pub k_head_stride: i32, // buffer(8)- pub k_seq_stride: i32, // buffer(9)- pub v_head_stride: i32, // buffer(10)- pub v_seq_stride: i32, // buffer(11)- pub scale: f32, // buffer(12)- pub mask_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(13/14)- pub mask_kv_seq_stride: i32, // buffer(15)- pub mask_q_seq_stride: i32, // buffer(16)- pub mask_head_stride: i32, // buffer(17)- pub sinks_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(18)- pub num_heads: usize,- pub suffix_length: usize,- pub head_dim: usize,- pub is_causal: bool,-}--pub struct KVCacheUpdateArguments<'a> {- pub rotated_keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)- pub qkv_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)- pub key_cache_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)- pub value_cache_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3)- pub num_groups: usize,- pub num_heads: usize,- pub head_dim: usize,- pub suffix_length: usize,- pub segment_prefix_length: usize,- pub max_sequence_length: usize,-}+const BQ: usize = 32;pub struct AttentionGemmArguments<'a> {pub queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)@@ -125,484 +37,63 @@ pub struct AttentionGemmArguments<'a> {pub scale: f32,}-fn make_function_constants(- has_mask_value: bool,- has_sinks_value: bool,- is_causal_value: bool,-) -> Retained<MTLFunctionConstantValues> {- let function_constants = MTLFunctionConstantValues::new();-- let query_transposed_value = false;- let bool_mask_value = false;- let float_mask_value = has_mask_value;-- function_constants.set_value(&has_mask_value, 20); // has_mask- function_constants.set_value(&query_transposed_value, 21); // query_transposed- function_constants.set_value(&bool_mask_value, 23); // bool_mask- function_constants.set_value(&float_mask_value, 24); // float_mask- function_constants.set_value(&is_causal_value, 22); // do_causal- function_constants.set_value(&has_sinks_value, 25); // has_sinks-- function_constants+pub struct AttentionGemmBlock {+ data_type: DataType,+ cache: RefCell<HashMap<KernelKey, AttentionGemmMetalKernel>>,}-impl AttentionKernel {- pub fn new(- context: &MetalContext,- data_type: DataType,- ) -> Result<Self, AttentionError> {- let data_suffix = data_type.metal_type();-- let supported_head_dims = [64, 128, 256];- let mut single_pass = HashMap::new();- let mut two_pass_1 = HashMap::new();- let mut two_pass_2 = HashMap::new();-- // Pre-generate all supported variants for sinks and causal configurations- for &has_sinks_value in &[false, true] {- for &is_causal_value in &[false, true] {- for &has_mask_value in &[false, true] {- let function_constants = make_function_constants(has_mask_value, has_sinks_value, is_causal_value);-- for &head_dim in &supported_head_dims {- if let Ok(pipeline) = context.compute_pipeline_state(- &format!("attention_single_pass_{}_{}", data_suffix, head_dim),- Some(&function_constants),- ) {- single_pass.insert((head_dim, has_sinks_value, is_causal_value, has_mask_value), pipeline);- }-- if let Ok(pipeline) = context.compute_pipeline_state(- &format!("attention_2pass_1_{}_{}", data_suffix, head_dim),- Some(&function_constants),- ) {- two_pass_1.insert((head_dim, has_sinks_value, is_causal_value, has_mask_value), pipeline);- }-- if !two_pass_2.contains_key(&head_dim) {- if let Ok(pipeline) = context.compute_pipeline_state(- &format!("attention_2pass_2_{}_{}", data_suffix, head_dim),- Some(&function_constants),- ) {- two_pass_2.insert(head_dim, pipeline);- }- }- }- }- }- }-- let kv_cache_update = context.compute_pipeline_state(&format!("update_kv_cache_{}", data_suffix), None).ok();-- Ok(Self {+impl AttentionGemmBlock {+ pub fn new(data_type: DataType) -> Self {+ let cache = RefCell::new(HashMap::new());+ Self {data_type,- pipelines: AttentionKernelPipelines {- single_pass,- two_pass_1,- two_pass_2,- kv_cache_update,- },- })- }-- pub fn supports_single_pass(- &self,- head_dim: usize,- is_causal: bool,- has_mask: bool,- ) -> bool {- self.pipelines.single_pass.contains_key(&(head_dim, false, is_causal, has_mask))- }-- pub fn supports_two_pass(- &self,- head_dim: usize,- is_causal: bool,- has_mask: bool,- ) -> bool {- self.pipelines.two_pass_1.contains_key(&(head_dim, false, is_causal, has_mask))- && self.pipelines.two_pass_2.contains_key(&head_dim)- }-- pub fn choose_variant(- &self,- sequence_length: usize,- head_dim: usize,- is_causal: bool,- has_mask: bool,- ) -> AttentionKernelVariant {- if self.supports_two_pass(head_dim, is_causal, has_mask) && sequence_length > 1024 {- AttentionKernelVariant::TwoPass- } else {- AttentionKernelVariant::SinglePass+ cache,}}- pub fn encode_single_pass(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: AttentionSinglePassArguments,- ) -> Result<(), AttentionError> {- let has_sinks = args.sinks_buffer.is_some();- let has_mask = args.mask_buffer.is_some();- let pipeline = self- .pipelines- .single_pass- .get(&(args.head_dim, has_sinks, args.is_causal, has_mask))- .ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;-- MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pipeline);-- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.queries_buffer), 0, 0);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.keys_buffer), 0, 1);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.values_buffer), 0, 2);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.output_buffer), 0, 3);-- unsafe {- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.gqa_factor as *const i32 as *mut _),- size_of::<i32>(),- 4,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.sequence_length as *const i32 as *mut _),- size_of::<i32>(),- 5,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.k_head_stride as *const i32 as *mut _),- size_of::<i32>(),- 6,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.k_seq_stride as *const i32 as *mut _),- size_of::<i32>(),- 7,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.v_head_stride as *const i32 as *mut _),- size_of::<i32>(),- 8,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.v_seq_stride as *const i32 as *mut _),- size_of::<i32>(),- 9,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.scale as *const f32 as *mut _),- size_of::<f32>(),- 10,- );- }-- if let Some(mask_buffer) = args.mask_buffer {- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(mask_buffer), 0, 12); // float_mask- unsafe {- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.mask_kv_seq_stride as *const i32 as *mut _),- size_of::<i32>(),- 13,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.mask_q_seq_stride as *const i32 as *mut _),- size_of::<i32>(),- 14,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.mask_head_stride as *const i32 as *mut _),- size_of::<i32>(),- 15,- );- }- }-- if let Some(sinks_buffer) = args.sinks_buffer {- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(sinks_buffer), 0, 16);- }-- let threads_per_threadgroup = MTLSize {- width: 32 * 32, // sequence_block_size * head_block_size- height: 1,- depth: 1,- };-- let threadgroups_per_grid = MTLSize::new(args.num_heads, args.suffix_length, 1);-- compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);- Ok(())- }-- pub fn encode_two_pass(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: AttentionTwoPassArguments,- ) -> Result<(), AttentionError> {- let has_sinks = args.sinks_buffer.is_some();- let has_mask = args.mask_buffer.is_some();- let pass1_pipeline = self- .pipelines- .two_pass_1- .get(&(args.head_dim, has_sinks, args.is_causal, has_mask))- .ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;-- let pass2_pipeline = self- .pipelines- .two_pass_2- .get(&args.head_dim)- .ok_or_else(|| AttentionError::UnsupportedHeadDim(args.head_dim))?;-- MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pass1_pipeline);-- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.queries_buffer), 0, 0);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.keys_buffer), 0, 1);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.values_buffer), 0, 2);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.partials_buffer), 0, 3);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.sums_buffer), 0, 4);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.maxs_buffer), 0, 5);-- unsafe {- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.gqa_factor as *const i32 as *mut _),- size_of::<i32>(),- 6,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.sequence_length as *const i32 as *mut _),- size_of::<i32>(),- 7,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.k_head_stride as *const i32 as *mut _),- size_of::<i32>(),- 8,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.k_seq_stride as *const i32 as *mut _),- size_of::<i32>(),- 9,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.v_head_stride as *const i32 as *mut _),- size_of::<i32>(),- 10,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.v_seq_stride as *const i32 as *mut _),- size_of::<i32>(),- 11,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.scale as *const f32 as *mut _),- size_of::<f32>(),- 12,- );- }-- // Set mask buffer if present- if let Some(mask_buffer) = args.mask_buffer {- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(mask_buffer), 0, 14); // float_mask- unsafe {- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.mask_kv_seq_stride as *const i32 as *mut _),- size_of::<i32>(),- 15,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.mask_q_seq_stride as *const i32 as *mut _),- size_of::<i32>(),- 16,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&args.mask_head_stride as *const i32 as *mut _),- size_of::<i32>(),- 17,- );- }- }-- if let Some(sinks_buffer) = args.sinks_buffer {- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(sinks_buffer), 0, 18);- }-- let total_blocks_count = 32u64;- let pass1_threads_per_threadgroup = MTLSize {- width: 8 * 32, // sequence_block_size * head_block_size- height: 1,- depth: 1,- };- let pass1_threadgroups_per_grid = MTLSize::new(args.num_heads, args.suffix_length, total_blocks_count as usize);-- compute_encoder.dispatch_threadgroups(pass1_threadgroups_per_grid, pass1_threads_per_threadgroup);-- MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pass2_pipeline);-- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.partials_buffer), 0, 0);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.sums_buffer), 0, 1);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.maxs_buffer), 0, 2);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.output_buffer), 0, 3);-- let pass2_threads_per_threadgroup = MTLSize {- width: 32 * 32,- height: 1,- depth: 1,- };- let pass2_threadgroups_per_grid = MTLSize::new(args.num_heads, args.suffix_length, 1);-- compute_encoder.dispatch_threadgroups(pass2_threadgroups_per_grid, pass2_threads_per_threadgroup);-- Ok(())- }-- pub fn encode_kv_cache_update(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: KVCacheUpdateArguments,- ) -> Result<(), AttentionError> {- let pipeline = self- .pipelines- .kv_cache_update- .as_ref()- .ok_or_else(|| AttentionError::FunctionNotFound("KV cache update kernel".to_string()))?;-- MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, pipeline);-- // Set buffers- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.rotated_keys_buffer), 0, 0);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.qkv_buffer), 0, 1);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.key_cache_buffer), 0, 2);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.value_cache_buffer), 0, 3);-- // Set constants- unsafe {- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&(args.num_groups as i32) as *const i32 as *mut _),- size_of::<i32>(),- 4,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&(args.num_heads as i32) as *const i32 as *mut _),- size_of::<i32>(),- 5,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&(args.head_dim as i32) as *const i32 as *mut _),- size_of::<i32>(),- 6,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&(args.suffix_length as i32) as *const i32 as *mut _),- size_of::<i32>(),- 7,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&(args.segment_prefix_length as i32) as *const i32 as *mut _),- size_of::<i32>(),- 8,- );- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&(args.max_sequence_length as i32) as *const i32 as *mut _),- size_of::<i32>(),- 9,- );- }-- let threads_per_grid = MTLSize::new(args.num_groups, args.suffix_length, args.head_dim as usize);-- let threadgroup_depth = std::cmp::min(args.head_dim.max(1), 64) as usize;-- MTLComputeCommandEncoder::dispatch_threads(- compute_encoder,- threads_per_grid,- MTLSize {- width: 1,- height: 1,- depth: threadgroup_depth,- },- );- Ok(())- }-- pub fn encode_gemm(+ pub fn encode(&self,context: &MetalContext,compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: AttentionGemmArguments,- ) -> Result<(), AttentionError> {- const BQ: usize = 32;- const WM: u64 = 4;- const WN: u64 = 1;-- if !matches!(args.head_dim, 64 | 128 | 256) {- return Err(AttentionError::UnsupportedHeadDim(args.head_dim));- }-+ args: &AttentionGemmArguments,+ ) -> Result<(), MetalError> {let bk: usize = if args.head_dim < 128 {32} else {16};-let align_q = (args.suffix_length % BQ) == 0;let align_k = (args.sequence_length % bk) == 0;let has_mask = args.mask_buffer.is_some();let has_sinks = args.sinks_buffer.is_some();+ let key = KernelKey {+ bk,+ head_dim: args.head_dim,+ align_q,+ align_k,+ is_causal: args.is_causal,+ has_mask,+ has_sinks,+ };- let fcv = MTLFunctionConstantValues::new();- fcv.set_value(&align_q, 200);- fcv.set_value(&align_k, 201);- fcv.set_value(&has_mask, 300);- fcv.set_value(&args.is_causal, 301);- fcv.set_value(&has_sinks, 302);-- // Kernel name matches gemm_attention.metal instantiations:- // attention_gemm_{f16|bf16|f32}_{head_dim}_bk{bk}- let type_name = dtype_suffix(self.data_type);-- let function_name = format!("attention_gemm_{}_{}_bk{}", type_name, args.head_dim, bk);-- let cache_key = format!(- "{}_aq{}_ak{}_m{}_c{}_s{}",- function_name, align_q as u8, align_k as u8, has_mask as u8, args.is_causal as u8, has_sinks as u8- );-- let pipeline = context- .compute_pipeline_state_cached(&cache_key, &function_name, Some(&fcv))- .map_err(AttentionError::MetalError)?;-- MTLComputeCommandEncoder::set_compute_pipeline_state(compute_encoder, &pipeline);-- // Buffers- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.queries_buffer), 0, 0);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.keys_buffer), 0, 1);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.values_buffer), 0, 2);- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(args.output_buffer), 0, 3);+ let mut map = self.cache.borrow_mut();+ let kernel = match map.entry(key) {+ Entry::Occupied(entry) => entry.into_mut(),+ Entry::Vacant(entry) => {+ let kernel = AttentionGemmMetalKernel::new(+ context,+ self.data_type,+ bk as u32,+ args.head_dim as u32,+ align_q,+ align_k,+ args.is_causal,+ has_mask,+ has_sinks,+ )?;+ entry.insert(kernel)+ },+ };// Params (all strides in elements)let q_head_stride = (args.suffix_length * args.head_dim) as i64;@@ -614,7 +105,6 @@ impl AttentionKernel {let o_head_stride = args.head_dim as i64;let o_seq_stride = (args.num_heads * args.head_dim) as i64;- let nq = (args.suffix_length + BQ - 1) / BQ;let nk = (args.sequence_length + bk - 1) / bk;let nq_aligned = args.suffix_length / BQ;let nk_aligned = args.sequence_length / bk;@@ -636,45 +126,42 @@ impl AttentionKernel {k_rem: (args.sequence_length - nk_aligned * bk) as i32,};- unsafe {- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(¶ms as *const AttnParams as *mut _),- size_of::<AttnParams>(),- 4,- );- }-- if let Some(mask_buffer) = args.mask_buffer {- let mask_params = AttnMaskParams {- // We use a shared bias matrix for all heads/batches.- m_strides: [0, 0, args.sequence_length as i64],- };- unsafe {- MTLComputeCommandEncoder::set_bytes(- compute_encoder,- NonNull::new_unchecked(&mask_params as *const AttnMaskParams as *mut _),- size_of::<AttnMaskParams>(),- 5,- );- }- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(mask_buffer), 0, 6);- }-- if let Some(sinks_buffer) = args.sinks_buffer {- MTLComputeCommandEncoder::set_buffer(compute_encoder, Some(sinks_buffer), 0, 16);- }-- // Dispatch- let threadgroups_per_grid = MTLSize::new(nq as usize, args.num_heads, 1);- let threads_per_threadgroup = MTLSize {- width: 32,- height: WM as usize,- depth: WN as usize,+ let attn_mask_params = AttnMaskParams {+ // We use a shared bias matrix for all heads/batches.+ m_strides: [0, 0, args.sequence_length as i64],};-- compute_encoder.dispatch_threadgroups(threadgroups_per_grid, threads_per_threadgroup);+ let attn_mask_params_opt = args.mask_buffer.map(|_| std::slice::from_ref(&attn_mask_params));++ let mask_buffer_retain = args.mask_buffer.map(|b| b.retain());+ let mask_buffer_opt = mask_buffer_retain.as_ref().map(|b| b);+ let sinks_buffer_retain = args.sinks_buffer.map(|b| b.retain());+ let sinks_buffer_opt = sinks_buffer_retain.as_ref().map(|b| b);++ kernel.encode(+ &args.queries_buffer.retain(),+ &args.keys_buffer.retain(),+ &args.values_buffer.retain(),+ &args.output_buffer.retain(),+ std::slice::from_ref(¶ms),+ attn_mask_params_opt,+ mask_buffer_opt,+ sinks_buffer_opt,+ args.num_heads as u32,+ args.suffix_length as u32,+ &compute_encoder,+ );Ok(())}}++#[derive(PartialEq, Eq, Hash, Clone, Copy)]+struct KernelKey {+ bk: usize,+ head_dim: usize,+ align_q: bool,+ align_k: bool,+ is_causal: bool,+ has_mask: bool,+ has_sinks: bool,+}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 828bf05c5..e1400f79a 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -15,10 +15,6 @@ pub mod moe;pub mod quant_matmul;pub mod ssm;-pub use attention::{- AttentionError, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments, AttentionTwoPassArguments,- KVCacheUpdateArguments,-};pub use matmul::{MatmulArguments, MatmulKernel};pub use moe::{MoeBlockBasesArguments, MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rsindex e4085aa52..3cec1dbd3 100644--- a/crates/uzu/tests/attention_kernel_test.rs+++ b/crates/uzu/tests/attention_kernel_test.rs@@ -11,12 +11,15 @@ use objc2::{rc::Retained, runtime::ProtocolObject};use uzu::{DataType,backends::{- common::Context,+ common::{+ Context,+ kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},+ },metal::{MetalContext,- kernel::attention::{- AttentionGemmArguments, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments,- AttentionTwoPassArguments,+ kernel::{+ attention::{AttentionGemmArguments, AttentionGemmBlock},+ dsl::{AttentionSinglePassMetalKernel, AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel},},metal_extensions::CommandBufferTimingExt,},@@ -293,7 +296,7 @@ fn convert_kernel_output(}fn run_single_pass_attention(- kernel: &AttentionKernel,+ kernel: &AttentionSinglePassMetalKernel,context: &MetalContext,queries: &Array4<f32>,keys: &Array4<f32>,@@ -320,32 +323,37 @@ fn run_single_pass_attention(let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- let args = AttentionSinglePassArguments {- queries_buffer: &query_buffer,- keys_buffer: &key_cache_buffer,- values_buffer: &value_cache_buffer,- output_buffer: &output_buffer,- gqa_factor: (num_heads / num_kv_heads) as i32,- sequence_length: seq_len as i32,- k_head_stride: (seq_len * head_dim) as i32,- k_seq_stride: head_dim as i32,- v_head_stride: (seq_len * head_dim) as i32,- v_seq_stride: head_dim as i32,- scale,- mask_buffer: mask_buffer.as_deref(),- mask_kv_seq_stride: 1,- mask_q_seq_stride: seq_len as i32,- mask_head_stride: 0,- sinks_buffer: sinks_buffer.as_deref(),- num_heads,- suffix_length: seq_len,- head_dim,- is_causal: false,- };+ let mut mask_kv_seq_stride: Option<u32> = None;+ let mut mask_q_seq_stride: Option<u32> = None;+ let mut mask_head_stride: Option<u32> = None;+ if mask_buffer.is_some() {+ mask_kv_seq_stride = Some(1);+ mask_q_seq_stride = Some(seq_len as u32);+ mask_head_stride = Some(0);+ }- let encode_result = kernel.encode_single_pass(&compute_encoder, args);+ kernel.encode(+ &query_buffer,+ &key_cache_buffer,+ &value_cache_buffer,+ &output_buffer,+ (num_heads / num_kv_heads) as u32,+ seq_len as u32,+ (seq_len * head_dim) as u32,+ head_dim as u32,+ (seq_len * head_dim) as u32,+ head_dim as u32,+ scale,+ mask_buffer.as_ref().map(|b| b),+ mask_kv_seq_stride,+ mask_q_seq_stride,+ mask_head_stride,+ sinks_buffer.as_ref().map(|b| b),+ num_heads as u32,+ seq_len as u32,+ &compute_encoder,+ );compute_encoder.end_encoding();- encode_result?;command_buffer.commit();command_buffer.wait_until_completed();@@ -359,7 +367,7 @@ fn run_single_pass_attention(}fn run_single_pass_attention_with_is_causal(- kernel: &AttentionKernel,+ kernel: &AttentionSinglePassMetalKernel,context: &MetalContext,queries: &Array4<f32>,keys: &Array4<f32>,@@ -367,7 +375,7 @@ fn run_single_pass_attention_with_is_causal(mask: Option<&Array3<f32>>,sinks: Option<&[f32]>,scale: f32,- is_causal: bool,+ _is_causal: bool,) -> Result<Array4<f32>, Box<dyn std::error::Error>> {let (batch_size, num_heads, seq_len, head_dim) = queries.dim();let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();@@ -376,6 +384,7 @@ fn run_single_pass_attention_with_is_causal(let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);+ let has_mask = mask.is_some();let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));@@ -387,32 +396,36 @@ fn run_single_pass_attention_with_is_causal(let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- let args = AttentionSinglePassArguments {- queries_buffer: &query_buffer,- keys_buffer: &key_cache_buffer,- values_buffer: &value_cache_buffer,- output_buffer: &output_buffer,- gqa_factor: (num_heads / num_kv_heads) as i32,- sequence_length: seq_len as i32,- k_head_stride: (seq_len * head_dim) as i32,- k_seq_stride: head_dim as i32,- v_head_stride: (seq_len * head_dim) as i32,- v_seq_stride: head_dim as i32,+ let mut mask_kv_seq_stride: Option<u32> = None;+ let mut mask_q_seq_stride: Option<u32> = None;+ let mut mask_head_stride: Option<u32> = None;+ if has_mask {+ mask_kv_seq_stride = Some(1);+ mask_q_seq_stride = Some(seq_len as u32);+ mask_head_stride = Some(0);+ }+ kernel.encode(+ &query_buffer,+ &key_cache_buffer,+ &value_cache_buffer,+ &output_buffer,+ (num_heads / num_kv_heads) as u32,+ seq_len as u32,+ (seq_len * head_dim) as u32,+ head_dim as u32,+ (seq_len * head_dim) as u32,+ head_dim as u32,scale,- mask_buffer: mask_buffer.as_deref(),- mask_kv_seq_stride: 1,- mask_q_seq_stride: seq_len as i32,- mask_head_stride: 0,- sinks_buffer: sinks_buffer.as_deref(),- num_heads,- suffix_length: seq_len,- head_dim,- is_causal,- };-- let encode_result = kernel.encode_single_pass(&compute_encoder, args);+ mask_buffer.as_ref().map(|b| b),+ mask_kv_seq_stride,+ mask_q_seq_stride,+ mask_head_stride,+ sinks_buffer.as_ref().map(|b| b),+ num_heads as u32,+ seq_len as u32,+ &compute_encoder,+ );compute_encoder.end_encoding();- encode_result?;command_buffer.commit();command_buffer.wait_until_completed();@@ -426,7 +439,7 @@ fn run_single_pass_attention_with_is_causal(}fn run_gemm_attention(- kernel: &AttentionKernel,+ kernel: &AttentionGemmBlock,context: &MetalContext,queries: &Array4<f32>,keys: &Array4<f32>,@@ -472,7 +485,7 @@ fn run_gemm_attention(scale,};- let encode_result = kernel.encode_gemm(context, &compute_encoder, args);+ let encode_result = kernel.encode(context, &compute_encoder, &args);compute_encoder.end_encoding();encode_result?;@@ -536,7 +549,6 @@ fn compare_results(}#[test]-#[ignore]fn test_single_pass_attention_basic() {let context = MetalContext::new().expect("Failed to create MetalContext");@@ -557,26 +569,10 @@ fn test_single_pass_attention_basic() {println!("Reference output range (no mask): [{}, {}]", ref_min, ref_max);println!("Reference sample values: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());- let kernel = match AttentionKernel::new(&context, DataType::F32) {- Ok(k) => k,- Err(e) => {- panic!("Failed to create AttentionKernel: {:?}", e);- },- };-let is_causal = false; // Non-causal attention for this test- let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);- println!("Using kernel variant: {:?}", variant);- println!(- "Supports single-pass for head_dim={}: {}",- head_dim,- kernel.supports_single_pass(head_dim, is_causal, false)- );- println!("Supports two-pass for head_dim={}: {}", head_dim, kernel.supports_two_pass(head_dim, is_causal, false));-- if !kernel.supports_single_pass(head_dim, false, false) {- panic!("Single-pass kernel not supported for head_dim={}", head_dim);- }+ let kernel =+ AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, false, is_causal)+ .expect("Failed to create attention single pass metal");let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale){@@ -593,7 +589,6 @@ fn test_single_pass_attention_basic() {}#[test]-#[ignore]fn test_gemm_attention_basic() {let context = MetalContext::new().expect("Failed to create MetalContext");@@ -606,7 +601,7 @@ fn test_gemm_attention_basic() {let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);- let kernel = AttentionKernel::new(&context, DataType::F32).expect("Failed to create AttentionKernel");+ let kernel = AttentionGemmBlock::new(DataType::F32);let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);@@ -621,7 +616,6 @@ fn test_gemm_attention_basic() {}#[test]-#[ignore]fn test_gemm_attention_f32_head_dim_128() {let context = MetalContext::new().expect("Failed to create MetalContext");@@ -634,7 +628,7 @@ fn test_gemm_attention_f32_head_dim_128() {let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);- let kernel = AttentionKernel::new(&context, DataType::F32).expect("Failed to create AttentionKernel");+ let kernel = AttentionGemmBlock::new(DataType::F32);let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);@@ -649,7 +643,6 @@ fn test_gemm_attention_f32_head_dim_128() {}#[test]-#[ignore]fn test_matrix_attention_matches_vector_and_cpu_seq256() {let context = MetalContext::new().expect("Failed to create MetalContext");@@ -663,12 +656,13 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);- let kernel = AttentionKernel::new(&context, DataType::F32).expect("Failed to create AttentionKernel");-// CPU reference (uses the same additive mask)let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);// vector attention path (single-pass)+ let kernel =+ AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, true, true, false, is_causal)+ .expect("Failed to create attention single pass metal");let vector_output = run_single_pass_attention_with_is_causal(&kernel,&context,@@ -683,8 +677,9 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {.expect("run vector attention");// matrix attention path (gemm)+ let gemm_kernel = AttentionGemmBlock::new(DataType::F32);let matrix_output =- run_gemm_attention(&kernel, &context, &queries, &keys, &values, Some(&mask), None, scale, is_causal)+ run_gemm_attention(&gemm_kernel, &context, &queries, &keys, &values, Some(&mask), None, scale, is_causal).expect("run matrix attention");// Compare both to CPU@@ -705,7 +700,6 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {}#[test]-#[ignore]fn test_single_pass_attention_with_mask() {let context = MetalContext::new().expect("Failed to create MetalContext");@@ -718,17 +712,9 @@ fn test_single_pass_attention_with_mask() {let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);- let kernel = match AttentionKernel::new(&context, DataType::F32) {- Ok(k) => k,- Err(e) => {- panic!("Failed to create AttentionKernel: {:?}", e);- },- };-- if !kernel.supports_single_pass(head_dim, false, true) {- panic!("Single-pass kernel not supported for head_dim={}", head_dim);- }-+ let kernel =+ AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, true, true, false, false)+ .expect("Failed to create AttentionSinglePassMetalKernel");let kernel_output =match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&mask), None, scale) {Ok(output) => output,@@ -754,7 +740,6 @@ fn test_single_pass_attention_with_mask() {}#[test]-#[ignore]fn test_single_pass_attention_with_sinks() {let context = MetalContext::new().expect("Failed to create MetalContext");@@ -770,16 +755,9 @@ fn test_single_pass_attention_with_sinks() {let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();println!("Using sinks: {:?}", sinks);- let kernel = match AttentionKernel::new(&context, DataType::F32) {- Ok(k) => k,- Err(e) => {- panic!("Failed to create AttentionKernel: {:?}", e);- },- };-- if !kernel.supports_single_pass(head_dim, false, false) {- panic!("Single-pass kernel not supported for head_dim={}", head_dim);- }+ let kernel =+ AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, true, false)+ .expect("Failed to create AttentionSinglePassMetalKernel");let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);@@ -798,7 +776,6 @@ fn test_single_pass_attention_with_sinks() {}#[test]-#[ignore]fn test_single_pass_attention_with_sinks_long_sequence() {let context = MetalContext::new().expect("Failed to create MetalContext");@@ -814,16 +791,9 @@ fn test_single_pass_attention_with_sinks_long_sequence() {let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();println!("Long sequence sinks: {:?}", sinks);- let kernel = match AttentionKernel::new(&context, DataType::F32) {- Ok(k) => k,- Err(e) => {- panic!("Failed to create AttentionKernel: {:?}", e);- },- };-- if !kernel.supports_single_pass(head_dim, false, false) {- panic!("Single-pass kernel not supported for head_dim={}", head_dim);- }+ let kernel =+ AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, true, false)+ .expect("Failed to create AttentionSinglePassMetalKernel");let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);@@ -847,7 +817,6 @@ fn test_single_pass_attention_with_sinks_long_sequence() {}#[test]-#[ignore]fn test_single_pass_attention_gqa() {let context = MetalContext::new().expect("Failed to create MetalContext");@@ -860,20 +829,10 @@ fn test_single_pass_attention_gqa() {let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);- let kernel = match AttentionKernel::new(&context, DataType::F32) {- Ok(k) => k,- Err(e) => {- panic!("Failed to create AttentionKernel: {:?}", e);- },- };-let is_causal = false; // Non-causal attention for this test- let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);- println!("Using kernel variant for GQA: {:?}", variant);-- if !kernel.supports_single_pass(head_dim, false, false) {- panic!("Single-pass kernel not supported for head_dim={}", head_dim);- }+ let kernel =+ AttentionSinglePassMetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, false, is_causal)+ .expect("Failed to create AttentionSinglePassMetalKernel");let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale){@@ -892,7 +851,8 @@ fn test_single_pass_attention_gqa() {}fn run_two_pass_attention(- kernel: &AttentionKernel,+ kernel_pass1: &AttentionTwoPass1MetalKernel,+ kernel_pass2: &AttentionTwoPass2MetalKernel,context: &MetalContext,queries: &Array4<f32>,keys: &Array4<f32>,@@ -940,35 +900,47 @@ fn run_two_pass_attention(let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- let args = AttentionTwoPassArguments {- queries_buffer: &queries_buffer,- keys_buffer: &keys_buffer,- values_buffer: &values_buffer,- partials_buffer: &partials_buffer,- sums_buffer: &sums_buffer,- maxs_buffer: &maxs_buffer,- output_buffer: &output_buffer,- gqa_factor: (num_heads / num_kv_heads) as i32,- sequence_length: seq_len as i32,- k_head_stride: (seq_len * head_dim) as i32,- k_seq_stride: head_dim as i32,- v_head_stride: (seq_len * head_dim) as i32,- v_seq_stride: head_dim as i32,+ let mut mask_kv_seq_stride: Option<u32> = None;+ let mut mask_q_seq_stride: Option<u32> = None;+ let mut mask_head_stride: Option<u32> = None;+ if mask_buffer.is_some() {+ mask_kv_seq_stride = Some(1);+ mask_q_seq_stride = Some(seq_len as u32);+ mask_head_stride = Some(0);+ }+ kernel_pass1.encode(+ &queries_buffer,+ &keys_buffer,+ &values_buffer,+ &partials_buffer,+ &sums_buffer,+ &maxs_buffer,+ (num_heads / num_kv_heads) as u32,+ seq_len as u32,+ (seq_len * head_dim) as u32,+ head_dim as u32,+ (seq_len * head_dim) as u32,+ head_dim as u32,scale,- mask_buffer: mask_buffer.as_deref(),- mask_kv_seq_stride: 1,- mask_q_seq_stride: seq_len as i32,- mask_head_stride: 0,- sinks_buffer: sinks_buffer.as_deref(),- num_heads,- suffix_length: seq_len,- head_dim,- is_causal: false,- };-- let encode_result = kernel.encode_two_pass(&compute_encoder, args);+ num_heads as u32,+ seq_len as u32,+ mask_buffer.as_ref().map(|b| b),+ mask_kv_seq_stride,+ mask_q_seq_stride,+ mask_head_stride,+ sinks_buffer.as_ref().map(|b| b),+ &compute_encoder,+ );+ kernel_pass2.encode(+ &partials_buffer,+ &sums_buffer,+ &maxs_buffer,+ &output_buffer,+ num_heads as u32,+ seq_len as u32,+ &compute_encoder,+ );compute_encoder.end_encoding();- encode_result?;command_buffer.commit();command_buffer.wait_until_completed();@@ -982,17 +954,9 @@ fn run_two_pass_attention(}#[test]-#[ignore]fn test_two_pass_attention() {let context = MetalContext::new().expect("Failed to create MetalContext");- let kernel = match AttentionKernel::new(&context, DataType::F32) {- Ok(k) => k,- Err(e) => {- panic!("Failed to create AttentionKernel: {:?}", e);- },- };-let batch_size = 1;let num_heads = 8;let num_kv_heads: usize = 8;@@ -1001,20 +965,26 @@ fn test_two_pass_attention() {let scale = 1.0 / (head_dim as f32).sqrt();let is_causal = false; // Non-causal attention for this test- if !kernel.supports_two_pass(head_dim, is_causal, false) {- panic!("Two-pass kernel not supported for head_dim={}", head_dim);- }-- let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);- if !matches!(variant, AttentionKernelVariant::TwoPass) {- panic!("Two-pass not selected for seq_len={}. Got {:?}", seq_len, variant);- }-let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);- let kernel_output = match run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale) {+ let kernel_pass1 =+ AttentionTwoPass1MetalKernel::new(&context, DataType::F32, head_dim as u32, false, false, false, is_causal)+ .expect("Failed to create AttentionTwoPass1MetalKernel");+ let kernel_pass2 = AttentionTwoPass2MetalKernel::new(&context, DataType::F32, head_dim as u32)+ .expect("Failed to create AttentionTwoPass2MetalKernel");+ let kernel_output = match run_two_pass_attention(+ &kernel_pass1,+ &kernel_pass2,+ &context,+ &queries,+ &keys,+ &values,+ None,+ None,+ scale,+ ) {Ok(output) => output,Err(e) => {panic!("Failed to run two-pass attention: {:?}", e);@@ -1028,17 +998,9 @@ fn test_two_pass_attention() {}#[test]-#[ignore]fn test_two_pass_attention_gqa() {let context = MetalContext::new().expect("Failed to create MetalContext");- let kernel = match AttentionKernel::new(&context, DataType::F32) {- Ok(k) => k,- Err(e) => {- panic!("Failed to create AttentionKernel: {:?}", e);- },- };-let batch_size = 1;let num_heads = 8;let num_kv_heads = 2;@@ -1047,20 +1009,26 @@ fn test_two_pass_attention_gqa() {let scale = 1.0 / (head_dim as f32).sqrt();let is_causal = false; // Non-causal attention for this test- if !kernel.supports_two_pass(head_dim, is_causal, false) {- panic!("Two-pass kernel not supported for head_dim={}", head_dim);- }-- let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);- if !matches!(variant, AttentionKernelVariant::TwoPass) {- panic!("Two-pass not selected for GQA seq_len={}. Got {:?}", seq_len, variant);- }-let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);- let kernel_output = match run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale) {+ let kernel_pass1 =+ AttentionTwoPass1MetalKernel::new(&context, DataType::F32, head_dim as u32, is_causal, false, false, false)+ .expect("Failed to create AttentionTwoPass1MetalKernel");+ let kernel_pass2 = AttentionTwoPass2MetalKernel::new(&context, DataType::F32, head_dim as u32)+ .expect("Failed to create AttentionTwoPass2MetalKernel");+ let kernel_output = match run_two_pass_attention(+ &kernel_pass1,+ &kernel_pass2,+ &context,+ &queries,+ &keys,+ &values,+ None,+ None,+ scale,+ ) {Ok(output) => output,Err(e) => {panic!("Failed to run two-pass attention GQA: {:?}", e);@@ -1074,19 +1042,11 @@ fn test_two_pass_attention_gqa() {}#[test]-#[ignore]fn perf_two_pass_attention() {use std::time::Instant;let context = MetalContext::new().expect("Failed to create MetalContext");- let kernel = match AttentionKernel::new(&context, DataType::F32) {- Ok(k) => k,- Err(e) => {- panic!("Failed to create AttentionKernel: {:?}", e);- },- };-// ---- Problem sizes requiring two-pass ----let batch_size = 1;let num_heads = 32;@@ -1097,17 +1057,6 @@ fn perf_two_pass_attention() {let scale = 1.0 / (head_dim as f32).sqrt();let is_causal = false;- if !kernel.supports_two_pass(head_dim, is_causal, false) {- println!("Skipping two-pass perf test: not supported for head_dim={}", head_dim);- return;- }-- let variant = kernel.choose_variant(seq_len, head_dim, is_causal, false);- if !matches!(variant, AttentionKernelVariant::TwoPass) {- println!("Skipping two-pass perf test: variant {:?} selected instead", variant);- return;- }-println!("Creating test data for two-pass performance test (prefix={}, suffix={})...",seq_len - suffix_length,@@ -1115,6 +1064,12 @@ fn perf_two_pass_attention() {);let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);+ let kernel_pass1 =+ AttentionTwoPass1MetalKernel::new(&context, DataType::F32, head_dim as u32, is_causal, false, false, false)+ .expect("Failed to create AttentionTwoPass1MetalKernel");+ let kernel_pass2 = AttentionTwoPass2MetalKernel::new(&context, DataType::F32, head_dim as u32)+ .expect("Failed to create AttentionTwoPass2MetalKernel");+// ---- Create buffers ----// For realistic inference, we only process queries for the suffix (new tokens)let queries_suffix = queries.slice(s![.., .., (seq_len - suffix_length).., ..]).to_owned();@@ -1150,35 +1105,41 @@ fn perf_two_pass_attention() {let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- let args = AttentionTwoPassArguments {- queries_buffer: &queries_buffer,- keys_buffer: &keys_buffer,- values_buffer: &values_buffer,- partials_buffer: &partials_buffer,- sums_buffer: &sums_buffer,- maxs_buffer: &maxs_buffer,- output_buffer: &output_buffer,- gqa_factor: (num_heads / num_kv_heads) as i32,- sequence_length: seq_len as i32,- k_head_stride: (seq_len * head_dim) as i32,- k_seq_stride: head_dim as i32,- v_head_stride: (seq_len * head_dim) as i32,- v_seq_stride: head_dim as i32,+ let mask_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;+ let sinks_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;+ kernel_pass1.encode(+ &queries_buffer,+ &keys_buffer,+ &values_buffer,+ &partials_buffer,+ &sums_buffer,+ &maxs_buffer,+ (num_heads / num_kv_heads) as u32,+ seq_len as u32,+ (seq_len * head_dim) as u32,+ head_dim as u32,+ (seq_len * head_dim) as u32,+ head_dim as u32,scale,- mask_buffer: None,- mask_kv_seq_stride: 1,- mask_q_seq_stride: suffix_length as i32,- mask_head_stride: 0,- sinks_buffer: None,- num_heads,- suffix_length, // Use actual suffix_length, not seq_len- head_dim,- is_causal: false,- };-- let encode_result = kernel.encode_two_pass(&compute_encoder, args);+ num_heads as u32,+ suffix_length as u32,+ mask_buffer.as_ref().map(|b| b),+ None,+ None,+ None,+ sinks_buffer.as_ref().map(|b| b),+ &compute_encoder,+ );+ kernel_pass2.encode(+ &partials_buffer,+ &sums_buffer,+ &maxs_buffer,+ &output_buffer,+ num_heads as u32,+ suffix_length as u32,+ &compute_encoder,+ );compute_encoder.end_encoding();- encode_result.expect("encode");// Time both host-side and GPU executionlet host_timer = Instant::now();