A high-performance inference engine for AI models
gemm attention fixes
LuckyIYI21157303bc20
diff --git a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metalindex 2190ff680..c348112a6 100644--- a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal@@ -442,18 +442,18 @@ template <}///////////////////////////////////////////////////////////////////////////////-// Kernel instantiations (BQ=32, BK=32, WM=2, WN=2)+// Kernel instantiations (BQ=32, BK in {32,16}, WM=2, WN=2)///////////////////////////////////////////////////////////////////////////////-#define instantiate_attention_gemm(type_name, element_type, head_dim_value) \- template [[host_name("attention_gemm_" #type_name "_" #head_dim_value)]] \+#define instantiate_attention_gemm(type_name, element_type, head_dim_value, bk_value) \+ template [[host_name("attention_gemm_" #type_name "_" #head_dim_value "_bk" #bk_value)]] \[[kernel]] void attention_gemm< \element_type, \32, \- 32, \+ bk_value, \head_dim_value, \- 2, \- 2, \+ 4, \+ 1, \float>( \const device element_type* Q [[buffer(0)]], \const device element_type* K [[buffer(1)]], \@@ -469,17 +469,17 @@ template <uint3 tid [[threadgroup_position_in_grid]], \uint3 lid [[thread_position_in_threadgroup]]);-instantiate_attention_gemm(f16, half, 64)-instantiate_attention_gemm(f16, half, 128)-instantiate_attention_gemm(f16, half, 256)+instantiate_attention_gemm(f16, half, 64, 32)+instantiate_attention_gemm(f16, half, 128, 16)+instantiate_attention_gemm(f16, half, 256, 16)-instantiate_attention_gemm(bf16, bfloat, 64)-instantiate_attention_gemm(bf16, bfloat, 128)-instantiate_attention_gemm(bf16, bfloat, 256)+instantiate_attention_gemm(bf16, bfloat, 64, 32)+instantiate_attention_gemm(bf16, bfloat, 128, 16)+instantiate_attention_gemm(bf16, bfloat, 256, 16)-instantiate_attention_gemm(f32, float, 64)-instantiate_attention_gemm(f32, float, 128)-instantiate_attention_gemm(f32, float, 256)+instantiate_attention_gemm(f32, float, 64, 32)+instantiate_attention_gemm(f32, float, 128, 16)+instantiate_attention_gemm(f32, float, 256, 16)#undef instantiate_attention_gemmdiff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rsindex ffec0bb84..2ee51879d 100644--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs@@ -585,16 +585,21 @@ impl AttentionKernel {args: AttentionGemmArguments,) -> Result<(), AttentionError> {const BQ: usize = 32;- const BK: usize = 32;- const WM: u64 = 2;- const WN: u64 = 2;+ const WM: u64 = 4;+ const WN: u64 = 1;if !matches!(args.head_dim, 64 | 128 | 256) {return Err(AttentionError::UnsupportedHeadDim(args.head_dim));}+ let bk: usize = if args.head_dim < 128 {+ 32+ } else {+ 16+ };+let align_q = (args.suffix_length % BQ) == 0;- let align_k = (args.sequence_length % BK) == 0;+ let align_k = (args.sequence_length % bk) == 0;let has_mask = args.mask_buffer.is_some();let has_sinks = args.sinks_buffer.is_some();@@ -626,7 +631,7 @@ impl AttentionKernel {);// Kernel name matches gemm_attention.metal instantiations:- // attention_gemm_{f16|bf16|f32}_{head_dim}+ // attention_gemm_{f16|bf16|f32}_{head_dim}_bk{bk}let type_name = match self.data_type {KernelDataType::Float16 => "f16",KernelDataType::BFloat16 => "bf16",@@ -634,7 +639,7 @@ impl AttentionKernel {};let function_name =- format!("attention_gemm_{}_{}", type_name, args.head_dim);+ format!("attention_gemm_{}_{}_bk{}", type_name, args.head_dim, bk);let cache_key = format!("{}_aq{}_ak{}_m{}_c{}_s{}",@@ -673,9 +678,9 @@ impl AttentionKernel {let o_seq_stride = (args.num_heads * args.head_dim) as i64;let nq = (args.suffix_length + BQ - 1) / BQ;- let nk = (args.sequence_length + BK - 1) / BK;+ let nk = (args.sequence_length + bk - 1) / bk;let nq_aligned = args.suffix_length / BQ;- let nk_aligned = args.sequence_length / BK;+ let nk_aligned = args.sequence_length / bk;let params = AttnParams {q_strides: [0, q_head_stride, q_seq_stride],@@ -691,7 +696,7 @@ impl AttentionKernel {q_rem: (args.suffix_length - nq_aligned * BQ) as i32,nk: nk as i32,nk_aligned: nk_aligned as i32,- k_rem: (args.sequence_length - nk_aligned * BK) as i32,+ k_rem: (args.sequence_length - nk_aligned * bk) as i32,};compute_encoder.set_bytes(diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rsindex e241ab646..88e4f7b3e 100644--- a/crates/uzu/tests/attention_kernel_test.rs+++ b/crates/uzu/tests/attention_kernel_test.rs@@ -709,7 +709,7 @@ fn test_single_pass_attention_basic() {},};- let tolerance = 1e-1;+ let tolerance = 1e-2;if let Err(e) = compare_results(&kernel_output,&reference_output,@@ -759,7 +759,7 @@ fn test_gemm_attention_basic() {).expect("run gemm attention");- let tolerance = 1e-1;+ let tolerance = 1e-2;if let Err(e) = compare_results(&kernel_output,&reference_output,@@ -770,6 +770,56 @@ fn test_gemm_attention_basic() {}}+#[test]+fn test_gemm_attention_f32_head_dim_128() {+ let device = Device::system_default().expect("No Metal device found");+ let command_queue = device.new_command_queue();+ let context = match MTLContext::new(device, command_queue) {+ Ok(ctx) => ctx,+ Err(e) => {+ panic!("Failed to create MTLContext: {:?}", e);+ },+ };++ let batch_size = 1;+ let num_heads = 4;+ let num_kv_heads = 4;+ let seq_len = 32;+ let head_dim = 128;+ let scale = 1.0 / (head_dim as f32).sqrt();++ let (queries, keys, values, _mask) = create_test_data(+ batch_size,+ num_heads,+ num_kv_heads,+ seq_len,+ head_dim,+ 456,+ );++ let kernel = AttentionKernel::new(&context, KernelDataType::Float32)+ .expect("Failed to create AttentionKernel");++ let reference_output =+ reference_attention(&queries, &keys, &values, None, None, scale);++ let kernel_output = run_gemm_attention(+ &kernel, &context, &queries, &keys, &values, None, None, scale,+ /*is_causal=*/ false,+ )+ .expect("run gemm attention f32 head_dim=128");++ let tolerance = 1e-2;+ if let Err(e) = compare_results(+ &kernel_output,+ &reference_output,+ tolerance,+ "Gemm attention f32 head_dim=128",+ ) {+ panic!("{}", e);+ }+}+#[test]fn test_matrix_attention_matches_vector_and_cpu_seq256() {let device = Device::system_default().expect("No Metal device found");@@ -1158,7 +1208,7 @@ fn test_single_pass_attention_gqa() {let reference_output =reference_attention(&queries, &keys, &values, None, None, scale);- let tolerance = 1e-1;+ let tolerance = 1e-2;if let Err(e) = compare_results(&kernel_output,&reference_output,@@ -1322,7 +1372,7 @@ fn test_two_pass_attention() {},};- let tolerance = 1e-1;+ let tolerance = 1e-2;if let Err(e) = compare_results(&kernel_output,&reference_output,@@ -1392,7 +1442,7 @@ fn test_two_pass_attention_gqa() {},};- let tolerance = 1e-1;+ let tolerance = 1e-2;if let Err(e) = compare_results(&kernel_output,&reference_output,