Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

gemm attention fixes

LuckyIYI21157303bc20
diff --git a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
index 2190ff680..c348112a6 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
@@ -442,18 +442,18 @@ template <
}
///////////////////////////////////////////////////////////////////////////////
-// Kernel instantiations (BQ=32, BK=32, WM=2, WN=2)
+// Kernel instantiations (BQ=32, BK in {32,16}, WM=2, WN=2)
///////////////////////////////////////////////////////////////////////////////
-#define instantiate_attention_gemm(type_name, element_type, head_dim_value) \
- template [[host_name("attention_gemm_" #type_name "_" #head_dim_value)]] \
+#define instantiate_attention_gemm(type_name, element_type, head_dim_value, bk_value) \
+ template [[host_name("attention_gemm_" #type_name "_" #head_dim_value "_bk" #bk_value)]] \
[[kernel]] void attention_gemm< \
element_type, \
32, \
- 32, \
+ bk_value, \
head_dim_value, \
- 2, \
- 2, \
+ 4, \
+ 1, \
float>( \
const device element_type* Q [[buffer(0)]], \
const device element_type* K [[buffer(1)]], \
@@ -469,17 +469,17 @@ template <
uint3 tid [[threadgroup_position_in_grid]], \
uint3 lid [[thread_position_in_threadgroup]]);
-instantiate_attention_gemm(f16, half, 64)
-instantiate_attention_gemm(f16, half, 128)
-instantiate_attention_gemm(f16, half, 256)
+instantiate_attention_gemm(f16, half, 64, 32)
+instantiate_attention_gemm(f16, half, 128, 16)
+instantiate_attention_gemm(f16, half, 256, 16)
-instantiate_attention_gemm(bf16, bfloat, 64)
-instantiate_attention_gemm(bf16, bfloat, 128)
-instantiate_attention_gemm(bf16, bfloat, 256)
+instantiate_attention_gemm(bf16, bfloat, 64, 32)
+instantiate_attention_gemm(bf16, bfloat, 128, 16)
+instantiate_attention_gemm(bf16, bfloat, 256, 16)
-instantiate_attention_gemm(f32, float, 64)
-instantiate_attention_gemm(f32, float, 128)
-instantiate_attention_gemm(f32, float, 256)
+instantiate_attention_gemm(f32, float, 64, 32)
+instantiate_attention_gemm(f32, float, 128, 16)
+instantiate_attention_gemm(f32, float, 256, 16)
#undef instantiate_attention_gemm
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
index ffec0bb84..2ee51879d 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
@@ -585,16 +585,21 @@ impl AttentionKernel {
args: AttentionGemmArguments,
) -> Result<(), AttentionError> {
const BQ: usize = 32;
- const BK: usize = 32;
- const WM: u64 = 2;
- const WN: u64 = 2;
+ const WM: u64 = 4;
+ const WN: u64 = 1;
if !matches!(args.head_dim, 64 | 128 | 256) {
return Err(AttentionError::UnsupportedHeadDim(args.head_dim));
}
+ let bk: usize = if args.head_dim < 128 {
+ 32
+ } else {
+ 16
+ };
+
let align_q = (args.suffix_length % BQ) == 0;
- let align_k = (args.sequence_length % BK) == 0;
+ let align_k = (args.sequence_length % bk) == 0;
let has_mask = args.mask_buffer.is_some();
let has_sinks = args.sinks_buffer.is_some();
@@ -626,7 +631,7 @@ impl AttentionKernel {
);
// Kernel name matches gemm_attention.metal instantiations:
- // attention_gemm_{f16|bf16|f32}_{head_dim}
+ // attention_gemm_{f16|bf16|f32}_{head_dim}_bk{bk}
let type_name = match self.data_type {
KernelDataType::Float16 => "f16",
KernelDataType::BFloat16 => "bf16",
@@ -634,7 +639,7 @@ impl AttentionKernel {
};
let function_name =
- format!("attention_gemm_{}_{}", type_name, args.head_dim);
+ format!("attention_gemm_{}_{}_bk{}", type_name, args.head_dim, bk);
let cache_key = format!(
"{}_aq{}_ak{}_m{}_c{}_s{}",
@@ -673,9 +678,9 @@ impl AttentionKernel {
let o_seq_stride = (args.num_heads * args.head_dim) as i64;
let nq = (args.suffix_length + BQ - 1) / BQ;
- let nk = (args.sequence_length + BK - 1) / BK;
+ let nk = (args.sequence_length + bk - 1) / bk;
let nq_aligned = args.suffix_length / BQ;
- let nk_aligned = args.sequence_length / BK;
+ let nk_aligned = args.sequence_length / bk;
let params = AttnParams {
q_strides: [0, q_head_stride, q_seq_stride],
@@ -691,7 +696,7 @@ impl AttentionKernel {
q_rem: (args.suffix_length - nq_aligned * BQ) as i32,
nk: nk as i32,
nk_aligned: nk_aligned as i32,
- k_rem: (args.sequence_length - nk_aligned * BK) as i32,
+ k_rem: (args.sequence_length - nk_aligned * bk) as i32,
};
compute_encoder.set_bytes(
diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rs
index e241ab646..88e4f7b3e 100644
--- a/crates/uzu/tests/attention_kernel_test.rs
+++ b/crates/uzu/tests/attention_kernel_test.rs
@@ -709,7 +709,7 @@ fn test_single_pass_attention_basic() {
},
};
- let tolerance = 1e-1;
+ let tolerance = 1e-2;
if let Err(e) = compare_results(
&kernel_output,
&reference_output,
@@ -759,7 +759,7 @@ fn test_gemm_attention_basic() {
)
.expect("run gemm attention");
- let tolerance = 1e-1;
+ let tolerance = 1e-2;
if let Err(e) = compare_results(
&kernel_output,
&reference_output,
@@ -770,6 +770,56 @@ fn test_gemm_attention_basic() {
}
}
+#[test]
+fn test_gemm_attention_f32_head_dim_128() {
+ let device = Device::system_default().expect("No Metal device found");
+ let command_queue = device.new_command_queue();
+ let context = match MTLContext::new(device, command_queue) {
+ Ok(ctx) => ctx,
+ Err(e) => {
+ panic!("Failed to create MTLContext: {:?}", e);
+ },
+ };
+
+ let batch_size = 1;
+ let num_heads = 4;
+ let num_kv_heads = 4;
+ let seq_len = 32;
+ let head_dim = 128;
+ let scale = 1.0 / (head_dim as f32).sqrt();
+
+ let (queries, keys, values, _mask) = create_test_data(
+ batch_size,
+ num_heads,
+ num_kv_heads,
+ seq_len,
+ head_dim,
+ 456,
+ );
+
+ let kernel = AttentionKernel::new(&context, KernelDataType::Float32)
+ .expect("Failed to create AttentionKernel");
+
+ let reference_output =
+ reference_attention(&queries, &keys, &values, None, None, scale);
+
+ let kernel_output = run_gemm_attention(
+ &kernel, &context, &queries, &keys, &values, None, None, scale,
+ /*is_causal=*/ false,
+ )
+ .expect("run gemm attention f32 head_dim=128");
+
+ let tolerance = 1e-2;
+ if let Err(e) = compare_results(
+ &kernel_output,
+ &reference_output,
+ tolerance,
+ "Gemm attention f32 head_dim=128",
+ ) {
+ panic!("{}", e);
+ }
+}
+
#[test]
fn test_matrix_attention_matches_vector_and_cpu_seq256() {
let device = Device::system_default().expect("No Metal device found");
@@ -1158,7 +1208,7 @@ fn test_single_pass_attention_gqa() {
let reference_output =
reference_attention(&queries, &keys, &values, None, None, scale);
- let tolerance = 1e-1;
+ let tolerance = 1e-2;
if let Err(e) = compare_results(
&kernel_output,
&reference_output,
@@ -1322,7 +1372,7 @@ fn test_two_pass_attention() {
},
};
- let tolerance = 1e-1;
+ let tolerance = 1e-2;
if let Err(e) = compare_results(
&kernel_output,
&reference_output,
@@ -1392,7 +1442,7 @@ fn test_two_pass_attention_gqa() {
},
};
- let tolerance = 1e-1;
+ let tolerance = 1e-2;
if let Err(e) = compare_results(
&kernel_output,
&reference_output,