A high-performance inference engine for AI models
GQA-aware two-pass attention (#317)
CCYeh6a537c5ad2c2
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rsindex ba23f1dd1..5bbfb3148 100644--- a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs@@ -8,7 +8,6 @@ use crate::{};const TOTAL_BLOCKS_COUNT: u32 = 32;-const SEQUENCE_BLOCK_SIZE_1: u32 = 8;#[kernel(AttentionTwoPass1)]#[variants(T, f32, f16, bf16)]@@ -80,55 +79,47 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(let mut sum_exp_score = 0.0f32;if has_sinks && block_idx == 0 {- let q_head_idx = head_idx % num_heads;- max_score = unsafe { *sinks.unwrap().add(q_head_idx as usize) };+ max_score = unsafe { *sinks.unwrap().add(head_idx as usize) };sum_exp_score = 1.0;}// Iterate over sequence positions assigned to this block- let mut chunk_start = block_idx * SEQUENCE_BLOCK_SIZE_1;- while chunk_start < sequence_length {- for t in 0..SEQUENCE_BLOCK_SIZE_1 {- let i = chunk_start + t;- if i >= sequence_length {- break;+ let mut i = block_idx;+ while i < sequence_length {+ if should_use_key(+ ring_params,+ trie,+ sliding_window_size,+ q_seq_idx,+ prefix_length,+ suffix_position,+ query_position,+ i,+ is_causal,+ ) {+ let keys_ptr = unsafe { keys_base.add((i * k_seq_stride) as usize) };++ // Compute the i-th score+ let mut score = 0.0f32;+ for j in 0..HEAD_DIM as usize {+ score += q[j] * unsafe { *keys_ptr.add(j) }.to_f32().unwrap();}- if should_use_key(- ring_params,- trie,- sliding_window_size,- q_seq_idx,- prefix_length,- suffix_position,- query_position,- i,- is_causal,- ) {- let keys_ptr = unsafe { keys_base.add((i * k_seq_stride) as usize) };-- // Compute the i-th score- let mut score = 0.0f32;- for j in 0..HEAD_DIM as usize {- score += q[j] * unsafe { *keys_ptr.add(j) }.to_f32().unwrap();- }-- // Update the accumulators- let new_max = f32::max(max_score, score);- let factor = (max_score - new_max).exp();- let exp_score = (score - new_max).exp();-- max_score = new_max;- sum_exp_score = sum_exp_score * factor + exp_score;-- // Update the output accumulator- let values_ptr = unsafe { values_base.add((i * v_seq_stride) as usize) };- for j in 0..HEAD_DIM as usize {- o[j] = o[j] * factor + exp_score * unsafe { *values_ptr.add(j) }.to_f32().unwrap();- }+ // Update the accumulators+ let new_max = f32::max(max_score, score);+ let factor = (max_score - new_max).exp();+ let exp_score = (score - new_max).exp();++ max_score = new_max;+ sum_exp_score = sum_exp_score * factor + exp_score;++ // Update the output accumulator+ let values_ptr = unsafe { values_base.add((i * v_seq_stride) as usize) };+ for j in 0..HEAD_DIM as usize {+ o[j] = o[j] * factor + exp_score * unsafe { *values_ptr.add(j) }.to_f32().unwrap();}}- chunk_start += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1;+ i += TOTAL_BLOCKS_COUNT;}// Write partial output, sum, and max for this blockdiff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metalindex ab3cf52f8..9c3223e1a 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -1,19 +1,22 @@#include <metal_stdlib>#include <metal_simdgroup>+#include "../common/defines.h"#include "../common/dsl.h"#include "../common/thread_context.h"#include "../generated/ring.h"#include "../generated/trie.h"#include "mask.h"-#define HEAD_BLOCK_SIZE 32-#define SEQUENCE_BLOCK_SIZE_1 8-#define SEQUENCE_BLOCK_SIZE_2 32#define TOTAL_BLOCKS_COUNT 32+#define SIMDGROUPS_PER_THREADGROUP 8using namespace uzu::ring;using namespace uzu::trie;+// GQA-aware two-pass attention (pass 1).+// Fixed 256-thread threadgroups (8 simdgroups) with flat query-head mapping.+// Each simdgroup independently handles one query head and derives its KV head+// via integer division. Simdgroups sharing a KV head benefit from L1 cache.template <typename T, uint HEAD_DIM>VARIANTS(T, float, half, bfloat)VARIANTS(HEAD_DIM, 64, 128, 256)@@ -42,82 +45,58 @@ PUBLIC KERNEL(AttentionTwoPass1)(const bool is_causal SPECIALIZE,const bool is_trie SPECIALIZE,const bool is_sliding_window SPECIALIZE,- threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE_1],- threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE_1],- threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_1 * HEAD_BLOCK_SIZE],const ThreadContext thread_context,- const uint head_idx GROUPS(num_heads),+ const uint head_group_idx GROUPS(num_heads.div_ceil(SIMDGROUPS_PER_THREADGROUP)),const uint q_seq_idx GROUPS(suffix_length),const uint block_idx GROUPS(TOTAL_BLOCKS_COUNT),- const uint tid THREADS(256)+ const uint tid THREADS(SIMDGROUPS_PER_THREADGROUP * METAL_SIMD_SIZE)) {- constexpr bool query_transposed = false;- constexpr uint value_dim = HEAD_DIM;- constexpr uint qk_elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;- constexpr uint value_elements_per_thread = value_dim / HEAD_BLOCK_SIZE;- uint inner_k_stride = SEQUENCE_BLOCK_SIZE_1 * k_seq_stride;- uint inner_v_stride = SEQUENCE_BLOCK_SIZE_1 * v_seq_stride;+ constexpr uint elements_per_thread = HEAD_DIM / METAL_SIMD_SIZE;typedef float U;- thread U q[qk_elements_per_thread];- thread U k[qk_elements_per_thread];- thread U o[value_elements_per_thread];+ // Flat query head mapping — fixed hardware shape, dynamic logical mapping+ const uint head_idx = head_group_idx * SIMDGROUPS_PER_THREADGROUP ++ thread_context.threadgroup_index;+ if (head_idx >= num_heads)+ return;- const uint o_offset = q_seq_idx * num_heads + head_idx; // Our custom layout- const uint q_offset =- query_transposed- ? num_heads * q_seq_idx + head_idx- : head_idx * suffix_length + q_seq_idx; // Consistent with single-passconst uint kv_head_idx = head_idx / gqa_factor;+ const uint o_offset = q_seq_idx * num_heads + head_idx;+ const uint q_offset = head_idx * suffix_length + q_seq_idx;const uint prefix_length = sequence_length - suffix_length;-const uint suffix_position =is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;-const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height: suffix_position + q_seq_idx;+ thread U q[elements_per_thread];+ thread U o[elements_per_thread] = {0};+queries += q_offset * HEAD_DIM +- thread_context.simdgroup_index * qk_elements_per_thread;- keys +=- kv_head_idx * k_head_stride +- (block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index) *- k_seq_stride +- thread_context.simdgroup_index * qk_elements_per_thread;- values +=- kv_head_idx * v_head_stride +- (block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index) *- v_seq_stride +- thread_context.simdgroup_index * value_elements_per_thread;- out += o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim +- thread_context.simdgroup_index * value_elements_per_thread;+ thread_context.simdgroup_index * elements_per_thread;+ keys += kv_head_idx * k_head_stride + block_idx * k_seq_stride ++ thread_context.simdgroup_index * elements_per_thread;+ values += kv_head_idx * v_head_stride + block_idx * v_seq_stride ++ thread_context.simdgroup_index * elements_per_thread;+ out += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM + block_idx * HEAD_DIM ++ thread_context.simdgroup_index * elements_per_thread;sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;maxs += o_offset * TOTAL_BLOCKS_COUNT + block_idx;- // Read the query and 0 the output accumulator- for (uint i = 0; i < qk_elements_per_thread; i++) {+ for (uint i = 0; i < elements_per_thread; i++) {q[i] = static_cast<U>(scale) * queries[i];}- for (uint i = 0; i < value_elements_per_thread; i++) {- o[i] = 0;- }-U max_score = -1e9;U sum_exp_score = 0;- if (has_sinks && block_idx == 0 && thread_context.threadgroup_index == 0) {- const uint num_q_heads = num_heads;- int q_head_idx = head_idx % num_q_heads;- max_score = static_cast<U>(sinks[q_head_idx]);+ if (has_sinks && block_idx == 0) {+ max_score = static_cast<U>(sinks[head_idx]);sum_exp_score = 1;}- // For each key- for (uint i =- block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index;- i < sequence_length;- i += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1) {+ // For each key position+ for (uint i = block_idx; i < sequence_length; i += TOTAL_BLOCKS_COUNT) {if (should_use_key(ring_params,trie,@@ -132,81 +111,36 @@ PUBLIC KERNEL(AttentionTwoPass1)(is_trie,is_sliding_window)) {- // Read the key- for (uint j = 0; j < qk_elements_per_thread; j++) {- k[j] = keys[j];- }-- // Compute the i-th score+ // Compute scoreU score = 0;- for (uint j = 0; j < qk_elements_per_thread; j++) {- score += q[j] * k[j];+ for (uint j = 0; j < elements_per_thread; j++) {+ score += q[j] * static_cast<U>(keys[j]);}score = simd_sum(score);- // Update the accumulators+ // Online softmax updateU new_max = max(max_score, score);U factor = fast::exp(max_score - new_max);U exp_score = fast::exp(score - new_max);-max_score = new_max;sum_exp_score = sum_exp_score * factor + exp_score;- // Update the output accumulator- for (uint j = 0; j < value_elements_per_thread; j++) {- o[j] = o[j] * factor + exp_score * values[j];+ for (uint j = 0; j < elements_per_thread; j++) {+ o[j] = o[j] * factor + exp_score * static_cast<U>(values[j]);}}- // Move the pointers to the next kv- keys += TOTAL_BLOCKS_COUNT * inner_k_stride;- values += TOTAL_BLOCKS_COUNT * inner_v_stride;+ keys += TOTAL_BLOCKS_COUNT * k_seq_stride;+ values += TOTAL_BLOCKS_COUNT * v_seq_stride;}- // Each thread has a partial part of the output so we need to combine them.+ // Write partial outputif (thread_context.simdgroup_index == 0) {- shared_max_scores[thread_context.threadgroup_index] = max_score;- shared_sum_exp_scores[thread_context.threadgroup_index] = sum_exp_score;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- max_score = (thread_context.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)- ? shared_max_scores[thread_context.simdgroup_index]- : -1e9;- U new_max = simd_max(max_score);- U factor = fast::exp(max_score - new_max);- sum_exp_score = (thread_context.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)- ? shared_sum_exp_scores[thread_context.simdgroup_index]- : 0;- sum_exp_score = simd_sum(sum_exp_score * factor);-- // Write the sum and new max- if (thread_context.threadgroup_index == 0) {sums[0] = sum_exp_score;- maxs[0] = new_max;+ maxs[0] = max_score;}-- // Now we need to aggregate all the outputs- for (uint i = 0; i < value_elements_per_thread; i++) {- shared_outputs- [thread_context.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 +- thread_context.threadgroup_index] =- o[i] *- fast::exp(- shared_max_scores[thread_context.threadgroup_index] - new_max- );- threadgroup_barrier(mem_flags::mem_threadgroup);-- // And write the output- if (thread_context.threadgroup_index == 0) {- U output = shared_outputs- [thread_context.simdgroup_index * SEQUENCE_BLOCK_SIZE_1];- for (uint j = 1; j < SEQUENCE_BLOCK_SIZE_1; j++) {- output += shared_outputs- [thread_context.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 + j];- }- out[i] = static_cast<T>(output);- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ for (uint i = 0; i < elements_per_thread; i++) {+ out[i] = o[i];}}@@ -220,58 +154,63 @@ PUBLIC KERNEL(AttentionTwoPass2)(device T* out,const constant uint& num_heads,const constant uint& suffix_length,- threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_2 * HEAD_BLOCK_SIZE],const ThreadContext thread_context,const uint head_idx GROUPS(num_heads),const uint q_seq_idx GROUPS(suffix_length),const uint tid THREADS(1024)) {- constexpr uint elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;+ constexpr uint elements_per_thread = HEAD_DIM / METAL_SIMD_SIZE;typedef float U;- thread U o[elements_per_thread];- const uint o_offset = q_seq_idx * num_heads + head_idx; // Our custom layout+ const uint o_offset = q_seq_idx * num_heads + head_idx;- partials += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +- thread_context.threadgroup_index * HEAD_DIM +- thread_context.simdgroup_index * elements_per_thread;sums += o_offset * TOTAL_BLOCKS_COUNT;maxs += o_offset * TOTAL_BLOCKS_COUNT;- out +=- o_offset * HEAD_DIM + thread_context.threadgroup_index *- elements_per_thread; // Our custom output layout- // First everybody reads the max and sum_exp- U max_score = maxs[thread_context.simdgroup_index];+ // Find global max across all blocks (each lane handles strided blocks)+ U max_score = -1e9;+ for (uint b = thread_context.simdgroup_index; b < TOTAL_BLOCKS_COUNT;+ b += METAL_SIMD_SIZE) {+ max_score = max(max_score, maxs[b]);+ }U new_max = simd_max(max_score);- U factor = fast::exp(max_score - new_max);- U sum_exp_score = simd_sum(sums[thread_context.simdgroup_index] * factor);- // Now read the block into registers and then use shared memory to transpose- // it- for (uint i = 0; i < elements_per_thread; i++) {- o[i] = partials[i];+ // Find global sum across all blocks+ U sum_exp = 0;+ for (uint b = thread_context.simdgroup_index; b < TOTAL_BLOCKS_COUNT;+ b += METAL_SIMD_SIZE) {+ sum_exp += sums[b] * fast::exp(maxs[b] - new_max);}- for (uint i = 0; i < elements_per_thread; i++) {- shared_outputs- [thread_context.simdgroup_index * HEAD_BLOCK_SIZE +- thread_context.threadgroup_index] = o[i];- threadgroup_barrier(mem_flags::mem_threadgroup);- o[i] = simd_sum(- shared_outputs- [thread_context.threadgroup_index * HEAD_BLOCK_SIZE +- thread_context.simdgroup_index] *- factor- ) /- sum_exp_score;- threadgroup_barrier(mem_flags::mem_threadgroup);+ U sum_exp_score = simd_sum(sum_exp);++ // Combine partial outputs — each lane reads a different block,+ // simd_sum reduces across blocks directly (no shared memory needed)+ thread U o[elements_per_thread] = {0};++ for (uint chunk = 0; chunk < TOTAL_BLOCKS_COUNT; chunk += METAL_SIMD_SIZE) {+ uint block_idx = chunk + thread_context.simdgroup_index;+ U factor = (block_idx < TOTAL_BLOCKS_COUNT)+ ? fast::exp(maxs[block_idx] - new_max)+ : 0;++ const device float* block_partials =+ partials + o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM ++ block_idx * HEAD_DIM ++ thread_context.threadgroup_index * elements_per_thread;++ for (uint i = 0; i < elements_per_thread; i++) {+ U val = (block_idx < TOTAL_BLOCKS_COUNT) ? block_partials[i] * factor : 0;+ o[i] += simd_sum(val);+ }}- // And write the output+ // Write the outputif (thread_context.simdgroup_index == 0) {+ out += o_offset * HEAD_DIM ++ thread_context.threadgroup_index * elements_per_thread;for (uint i = 0; i < elements_per_thread; i++) {- out[i] = static_cast<T>(o[i]);+ out[i] = static_cast<T>(o[i] / sum_exp_score);}}}diff --git a/crates/uzu/tests/unit/encodable_block/attention_test.rs b/crates/uzu/tests/unit/encodable_block/attention_test.rsindex 82bf895b3..3f08b7e08 100644--- a/crates/uzu/tests/unit/encodable_block/attention_test.rs+++ b/crates/uzu/tests/unit/encodable_block/attention_test.rs@@ -968,13 +968,13 @@ fn perf_two_pass_attention() {false,false,)- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");+ .expect("Failed to create AttentionTwoPass1Kernel");let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(&context,DataType::F32,head_dim as u32,)- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");+ .expect("Failed to create AttentionTwoPass2Kernel");// ---- Create buffers ----// For realistic inference, we only process queries for the suffix (new tokens)