Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

GQA-aware two-pass attention (#317)

CCYeh6a537c5ad2c2
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
index ba23f1dd1..5bbfb3148 100644
--- a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
@@ -8,7 +8,6 @@ use crate::{
};
const TOTAL_BLOCKS_COUNT: u32 = 32;
-const SEQUENCE_BLOCK_SIZE_1: u32 = 8;
#[kernel(AttentionTwoPass1)]
#[variants(T, f32, f16, bf16)]
@@ -80,55 +79,47 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(
let mut sum_exp_score = 0.0f32;
if has_sinks && block_idx == 0 {
- let q_head_idx = head_idx % num_heads;
- max_score = unsafe { *sinks.unwrap().add(q_head_idx as usize) };
+ max_score = unsafe { *sinks.unwrap().add(head_idx as usize) };
sum_exp_score = 1.0;
}
// Iterate over sequence positions assigned to this block
- let mut chunk_start = block_idx * SEQUENCE_BLOCK_SIZE_1;
- while chunk_start < sequence_length {
- for t in 0..SEQUENCE_BLOCK_SIZE_1 {
- let i = chunk_start + t;
- if i >= sequence_length {
- break;
+ let mut i = block_idx;
+ while i < sequence_length {
+ if should_use_key(
+ ring_params,
+ trie,
+ sliding_window_size,
+ q_seq_idx,
+ prefix_length,
+ suffix_position,
+ query_position,
+ i,
+ is_causal,
+ ) {
+ let keys_ptr = unsafe { keys_base.add((i * k_seq_stride) as usize) };
+
+ // Compute the i-th score
+ let mut score = 0.0f32;
+ for j in 0..HEAD_DIM as usize {
+ score += q[j] * unsafe { *keys_ptr.add(j) }.to_f32().unwrap();
}
- if should_use_key(
- ring_params,
- trie,
- sliding_window_size,
- q_seq_idx,
- prefix_length,
- suffix_position,
- query_position,
- i,
- is_causal,
- ) {
- let keys_ptr = unsafe { keys_base.add((i * k_seq_stride) as usize) };
-
- // Compute the i-th score
- let mut score = 0.0f32;
- for j in 0..HEAD_DIM as usize {
- score += q[j] * unsafe { *keys_ptr.add(j) }.to_f32().unwrap();
- }
-
- // Update the accumulators
- let new_max = f32::max(max_score, score);
- let factor = (max_score - new_max).exp();
- let exp_score = (score - new_max).exp();
-
- max_score = new_max;
- sum_exp_score = sum_exp_score * factor + exp_score;
-
- // Update the output accumulator
- let values_ptr = unsafe { values_base.add((i * v_seq_stride) as usize) };
- for j in 0..HEAD_DIM as usize {
- o[j] = o[j] * factor + exp_score * unsafe { *values_ptr.add(j) }.to_f32().unwrap();
- }
+ // Update the accumulators
+ let new_max = f32::max(max_score, score);
+ let factor = (max_score - new_max).exp();
+ let exp_score = (score - new_max).exp();
+
+ max_score = new_max;
+ sum_exp_score = sum_exp_score * factor + exp_score;
+
+ // Update the output accumulator
+ let values_ptr = unsafe { values_base.add((i * v_seq_stride) as usize) };
+ for j in 0..HEAD_DIM as usize {
+ o[j] = o[j] * factor + exp_score * unsafe { *values_ptr.add(j) }.to_f32().unwrap();
}
}
- chunk_start += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1;
+ i += TOTAL_BLOCKS_COUNT;
}
// Write partial output, sum, and max for this block
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
index ab3cf52f8..9c3223e1a 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -1,19 +1,22 @@
#include <metal_stdlib>
#include <metal_simdgroup>
+#include "../common/defines.h"
#include "../common/dsl.h"
#include "../common/thread_context.h"
#include "../generated/ring.h"
#include "../generated/trie.h"
#include "mask.h"
-#define HEAD_BLOCK_SIZE 32
-#define SEQUENCE_BLOCK_SIZE_1 8
-#define SEQUENCE_BLOCK_SIZE_2 32
#define TOTAL_BLOCKS_COUNT 32
+#define SIMDGROUPS_PER_THREADGROUP 8
using namespace uzu::ring;
using namespace uzu::trie;
+// GQA-aware two-pass attention (pass 1).
+// Fixed 256-thread threadgroups (8 simdgroups) with flat query-head mapping.
+// Each simdgroup independently handles one query head and derives its KV head
+// via integer division. Simdgroups sharing a KV head benefit from L1 cache.
template <typename T, uint HEAD_DIM>
VARIANTS(T, float, half, bfloat)
VARIANTS(HEAD_DIM, 64, 128, 256)
@@ -42,82 +45,58 @@ PUBLIC KERNEL(AttentionTwoPass1)(
const bool is_causal SPECIALIZE,
const bool is_trie SPECIALIZE,
const bool is_sliding_window SPECIALIZE,
- threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE_1],
- threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE_1],
- threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_1 * HEAD_BLOCK_SIZE],
const ThreadContext thread_context,
- const uint head_idx GROUPS(num_heads),
+ const uint head_group_idx GROUPS(num_heads.div_ceil(SIMDGROUPS_PER_THREADGROUP)),
const uint q_seq_idx GROUPS(suffix_length),
const uint block_idx GROUPS(TOTAL_BLOCKS_COUNT),
- const uint tid THREADS(256)
+ const uint tid THREADS(SIMDGROUPS_PER_THREADGROUP * METAL_SIMD_SIZE)
) {
- constexpr bool query_transposed = false;
- constexpr uint value_dim = HEAD_DIM;
- constexpr uint qk_elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;
- constexpr uint value_elements_per_thread = value_dim / HEAD_BLOCK_SIZE;
- uint inner_k_stride = SEQUENCE_BLOCK_SIZE_1 * k_seq_stride;
- uint inner_v_stride = SEQUENCE_BLOCK_SIZE_1 * v_seq_stride;
+ constexpr uint elements_per_thread = HEAD_DIM / METAL_SIMD_SIZE;
typedef float U;
- thread U q[qk_elements_per_thread];
- thread U k[qk_elements_per_thread];
- thread U o[value_elements_per_thread];
+ // Flat query head mapping — fixed hardware shape, dynamic logical mapping
+ const uint head_idx = head_group_idx * SIMDGROUPS_PER_THREADGROUP +
+ thread_context.threadgroup_index;
+ if (head_idx >= num_heads)
+ return;
- const uint o_offset = q_seq_idx * num_heads + head_idx; // Our custom layout
- const uint q_offset =
- query_transposed
- ? num_heads * q_seq_idx + head_idx
- : head_idx * suffix_length + q_seq_idx; // Consistent with single-pass
const uint kv_head_idx = head_idx / gqa_factor;
+ const uint o_offset = q_seq_idx * num_heads + head_idx;
+ const uint q_offset = head_idx * suffix_length + q_seq_idx;
const uint prefix_length = sequence_length - suffix_length;
-
const uint suffix_position =
is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;
-
const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height
: suffix_position + q_seq_idx;
+ thread U q[elements_per_thread];
+ thread U o[elements_per_thread] = {0};
+
queries += q_offset * HEAD_DIM +
- thread_context.simdgroup_index * qk_elements_per_thread;
- keys +=
- kv_head_idx * k_head_stride +
- (block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index) *
- k_seq_stride +
- thread_context.simdgroup_index * qk_elements_per_thread;
- values +=
- kv_head_idx * v_head_stride +
- (block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index) *
- v_seq_stride +
- thread_context.simdgroup_index * value_elements_per_thread;
- out += o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim +
- thread_context.simdgroup_index * value_elements_per_thread;
+ thread_context.simdgroup_index * elements_per_thread;
+ keys += kv_head_idx * k_head_stride + block_idx * k_seq_stride +
+ thread_context.simdgroup_index * elements_per_thread;
+ values += kv_head_idx * v_head_stride + block_idx * v_seq_stride +
+ thread_context.simdgroup_index * elements_per_thread;
+ out += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM + block_idx * HEAD_DIM +
+ thread_context.simdgroup_index * elements_per_thread;
sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
maxs += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
- // Read the query and 0 the output accumulator
- for (uint i = 0; i < qk_elements_per_thread; i++) {
+ for (uint i = 0; i < elements_per_thread; i++) {
q[i] = static_cast<U>(scale) * queries[i];
}
- for (uint i = 0; i < value_elements_per_thread; i++) {
- o[i] = 0;
- }
-
U max_score = -1e9;
U sum_exp_score = 0;
- if (has_sinks && block_idx == 0 && thread_context.threadgroup_index == 0) {
- const uint num_q_heads = num_heads;
- int q_head_idx = head_idx % num_q_heads;
- max_score = static_cast<U>(sinks[q_head_idx]);
+ if (has_sinks && block_idx == 0) {
+ max_score = static_cast<U>(sinks[head_idx]);
sum_exp_score = 1;
}
- // For each key
- for (uint i =
- block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index;
- i < sequence_length;
- i += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1) {
+ // For each key position
+ for (uint i = block_idx; i < sequence_length; i += TOTAL_BLOCKS_COUNT) {
if (should_use_key(
ring_params,
trie,
@@ -132,81 +111,36 @@ PUBLIC KERNEL(AttentionTwoPass1)(
is_trie,
is_sliding_window
)) {
- // Read the key
- for (uint j = 0; j < qk_elements_per_thread; j++) {
- k[j] = keys[j];
- }
-
- // Compute the i-th score
+ // Compute score
U score = 0;
- for (uint j = 0; j < qk_elements_per_thread; j++) {
- score += q[j] * k[j];
+ for (uint j = 0; j < elements_per_thread; j++) {
+ score += q[j] * static_cast<U>(keys[j]);
}
score = simd_sum(score);
- // Update the accumulators
+ // Online softmax update
U new_max = max(max_score, score);
U factor = fast::exp(max_score - new_max);
U exp_score = fast::exp(score - new_max);
-
max_score = new_max;
sum_exp_score = sum_exp_score * factor + exp_score;
- // Update the output accumulator
- for (uint j = 0; j < value_elements_per_thread; j++) {
- o[j] = o[j] * factor + exp_score * values[j];
+ for (uint j = 0; j < elements_per_thread; j++) {
+ o[j] = o[j] * factor + exp_score * static_cast<U>(values[j]);
}
}
- // Move the pointers to the next kv
- keys += TOTAL_BLOCKS_COUNT * inner_k_stride;
- values += TOTAL_BLOCKS_COUNT * inner_v_stride;
+ keys += TOTAL_BLOCKS_COUNT * k_seq_stride;
+ values += TOTAL_BLOCKS_COUNT * v_seq_stride;
}
- // Each thread has a partial part of the output so we need to combine them.
+ // Write partial output
if (thread_context.simdgroup_index == 0) {
- shared_max_scores[thread_context.threadgroup_index] = max_score;
- shared_sum_exp_scores[thread_context.threadgroup_index] = sum_exp_score;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- max_score = (thread_context.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)
- ? shared_max_scores[thread_context.simdgroup_index]
- : -1e9;
- U new_max = simd_max(max_score);
- U factor = fast::exp(max_score - new_max);
- sum_exp_score = (thread_context.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)
- ? shared_sum_exp_scores[thread_context.simdgroup_index]
- : 0;
- sum_exp_score = simd_sum(sum_exp_score * factor);
-
- // Write the sum and new max
- if (thread_context.threadgroup_index == 0) {
sums[0] = sum_exp_score;
- maxs[0] = new_max;
+ maxs[0] = max_score;
}
-
- // Now we need to aggregate all the outputs
- for (uint i = 0; i < value_elements_per_thread; i++) {
- shared_outputs
- [thread_context.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 +
- thread_context.threadgroup_index] =
- o[i] *
- fast::exp(
- shared_max_scores[thread_context.threadgroup_index] - new_max
- );
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // And write the output
- if (thread_context.threadgroup_index == 0) {
- U output = shared_outputs
- [thread_context.simdgroup_index * SEQUENCE_BLOCK_SIZE_1];
- for (uint j = 1; j < SEQUENCE_BLOCK_SIZE_1; j++) {
- output += shared_outputs
- [thread_context.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 + j];
- }
- out[i] = static_cast<T>(output);
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (uint i = 0; i < elements_per_thread; i++) {
+ out[i] = o[i];
}
}
@@ -220,58 +154,63 @@ PUBLIC KERNEL(AttentionTwoPass2)(
device T* out,
const constant uint& num_heads,
const constant uint& suffix_length,
- threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_2 * HEAD_BLOCK_SIZE],
const ThreadContext thread_context,
const uint head_idx GROUPS(num_heads),
const uint q_seq_idx GROUPS(suffix_length),
const uint tid THREADS(1024)
) {
- constexpr uint elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;
+ constexpr uint elements_per_thread = HEAD_DIM / METAL_SIMD_SIZE;
typedef float U;
- thread U o[elements_per_thread];
- const uint o_offset = q_seq_idx * num_heads + head_idx; // Our custom layout
+ const uint o_offset = q_seq_idx * num_heads + head_idx;
- partials += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +
- thread_context.threadgroup_index * HEAD_DIM +
- thread_context.simdgroup_index * elements_per_thread;
sums += o_offset * TOTAL_BLOCKS_COUNT;
maxs += o_offset * TOTAL_BLOCKS_COUNT;
- out +=
- o_offset * HEAD_DIM + thread_context.threadgroup_index *
- elements_per_thread; // Our custom output layout
- // First everybody reads the max and sum_exp
- U max_score = maxs[thread_context.simdgroup_index];
+ // Find global max across all blocks (each lane handles strided blocks)
+ U max_score = -1e9;
+ for (uint b = thread_context.simdgroup_index; b < TOTAL_BLOCKS_COUNT;
+ b += METAL_SIMD_SIZE) {
+ max_score = max(max_score, maxs[b]);
+ }
U new_max = simd_max(max_score);
- U factor = fast::exp(max_score - new_max);
- U sum_exp_score = simd_sum(sums[thread_context.simdgroup_index] * factor);
- // Now read the block into registers and then use shared memory to transpose
- // it
- for (uint i = 0; i < elements_per_thread; i++) {
- o[i] = partials[i];
+ // Find global sum across all blocks
+ U sum_exp = 0;
+ for (uint b = thread_context.simdgroup_index; b < TOTAL_BLOCKS_COUNT;
+ b += METAL_SIMD_SIZE) {
+ sum_exp += sums[b] * fast::exp(maxs[b] - new_max);
}
- for (uint i = 0; i < elements_per_thread; i++) {
- shared_outputs
- [thread_context.simdgroup_index * HEAD_BLOCK_SIZE +
- thread_context.threadgroup_index] = o[i];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- o[i] = simd_sum(
- shared_outputs
- [thread_context.threadgroup_index * HEAD_BLOCK_SIZE +
- thread_context.simdgroup_index] *
- factor
- ) /
- sum_exp_score;
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ U sum_exp_score = simd_sum(sum_exp);
+
+ // Combine partial outputs — each lane reads a different block,
+ // simd_sum reduces across blocks directly (no shared memory needed)
+ thread U o[elements_per_thread] = {0};
+
+ for (uint chunk = 0; chunk < TOTAL_BLOCKS_COUNT; chunk += METAL_SIMD_SIZE) {
+ uint block_idx = chunk + thread_context.simdgroup_index;
+ U factor = (block_idx < TOTAL_BLOCKS_COUNT)
+ ? fast::exp(maxs[block_idx] - new_max)
+ : 0;
+
+ const device float* block_partials =
+ partials + o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +
+ block_idx * HEAD_DIM +
+ thread_context.threadgroup_index * elements_per_thread;
+
+ for (uint i = 0; i < elements_per_thread; i++) {
+ U val = (block_idx < TOTAL_BLOCKS_COUNT) ? block_partials[i] * factor : 0;
+ o[i] += simd_sum(val);
+ }
}
- // And write the output
+ // Write the output
if (thread_context.simdgroup_index == 0) {
+ out += o_offset * HEAD_DIM +
+ thread_context.threadgroup_index * elements_per_thread;
for (uint i = 0; i < elements_per_thread; i++) {
- out[i] = static_cast<T>(o[i]);
+ out[i] = static_cast<T>(o[i] / sum_exp_score);
}
}
}
diff --git a/crates/uzu/tests/unit/encodable_block/attention_test.rs b/crates/uzu/tests/unit/encodable_block/attention_test.rs
index 82bf895b3..3f08b7e08 100644
--- a/crates/uzu/tests/unit/encodable_block/attention_test.rs
+++ b/crates/uzu/tests/unit/encodable_block/attention_test.rs
@@ -968,13 +968,13 @@ fn perf_two_pass_attention() {
false,
false,
)
- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");
+ .expect("Failed to create AttentionTwoPass1Kernel");
let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(
&context,
DataType::F32,
head_dim as u32,
)
- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");
+ .expect("Failed to create AttentionTwoPass2Kernel");
// ---- Create buffers ----
// For realistic inference, we only process queries for the suffix (new tokens)