A high-performance inference engine for AI models
Support Attention head_dim=512 (#412)
CCYeh5ff6f3a180e0
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul.rs b/crates/backend-uzu/src/backends/common/kernel/matmul.rsindex e8ef8c962..4986dbe47 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul.rs@@ -21,13 +21,16 @@ pub enum MatmulArgumentC<'a, B: Backend> {Bias(&'a Allocation<B>),}-// D = ab_scale * (A @ B.T) + C+// D = ab_scale * (A @ op(B)) + C, where op(B) = B^T when b_transpose else B.pub struct MatmulArguments<'a, B: Backend> {/// A: [M, K]pub a: &'a Allocation<B>,pub a_offset: usize,- /// B: [N, K]+ /// B: [N, K] when b_transpose, else [K, N]pub b: &'a Allocation<B>,+ pub b_offset: usize,+ pub b_leading_dimension: Option<u32>,+ pub b_transpose: bool,/// AB scale: also known as alphapub ab_scale: f32,/// C: behavior depends on enum variantdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_fallback.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_fallback.rsnew file mode 100644index 000000000..bbb825465--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_fallback.rs@@ -0,0 +1,94 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::{+ ArrayElement,+ backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},+};++#[kernel(AttentionFallbackScatterScores)]+#[variants(T, f32, f16, bf16)]+pub fn attention_fallback_scatter_scores<T: ArrayElement + Float>(+ group_scores: *const T,+ scores: *mut T,+ #[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,+ #[optional(is_trie)] trie: Option<*const TrieNode>,+ #[optional(is_sliding_window)] sliding_window_size: Option<u32>,+ group_index: u32,+ gqa_factor: u32,+ sequence_length: u32,+ suffix_length: u32,+ #[allow(unused)] total_elements: u32,+ #[specialize] is_kv_cache_ring: bool,+ #[specialize] is_causal: bool,+ #[specialize] is_trie: bool,+ #[specialize] is_sliding_window: bool,+) {+ let prefix_length = sequence_length - suffix_length;+ let suffix_position = ring_params.map(|params| params.ring_length).unwrap_or(prefix_length);++ for head_in_group in 0..gqa_factor {+ let head_index = group_index * gqa_factor + head_in_group;+ for query_index in 0..suffix_length {+ let query_position = if is_trie {+ let trie_node = unsafe { &*trie.unwrap().add(query_index as usize) };+ suffix_position + trie_node.height+ } else {+ suffix_position + query_index+ };++ for sequence_index in 0..sequence_length {+ let source_index =+ ((head_in_group * suffix_length + query_index) * sequence_length + sequence_index) as usize;+ let destination_index =+ ((head_index * suffix_length + query_index) * sequence_length + sequence_index) as usize;+ let use_key = should_use_key(+ ring_params,+ trie,+ sliding_window_size,+ query_index,+ prefix_length,+ suffix_position,+ query_position,+ sequence_index,+ is_causal,+ );+ unsafe {+ *scores.add(destination_index) = if use_key {+ *group_scores.add(source_index)+ } else {+ T::from(f32::NEG_INFINITY).unwrap()+ };+ }+ }+ }+ }+}++#[kernel(AttentionFallbackScatterValues)]+#[variants(T, f32, f16, bf16)]+pub fn attention_fallback_scatter_values<T: ArrayElement + Float>(+ group_output: *const T,+ out: *mut T,+ group_index: u32,+ gqa_factor: u32,+ suffix_length: u32,+ num_heads: u32,+ head_dim: u32,+ #[allow(unused)] total_elements: u32,+) {+ for head_in_group in 0..gqa_factor as usize {+ for query_index in 0..suffix_length as usize {+ for dim_index in 0..head_dim as usize {+ let source_index =+ (head_in_group * suffix_length as usize + query_index) * head_dim as usize + dim_index;+ let head_index = group_index as usize * gqa_factor as usize + head_in_group;+ let destination_index = (query_index * num_heads as usize + head_index) * head_dim as usize + dim_index;+ unsafe {+ *out.add(destination_index) = *group_output.add(source_index);+ }+ }+ }+ }+}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rsindex 786a4a744..90282d1e9 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs@@ -9,7 +9,7 @@ use crate::{#[kernel(AttentionSinglePass)]#[variants(T, f32, f16, bf16)]-#[variants(HEAD_DIM, 64, 128, 256)]+#[variants(HEAD_DIM, 64, 128, 256, 512)]pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(queries: *const T,keys: *const T,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rsindex 5bbfb3148..da4a7695f 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs@@ -11,7 +11,7 @@ const TOTAL_BLOCKS_COUNT: u32 = 32;#[kernel(AttentionTwoPass1)]#[variants(T, f32, f16, bf16)]-#[variants(HEAD_DIM, 64, 128, 256)]+#[variants(HEAD_DIM, 64, 128, 256, 512)]pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(queries: *const T,keys: *const T,@@ -139,7 +139,7 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(#[kernel(AttentionTwoPass2)]#[variants(T, f32, f16, bf16)]-#[variants(HEAD_DIM, 64, 128, 256)]+#[variants(HEAD_DIM, 64, 128, 256, 512)]pub fn attention_two_pass2<T: ArrayElement + Float, const HEAD_DIM: u32>(partials: *const f32,sums: *const f32,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rsindex c862aeaf0..d84c4a40d 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs@@ -1,3 +1,4 @@+pub mod attention_fallback;pub mod attention_gemm;pub mod attention_single_pass;pub mod attention_two_pass;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rsindex eb9fe033b..cbafef074 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs@@ -17,6 +17,7 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(leading_dimension_a: usize,leading_dimension_b: usize,leading_dimension_d: usize,+ b_transpose: bool,is_accumulate: bool,) {unsafe {@@ -24,7 +25,11 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(let mut acc: f32 = 0.0;for i in 0..k {let a_val = *a.0.add(row * leading_dimension_a + i);- let b_val = *b.0.add(col * leading_dimension_b + i);+ let b_val = if b_transpose {+ *b.0.add(col * leading_dimension_b + i)+ } else {+ *b.0.add(i * leading_dimension_b + col)+ };acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();}let d_element = d.0.add(row * leading_dimension_d + col);@@ -48,6 +53,7 @@ pub fn matmul_gemm_impl<T: ArrayElement + Float>(leading_dimension_a: usize,leading_dimension_b: usize,leading_dimension_d: usize,+ b_transpose: bool,is_accumulate: bool,) {let num_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);@@ -83,6 +89,7 @@ pub fn matmul_gemm_impl<T: ArrayElement + Float>(leading_dimension_a,leading_dimension_b,leading_dimension_d,+ b_transpose,is_accumulate,);}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rsindex fe73bacc4..3cc122c5d 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs@@ -40,6 +40,9 @@ impl MatmulKernel for MatmulCpuKernel {a,a_offset,b,+ b_offset,+ b_leading_dimension,+ b_transpose,ab_scale,c,d,@@ -52,14 +55,18 @@ impl MatmulKernel for MatmulCpuKernel {let n = output_dim as usize;let k = input_dim as usize;let lda = k;- let ldb = k;+ let ldb = b_leading_dimension.map(|n| n as usize).unwrap_or(if b_transpose {+ k+ } else {+ n+ });let ldd = n;let data_type = self.data_type;let a_buffer_range = a.as_buffer_range_ref();let b_buffer_range = b.as_buffer_range_ref();let d_buffer_range = d.as_buffer_range_mut();let a_offset = a_buffer_range.range().start + a_offset;- let b_offset = b_buffer_range.range().start;+ let b_offset = b_buffer_range.range().start + b_offset;let d_offset = d_buffer_range.range().start;let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_offset));@@ -95,6 +102,7 @@ impl MatmulKernel for MatmulCpuKernel {lda,ldb,ldd,+ b_transpose,is_accumulate,);if let Some(bias) = bias_ptr {@@ -113,6 +121,7 @@ impl MatmulKernel for MatmulCpuKernel {lda,ldb,ldd,+ b_transpose,is_accumulate,);if let Some(bias) = bias_ptr {@@ -131,6 +140,7 @@ impl MatmulKernel for MatmulCpuKernel {lda,ldb,ldd,+ b_transpose,is_accumulate,);if let Some(bias) = bias_ptr {diff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rsindex 1e14e6cd4..421ec7c18 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs@@ -17,6 +17,7 @@ mod rms_norm;mod rope;mod sampling;mod short_conv;+mod softmax;mod ssm;mod tensor_add_bias;mod tensor_add_scale;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/softmax/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/softmax/mod.rsnew file mode 100644index 000000000..c85937510--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/softmax/mod.rs@@ -0,0 +1 @@+pub mod softmax;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rs b/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rsnew file mode 100644index 000000000..3399ddf5e--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rs@@ -0,0 +1,57 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(Softmax)]+#[variants(T, f32, f16, bf16)]+pub fn softmax<T: ArrayElement + Float>(+ values: *mut T,+ #[optional(has_sinks)] sinks: Option<*const f32>,+ row_length: u32,+ outer_dim: u32,+ batch_dim: u32,+ #[specialize] has_sinks: bool,+) {+ for outer_index in 0..outer_dim {+ for batch_index in 0..batch_dim {+ let row_offset = ((outer_index * batch_dim + batch_index) * row_length) as usize;+ let sink = if has_sinks {+ unsafe { *sinks.unwrap().add(outer_index as usize) }+ } else {+ f32::NEG_INFINITY+ };++ let mut max_value = if has_sinks {+ sink+ } else {+ f32::NEG_INFINITY+ };+ for i in 0..row_length as usize {+ max_value = max_value.max(unsafe { *values.add(row_offset + i) }.to_f32().unwrap());+ }++ let mut sum_exp = if has_sinks {+ (sink - max_value).exp()+ } else {+ 0.0+ };+ for i in 0..row_length as usize {+ let p = (unsafe { *values.add(row_offset + i) }.to_f32().unwrap() - max_value).exp();+ sum_exp += p;+ unsafe {+ *values.add(row_offset + i) = T::from(p).unwrap();+ }+ }++ let inv = 1.0 / sum_exp;+ for i in 0..row_length as usize {+ unsafe {+ let p = *values.add(row_offset + i);+ *values.add(row_offset + i) = T::from(p.to_f32().unwrap() * inv).unwrap();+ }+ }+ }+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metalnew file mode 100644index 000000000..cccd17c89--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metal@@ -0,0 +1,86 @@+#include <metal_stdlib>+#include <metal_simdgroup>+#include "../common/defines.h"+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../generated/ring.h"+#include "../generated/trie.h"+#include "mask.h"++using namespace uzu::ring;+using namespace uzu::trie;++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AttentionFallbackScatterScores)(+ const device T* group_scores,+ device T* scores,+ const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),+ const device TrieNode* trie OPTIONAL(is_trie),+ const constant uint& sliding_window_size OPTIONAL(is_sliding_window),+ const constant uint& group_index,+ const constant uint& gqa_factor,+ const constant uint& sequence_length,+ const constant uint& suffix_length,+ const constant uint& total_elements,+ const bool is_kv_cache_ring SPECIALIZE,+ const bool is_causal SPECIALIZE,+ const bool is_trie SPECIALIZE,+ const bool is_sliding_window SPECIALIZE,+ const uint group_score_index AXIS(total_elements, 256)+) {+ const uint sequence_index = group_score_index % sequence_length;+ const uint row_index = group_score_index / sequence_length;+ const uint query_index = row_index % suffix_length;+ const uint head_in_group = row_index / suffix_length;+ const uint head_index = group_index * gqa_factor + head_in_group;++ const uint prefix_length = sequence_length - suffix_length;+ const uint suffix_position =+ is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;+ const uint query_position = is_trie+ ? suffix_position + trie[query_index].height+ : suffix_position + query_index;++ const bool use_key = should_use_key(+ ring_params,+ trie,+ sliding_window_size,+ query_index,+ prefix_length,+ suffix_position,+ query_position,+ sequence_index,+ is_kv_cache_ring,+ is_causal,+ is_trie,+ is_sliding_window+ );++ scores+ [(head_index * suffix_length + query_index) * sequence_length ++ sequence_index] =+ use_key ? group_scores[group_score_index] : T(-INFINITY);+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AttentionFallbackScatterValues)(+ const device T* group_output,+ device T* out,+ const constant uint& group_index,+ const constant uint& gqa_factor,+ const constant uint& suffix_length,+ const constant uint& num_heads,+ const constant uint& head_dim,+ const constant uint& total_elements,+ const uint group_output_index AXIS(total_elements, 256)+) {+ const uint dim_index = group_output_index % head_dim;+ const uint row_index = group_output_index / head_dim;+ const uint query_index = row_index % suffix_length;+ const uint head_in_group = row_index / suffix_length;+ const uint head_index = group_index * gqa_factor + head_in_group;+ out[(query_index * num_heads + head_index) * head_dim + dim_index] =+ group_output[group_output_index];+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metalindex 5f75bbfde..368cc31e6 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -14,7 +14,7 @@ using namespace uzu::trie;template <typename T, uint HEAD_DIM>VARIANTS(T, float, half, bfloat)-VARIANTS(HEAD_DIM, 64, 128, 256)+VARIANTS(HEAD_DIM, 64, 128, 256, 512)PUBLIC KERNEL(AttentionSinglePass)(const device T* queries,const device T* keys,diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metalindex 9c3223e1a..3c36971e8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -19,7 +19,7 @@ using namespace uzu::trie;// via integer division. Simdgroups sharing a KV head benefit from L1 cache.template <typename T, uint HEAD_DIM>VARIANTS(T, float, half, bfloat)-VARIANTS(HEAD_DIM, 64, 128, 256)+VARIANTS(HEAD_DIM, 64, 128, 256, 512)PUBLIC KERNEL(AttentionTwoPass1)(const device T* queries,const device T* keys,@@ -146,7 +146,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(template <typename T, uint HEAD_DIM>VARIANTS(T, float, half, bfloat)-VARIANTS(HEAD_DIM, 64, 128, 256)+VARIANTS(HEAD_DIM, 64, 128, 256, 512)PUBLIC KERNEL(AttentionTwoPass2)(const device float* partials,const device float* sums,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metalindex f790479f0..02ea47e3a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal@@ -8,7 +8,7 @@ using namespace uzu::matmul;#define GEMM_MAX_THREADGROUP_A 2560#define GEMM_MAX_THREADGROUP_B 1536-template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint BLOCK_DEPTH, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN, bool MN_ALIGNED, bool K_ALIGNED>+template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint BLOCK_DEPTH, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN, bool MN_ALIGNED, bool K_ALIGNED, bool TRANSPOSE_B>VARIANTS(T, float, half, bfloat)VARIANTS(BLOCK_ROWS, 32, 64)VARIANTS(BLOCK_COLS, 32, 64)@@ -17,6 +17,7 @@ VARIANTS(SIMDGROUPS_PER_ROW, 1, 2, 4)VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2)VARIANTS(MN_ALIGNED, false, true)VARIANTS(K_ALIGNED, false, true)+VARIANTS(TRANSPOSE_B, false, true)CONSTRAINT(max(BLOCK_ROWS, BLOCK_COLS) <= 32 * SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN)KERNEL(MatmulGemm)(const device T* a,@@ -45,7 +46,7 @@ KERNEL(MatmulGemm)(SIMDGROUPS_PER_ROW,SIMDGROUPS_PER_COLUMN,false, // transpose_a- true, // transpose_b+ TRANSPOSE_B,MN_ALIGNED,K_ALIGNED,float>::diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rsindex 3350dfe46..96a346750 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs@@ -15,6 +15,7 @@ pub struct GemmSpecialization {pub simdgroups_per_column: u32,pub align_mn: bool,pub align_k: bool,+ pub transpose_b: bool,pub is_accumulate: bool,}@@ -30,6 +31,7 @@ impl GemmSpecialization {simdgroups_per_column: 2,align_mn: false,align_k: true,+ transpose_b: true,is_accumulate: false,},Self {@@ -40,6 +42,7 @@ impl GemmSpecialization {simdgroups_per_column: 2,align_mn: true,align_k: true,+ transpose_b: true,is_accumulate: false,},Self {@@ -50,6 +53,7 @@ impl GemmSpecialization {simdgroups_per_column: 2,align_mn: false,align_k: true,+ transpose_b: true,is_accumulate: false,},Self {@@ -60,6 +64,7 @@ impl GemmSpecialization {simdgroups_per_column: 2,align_mn: true,align_k: true,+ transpose_b: true,is_accumulate: false,},Self {@@ -70,6 +75,7 @@ impl GemmSpecialization {simdgroups_per_column: 2,align_mn: true,align_k: true,+ transpose_b: true,is_accumulate: false,},],@@ -82,6 +88,7 @@ impl GemmSpecialization {simdgroups_per_column: 2,align_mn: true,align_k: true,+ transpose_b: true,is_accumulate: false,},Self {@@ -92,6 +99,7 @@ impl GemmSpecialization {simdgroups_per_column: 2,align_mn: false,align_k: true,+ transpose_b: true,is_accumulate: false,},],@@ -104,6 +112,7 @@ impl GemmSpecialization {simdgroups_per_column: 2,align_mn: false,align_k: true,+ transpose_b: true,is_accumulate: false,},Self {@@ -114,6 +123,7 @@ impl GemmSpecialization {simdgroups_per_column: 2,align_mn: true,align_k: true,+ transpose_b: true,is_accumulate: false,},],@@ -127,6 +137,7 @@ impl GemmSpecialization {batch_dim: u32,input_dim: u32,output_dim: u32,+ transpose_b: bool,c: &MatmulArgumentC<Metal>,) -> Self {let overall_work_elements = batch_dim * output_dim;@@ -158,6 +169,7 @@ impl GemmSpecialization {simdgroups_per_column,align_mn: (batch_dim % block_rows) == 0 && (output_dim % block_cols) == 0,align_k: (input_dim % block_depth) == 0,+ transpose_b,is_accumulate: matches!(c, MatmulArgumentC::Accumulate),}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex bdb779ca4..1939c7c37 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -99,6 +99,7 @@ impl MatmulMetalKernel {specialization.simdgroups_per_column as u32,specialization.align_mn,specialization.align_k,+ specialization.transpose_b,specialization.is_accumulate,).map_err(MatmulError::BackendError)?;@@ -143,6 +144,9 @@ impl MatmulMetalKernel {a,a_offset,b,+ b_offset,+ b_leading_dimension,+ b_transpose,ab_scale,c,d,@@ -150,6 +154,7 @@ impl MatmulMetalKernel {input_dim,output_dim,} = arguments;+ assert!(b_transpose, "encode_gemv does not yet support b_transpose=false");let (is_accumulate, output_bias) = match c {MatmulArgumentC::Accumulate => (true, None),@@ -161,13 +166,13 @@ impl MatmulMetalKernel {gemv::GemvSpecialization::select(input_dim, output_dim, is_accumulate, output_bias.is_some());self.get_or_create_gemv(encoder.context(), specialization)?.encode(- b,+ (b, b_offset),(a, a_offset),output_bias,d,input_dim,output_dim,- input_dim,+ b_leading_dimension.unwrap_or(input_dim),ab_scale,batch_dim,specialization.output_rows_per_threadgroup(),@@ -186,6 +191,9 @@ impl MatmulMetalKernel {a,a_offset,b,+ b_offset,+ b_leading_dimension,+ b_transpose,ab_scale,c,d,@@ -194,18 +202,30 @@ impl MatmulMetalKernel {output_dim,} = arguments;- let specialization =- gemm::GemmSpecialization::select(encoder.context(), self.data_type, batch_dim, input_dim, output_dim, &c);+ let specialization = gemm::GemmSpecialization::select(+ encoder.context(),+ self.data_type,+ batch_dim,+ input_dim,+ output_dim,+ b_transpose,+ &c,+ );let threadgroups_per_row = output_dim.div_ceil(specialization.block_cols);let threadgroups_per_column = batch_dim.div_ceil(specialization.block_rows);+ let default_ldb = if b_transpose {+ input_dim+ } else {+ output_dim+ };let params = GemmParams {M: batch_dim,N: output_dim,K: input_dim,leading_dimension_a: input_dim,- leading_dimension_b: input_dim,+ leading_dimension_b: b_leading_dimension.unwrap_or(default_ldb),leading_dimension_d: output_dim,threadgroups_per_row,threadgroups_per_column,@@ -218,7 +238,7 @@ impl MatmulMetalKernel {kernel.encode((a, a_offset),- b,+ (b, b_offset),&mut *d,std::slice::from_ref(¶ms),threadgroups_per_row,@@ -243,6 +263,9 @@ impl MatmulMetalKernel {a,a_offset,b,+ b_offset,+ b_leading_dimension,+ b_transpose,ab_scale,c,d,@@ -250,6 +273,7 @@ impl MatmulMetalKernel {input_dim,output_dim,} = arguments;+ assert!(b_transpose, "encode_gemm_mpp does not yet support b_transpose=false");let is_accumulate = matches!(c, MatmulArgumentC::Accumulate);let specialization =gemm_mpp::GemmMppSpecialization::select(batch_dim, output_dim, input_dim, is_accumulate, ab_scale != 1.0);@@ -271,7 +295,7 @@ impl MatmulMetalKernel {N: output_dim,K: input_dim,leading_dimension_a: input_dim,- leading_dimension_b: input_dim,+ leading_dimension_b: b_leading_dimension.unwrap_or(input_dim),leading_dimension_d: output_dim,threadgroups_per_row,threadgroups_per_column,@@ -289,7 +313,7 @@ impl MatmulMetalKernel {let kernel = self.get_or_create_gemm_mpp(encoder.context(), specialization)?;kernel.encode((a, a_offset),- b,+ (b, b_offset),&mut *d,std::slice::from_ref(¶ms),group_count_x,@@ -372,7 +396,9 @@ impl MatmulKernel for MatmulMetalKernel {arguments: MatmulArguments<Metal>,encoder: &mut Encoder<Metal>,) {- if arguments.batch_dim <= max_gemv_batch_threshold() {+ if !arguments.b_transpose {+ self.encode_gemm(encoder, arguments).expect("Failed to encode GEMM kernel");+ } else if arguments.batch_dim <= max_gemv_batch_threshold() {self.encode_gemv(encoder, arguments).expect("Failed to encode GEMV kernel");} else if self.is_mpp_eligible(encoder.context()) {self.encode_gemm_mpp(encoder, arguments).expect("Failed to encode GEMM MPP kernel");diff --git a/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal b/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metalnew file mode 100644index 000000000..d53fdd198--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal@@ -0,0 +1,98 @@+#include <metal_stdlib>++#include "../common/defines.h"+#include "../common/dsl.h"++// Online row-wise softmax (MLX-style: simd intrinsics + 2-pass online+// algorithm). `has_sinks` seeds thread 0 with (sinks[outer_index], 1) as a+// virtual element.++#define SOFTMAX_THREADS 256+// Number of consecutive elements each thread loads per iteration. Vectorizing+// the per-thread loop this way improves memory coalescing on Apple Silicon.+#define SOFTMAX_ELEMENTS_PER_THREAD 4++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(Softmax)(+ device T* values,+ const device float* sinks OPTIONAL(has_sinks),+ const constant uint& row_length,+ const constant uint& outer_dim,+ const constant uint& batch_dim,+ threadgroup float shared_max[METAL_SIMD_SIZE],+ threadgroup float shared_norm[METAL_SIMD_SIZE],+ const bool has_sinks SPECIALIZE,+ const uint outer_index GROUPS(outer_dim),+ const uint batch_index GROUPS(batch_dim),+ const uint tid THREADS(SOFTMAX_THREADS)+) {+ const uint lane = tid & (METAL_SIMD_SIZE - 1);+ const uint simdgroup_id = tid / METAL_SIMD_SIZE;+ const uint stride = SOFTMAX_THREADS * SOFTMAX_ELEMENTS_PER_THREAD;+ device T* row = values + (outer_index * batch_dim + batch_index) * row_length;++ float maxval = (has_sinks && tid == 0) ? sinks[outer_index] : -FLT_MAX;+ float norm = (has_sinks && tid == 0) ? 1.0f : 0.0f;++ if (simdgroup_id == 0) {+ shared_max[lane] = -FLT_MAX;+ shared_norm[lane] = 0.0f;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Pass 1: online (max, normalizer) accumulation over the row.+ for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length;+ base += stride) {+ float vals[SOFTMAX_ELEMENTS_PER_THREAD];+ for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {+ vals[i] = (base + i < row_length) ? float(row[base + i]) : -FLT_MAX;+ }+ float prev_max = maxval;+ for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {+ maxval = max(maxval, vals[i]);+ }+ norm *= fast::exp(prev_max - maxval);+ for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {+ norm += fast::exp(vals[i] - maxval);+ }+ }++ // Combine across the simdgroup, then across simdgroups via shared memory.+ float prev_max = maxval;+ maxval = simd_max(maxval);+ norm *= fast::exp(prev_max - maxval);+ norm = simd_sum(norm);++ if (lane == 0) {+ shared_max[simdgroup_id] = maxval;+ shared_norm[simdgroup_id] = norm;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (simdgroup_id == 0) {+ float simdgroup_max = shared_max[lane];+ float simdgroup_norm = shared_norm[lane];+ float global_max = simd_max(simdgroup_max);+ simdgroup_norm *= fast::exp(simdgroup_max - global_max);+ float global_norm = simd_sum(simdgroup_norm);+ if (lane == 0) {+ shared_max[0] = global_max;+ shared_norm[0] = global_norm;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ const float final_max = shared_max[0];+ const float inv_norm = 1.0f / shared_norm[0];++ // Pass 2: normalize.+ for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length;+ base += stride) {+ for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {+ if (base + i < row_length) {+ row[base + i] =+ T(fast::exp(float(row[base + i]) - final_max) * inv_norm);+ }+ }+ }+}diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rsindex c6906f248..b9152eff9 100644--- a/crates/backend-uzu/src/encodable_block/attention.rs+++ b/crates/backend-uzu/src/encodable_block/attention.rs@@ -1,6 +1,6 @@//! Attention kernel encodable.-use std::collections::HashMap;+use std::{cell::RefCell, collections::HashMap};use itertools::iproduct;@@ -11,9 +11,11 @@ use crate::{Allocation, Backend, Encoder, Kernels,gpu_types::ring::RingParams,kernel::{- AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,- SigmoidGateKernel,+ AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, AttentionSinglePassKernel,+ AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel, ManualKernels,+ SigmoidGateKernel, SoftmaxKernel,attention::{AttentionGemmArguments, AttentionGemmBlock},+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},},},config::AttentionConfig,@@ -40,6 +42,11 @@ pub struct Attention<B: Backend> {single_pass_kernels: HashMap<KernelKey, <B::Kernels as Kernels>::AttentionSinglePassKernel>,two_pass_1_kernels: HashMap<KernelKey, <B::Kernels as Kernels>::AttentionTwoPass1Kernel>,two_pass_2_kernels: HashMap<u32, <B::Kernels as Kernels>::AttentionTwoPass2Kernel>,+ softmax_kernel: <B::Kernels as Kernels>::SoftmaxKernel,+ softmax_sinks_kernel: Option<<B::Kernels as Kernels>::SoftmaxKernel>,+ fallback_scatter_scores_kernels: HashMap<bool, <B::Kernels as Kernels>::AttentionFallbackScatterScoresKernel>,+ fallback_scatter_values_kernel: <B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel,+ matmul_kernel: Option<RefCell<<B::Kernels as ManualKernels>::MatmulKernel>>,update_kv_cache_kernel: <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel,update_kv_cache_inplace_kernel: <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel,gate_kernel: Option<<B::Kernels as Kernels>::SigmoidGateKernel>,@@ -66,8 +73,11 @@ impl<B: Backend> Attention<B> {let mut single_pass_kernels = HashMap::new();let mut two_pass_1_kernels = HashMap::new();let mut two_pass_2_kernels = HashMap::new();+ let mut fallback_scatter_scores_kernels = HashMap::new();- for (head_dim, is_trie, is_kv_cache_ring) in iproduct!([64u32, 128u32, 256u32], [false, true], [false, true]) {+ for (head_dim, is_trie, is_kv_cache_ring) in+ iproduct!([64u32, 128u32, 256u32, 512u32], [false, true], [false, true])+ {let key = KernelKey {head_dim,is_trie,@@ -102,6 +112,28 @@ impl<B: Backend> Attention<B> {two_pass_2_kernels.insert(head_dim, tp2_kernel);}+ for is_kv_cache_ring in [false, true] {+ let scatter = <B::Kernels as Kernels>::AttentionFallbackScatterScoresKernel::new(+ context,+ data_type,+ is_kv_cache_ring,+ config.is_causal,+ false,+ config.sliding_window_size.is_some(),+ )?;+ fallback_scatter_scores_kernels.insert(is_kv_cache_ring, scatter);+ }++ let softmax_kernel = <B::Kernels as Kernels>::SoftmaxKernel::new(context, data_type, false)?;+ let softmax_sinks_kernel = if config.has_sinks {+ Some(<B::Kernels as Kernels>::SoftmaxKernel::new(context, data_type, true)?)+ } else {+ None+ };+ let fallback_scatter_values_kernel =+ <B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(context, data_type)?;+ let matmul_kernel =+ <<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).ok().map(RefCell::new);let update_kv_cache_kernel =<B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, false)?;let update_kv_cache_inplace_kernel =@@ -117,6 +149,11 @@ impl<B: Backend> Attention<B> {single_pass_kernels,two_pass_1_kernels,two_pass_2_kernels,+ softmax_kernel,+ softmax_sinks_kernel,+ fallback_scatter_scores_kernels,+ fallback_scatter_values_kernel,+ matmul_kernel,update_kv_cache_kernel,update_kv_cache_inplace_kernel,gate_kernel,@@ -137,8 +174,12 @@ impl<B: Backend> Attention<B> {is_trie: bool,is_kv_cache_ring: bool,) -> KernelVariant {- let use_gemm = gemm_enabled && suffix_length > 8 && matches!(head_dim, 64 | 128 | 256);- if use_gemm {+ // head_dim=512 doesn't fit the fused GEMM tile in threadgroup memory.+ if head_dim == 512 && suffix_length > 8 && !is_trie {+ return KernelVariant::Fallback;+ }++ if gemm_enabled && suffix_length > 8 && matches!(head_dim, 64 | 128 | 256) {return KernelVariant::Gemm;}@@ -365,11 +406,31 @@ impl<B: Backend> Attention<B> {};self.gemm_block.encode(encoder, args)?;},+ KernelVariant::Fallback => {+ self.encode_fallback(+ kernel_key.is_kv_cache_ring,+ queries,+ keys,+ values,+ &mut attention_output,+ sinks_allocation,+ gqa_factor,+ sequence_length,+ k_head_stride,+ k_seq_stride,+ v_head_stride,+ v_seq_stride,+ ring_params,+ scale,+ num_heads,+ num_groups,+ suffix_length,+ head_dim,+ encoder,+ )?;+ },KernelVariant::SinglePass => {- let kernel = self- .single_pass_kernels- .get(kernel_key)- .unwrap_or_else(|| panic!("Can not find AttentionSinglePassKernel for key {:?}", kernel_key));+ let kernel = self.single_pass_kernels.get(kernel_key).expect("single_pass kernel missing");kernel.encode(queries,keys,@@ -392,14 +453,8 @@ impl<B: Backend> Attention<B> {);},KernelVariant::TwoPass => {- let kernel_pass1 = self- .two_pass_1_kernels- .get(kernel_key)- .unwrap_or_else(|| panic!("Can not find AttentionTwoPass1Kernel for key {:?}", kernel_key));- let kernel_pass2 = self- .two_pass_2_kernels- .get(&(head_dim as u32))- .unwrap_or_else(|| panic!("Can not find AttentionTwoPass2Kernel for key {:?}", kernel_key));+ let kernel_pass1 = self.two_pass_1_kernels.get(kernel_key).expect("two_pass_1 kernel missing");+ let kernel_pass2 = self.two_pass_2_kernels.get(&(head_dim as u32)).expect("two_pass_2 kernel missing");let mut partials = encoder.allocate_scratch(size_for_shape(&[num_heads * suffix_length * 32 * head_dim], DataType::F32))?;let mut sums =@@ -442,11 +497,129 @@ impl<B: Backend> Attention<B> {Ok(attention_output)}++ #[allow(clippy::too_many_arguments)]+ fn encode_fallback(+ &self,+ is_kv_cache_ring: bool,+ queries: &Allocation<B>,+ keys: &Allocation<B>,+ values: &Allocation<B>,+ attention_output: &mut Allocation<B>,+ sinks_allocation: Option<&Allocation<B>>,+ gqa_factor: usize,+ sequence_length: usize,+ k_head_stride: u32,+ k_seq_stride: u32,+ v_head_stride: u32,+ v_seq_stride: u32,+ ring_params: Option<RingParams>,+ scale: f32,+ num_heads: usize,+ num_groups: usize,+ suffix_length: usize,+ head_dim: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<(), B::Error> {+ let matmul = self.matmul_kernel.as_ref().expect("MatmulKernel required for head_dim=512 fallback");+ let scatter_scores =+ self.fallback_scatter_scores_kernels.get(&is_kv_cache_ring).expect("scatter_scores kernel missing");+ let dt_bytes = self.data_type.size_in_bytes();++ let mut scores =+ encoder.allocate_scratch(size_for_shape(&[num_heads * suffix_length * sequence_length], self.data_type))?;+ let mut group_scores =+ encoder.allocate_scratch(size_for_shape(&[gqa_factor * suffix_length, sequence_length], self.data_type))?;++ for group_index in 0..num_groups {+ matmul.borrow_mut().encode(+ MatmulArguments {+ a: queries,+ a_offset: group_index * gqa_factor * suffix_length * head_dim * dt_bytes,+ b: keys,+ b_offset: group_index * k_head_stride as usize * dt_bytes,+ b_leading_dimension: Some(k_seq_stride),+ b_transpose: true,+ ab_scale: scale,+ c: MatmulArgumentC::None,+ d: &mut group_scores,+ batch_dim: (gqa_factor * suffix_length) as u32,+ input_dim: head_dim as u32,+ output_dim: sequence_length as u32,+ },+ encoder,+ );+ scatter_scores.encode(+ &group_scores,+ &mut scores,+ ring_params,+ None::<&Allocation<B>>,+ self.sliding_window_size.map(|s| s as u32),+ group_index as u32,+ gqa_factor as u32,+ sequence_length as u32,+ suffix_length as u32,+ (gqa_factor * suffix_length * sequence_length) as u32,+ encoder,+ );+ }++ let softmax = if sinks_allocation.is_some() {+ self.softmax_sinks_kernel.as_ref().expect("softmax_sinks_kernel missing but sinks provided")+ } else {+ &self.softmax_kernel+ };+ softmax.encode(+ &mut scores,+ sinks_allocation,+ sequence_length as u32,+ num_heads as u32,+ suffix_length as u32,+ encoder,+ );++ let mut group_output =+ encoder.allocate_scratch(size_for_shape(&[gqa_factor * suffix_length, head_dim], self.data_type))?;++ for group_index in 0..num_groups {+ matmul.borrow_mut().encode(+ MatmulArguments {+ a: &scores,+ a_offset: group_index * gqa_factor * suffix_length * sequence_length * dt_bytes,+ b: values,+ b_offset: group_index * v_head_stride as usize * dt_bytes,+ b_leading_dimension: Some(v_seq_stride),+ b_transpose: false,+ ab_scale: 1.0,+ c: MatmulArgumentC::None,+ d: &mut group_output,+ batch_dim: (gqa_factor * suffix_length) as u32,+ input_dim: sequence_length as u32,+ output_dim: head_dim as u32,+ },+ encoder,+ );+ self.fallback_scatter_values_kernel.encode(+ &group_output,+ &mut *attention_output,+ group_index as u32,+ gqa_factor as u32,+ suffix_length as u32,+ num_heads as u32,+ head_dim as u32,+ (gqa_factor * suffix_length * head_dim) as u32,+ encoder,+ );+ }++ Ok(())+ }}#[derive(Clone, Copy)]enum KernelVariant {Gemm,+ Fallback,SinglePass,TwoPass,}diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 3d9fa2414..0a7047244 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -563,6 +563,9 @@ impl<B: Backend> Embedding<B> {a: input_allocation,a_offset: 0,b: weights,+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,ab_scale: 1.0,c: MatmulArgumentC::None,d: &mut output_allocation,diff --git a/crates/backend-uzu/src/encodable_block/linear/full_precision.rs b/crates/backend-uzu/src/encodable_block/linear/full_precision.rsindex 32c2ec921..7c5e1063e 100644--- a/crates/backend-uzu/src/encodable_block/linear/full_precision.rs+++ b/crates/backend-uzu/src/encodable_block/linear/full_precision.rs@@ -133,6 +133,9 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {a: &input,a_offset: 0,b: &self.weights,+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,ab_scale: 1.0,c: match self.bias.as_ref() {Some(bias) => MatmulArgumentC::Bias(bias),diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rsindex 5d5d1048d..08b5516f7 100644--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -106,6 +106,9 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {a: &input,a_offset: 0,b: &self.adapter_down,+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,ab_scale: 1.0,c: MatmulArgumentC::None,d: &mut intermediate,@@ -123,6 +126,9 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {a: &intermediate,a_offset: 0,b: &self.adapter_up,+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,ab_scale: self.lora_scale,c: MatmulArgumentC::Accumulate,d: &mut output,diff --git a/crates/backend-uzu/src/session/tts_session/fishaudio.rs b/crates/backend-uzu/src/session/tts_session/fishaudio.rsindex 108248bc4..3bb6aac09 100644--- a/crates/backend-uzu/src/session/tts_session/fishaudio.rs+++ b/crates/backend-uzu/src/session/tts_session/fishaudio.rs@@ -519,6 +519,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {a: slow_hidden_capture,a_offset: 0,b: weights,+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,ab_scale: 1.0,c: MatmulArgumentC::None,d: output_embedding,diff --git a/crates/backend-uzu/tests/performance/matmul/bench.rs b/crates/backend-uzu/tests/performance/matmul/bench.rsindex d21ef12ae..980660d38 100644--- a/crates/backend-uzu/tests/performance/matmul/bench.rs+++ b/crates/backend-uzu/tests/performance/matmul/bench.rs@@ -53,6 +53,9 @@ fn encode_and_run(a: a_allocation,a_offset: 0,b: b_allocation,+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,ab_scale: 1.0,c: MatmulArgumentC::None,d: d_allocation,diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rsnew file mode 100644index 000000000..5e6969ae5--- /dev/null+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs@@ -0,0 +1,139 @@+use backend_uzu::{+ DataType,+ backends::{+ common::{+ Allocation, Backend, Context, Encoder, Kernels,+ kernel::{+ AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, AttentionSinglePassKernel,+ ManualKernels, SoftmaxKernel,+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},+ },+ },+ cpu::Cpu,+ },+};+use half::bf16;++use crate::{+ common::{+ assert::assert_eq_float,+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ },+ uzu_test,+};++const NUM_HEADS: u32 = 4;+const NUM_GROUPS: u32 = 2;+const SEQ: u32 = 16;+const SUFFIX: u32 = 3;+const HEAD_DIM: u32 = 512;+const GQA: u32 = NUM_HEADS / NUM_GROUPS;++fn make_inputs() -> (Box<[bf16]>, Box<[bf16]>, Box<[bf16]>) {+ let q = (0..(NUM_HEADS * SUFFIX * HEAD_DIM) as usize)+ .map(|i| bf16::from_f32((i as f32 * 0.013 + 0.5).sin() * 0.5))+ .collect();+ let k = (0..(NUM_GROUPS * SEQ * HEAD_DIM) as usize)+ .map(|i| bf16::from_f32((i as f32 * 0.007 + 1.0).cos() * 0.5))+ .collect();+ let v = (0..(NUM_GROUPS * SEQ * HEAD_DIM) as usize)+ .map(|i| bf16::from_f32((i as f32 * 0.011 + 2.0).sin() * 0.5))+ .collect();+ (q, k, v)+}++// Ground truth: fused single-pass attention on the CPU backend.+fn reference(q: &[bf16], k: &[bf16], v: &[bf16], scale: f32) -> Vec<bf16> {+ let ctx = <Cpu as Backend>::Context::new().unwrap();+ let kernel = <<Cpu as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(+ &ctx, DataType::BF16, HEAD_DIM, false, false, true, false, false,+ )+ .unwrap();+ let qa = alloc_allocation_with_data::<Cpu, bf16>(&ctx, q);+ let ka = alloc_allocation_with_data::<Cpu, bf16>(&ctx, k);+ let va = alloc_allocation_with_data::<Cpu, bf16>(&ctx, v);+ let mut out = alloc_allocation::<Cpu, bf16>(&ctx, (NUM_HEADS * SUFFIX * HEAD_DIM) as usize);+ let mut enc = Encoder::new(ctx.as_ref()).unwrap();+ kernel.encode(+ &qa, &ka, &va, &mut out, GQA, SEQ, SEQ * HEAD_DIM, HEAD_DIM, SEQ * HEAD_DIM, HEAD_DIM, None,+ scale, None::<&Allocation<Cpu>>, None, None::<&Allocation<Cpu>>, NUM_HEADS, SUFFIX, &mut enc,+ );+ enc.end_encoding().submit().wait_until_completed().unwrap();+ allocation_to_vec::<Cpu, bf16>(&out)+}++// Production fallback pipeline (matmul → scatter → softmax → matmul → scatter)+// driven directly via the kernel-cache traits, for backends other than CPU.+fn pipeline_output<B: Backend>(q: &[bf16], k: &[bf16], v: &[bf16], scale: f32) -> Vec<bf16> {+ let ctx = B::Context::new().unwrap();+ let softmax = <<B as Backend>::Kernels as Kernels>::SoftmaxKernel::new(&ctx, DataType::BF16, false).unwrap();+ let scatter_scores =+ <<B as Backend>::Kernels as Kernels>::AttentionFallbackScatterScoresKernel::new(+ &ctx, DataType::BF16, false, true, false, false,+ )+ .unwrap();+ let scatter_values =+ <<B as Backend>::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(&ctx, DataType::BF16).unwrap();+ let mut matmul = <<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&ctx, DataType::BF16).unwrap();++ let qa = alloc_allocation_with_data::<B, bf16>(&ctx, q);+ let ka = alloc_allocation_with_data::<B, bf16>(&ctx, k);+ let va = alloc_allocation_with_data::<B, bf16>(&ctx, v);+ let mut scores = alloc_allocation::<B, bf16>(&ctx, (NUM_HEADS * SUFFIX * SEQ) as usize);+ let mut grp_s = alloc_allocation::<B, bf16>(&ctx, (GQA * SUFFIX * SEQ) as usize);+ let mut grp_o = alloc_allocation::<B, bf16>(&ctx, (GQA * SUFFIX * HEAD_DIM) as usize);+ let mut out = alloc_allocation::<B, bf16>(&ctx, (NUM_HEADS * SUFFIX * HEAD_DIM) as usize);++ let dt = DataType::BF16.size_in_bytes();+ let q_row = HEAD_DIM as usize * dt;+ let s_row = SEQ as usize * dt;+ let head_stride = (SEQ * HEAD_DIM) as usize;+ let mut enc = Encoder::new(ctx.as_ref()).unwrap();++ for g in 0..NUM_GROUPS {+ matmul.encode(+ MatmulArguments {+ a: &qa, a_offset: g as usize * GQA as usize * SUFFIX as usize * q_row,+ b: &ka, b_offset: g as usize * head_stride * dt,+ b_leading_dimension: Some(HEAD_DIM), b_transpose: true,+ ab_scale: scale, c: MatmulArgumentC::None, d: &mut grp_s,+ batch_dim: GQA * SUFFIX, input_dim: HEAD_DIM, output_dim: SEQ,+ },+ &mut enc,+ );+ scatter_scores.encode(+ &grp_s, &mut scores, None, None::<&Allocation<B>>, None,+ g, GQA, SEQ, SUFFIX, GQA * SUFFIX * SEQ, &mut enc,+ );+ }+ softmax.encode(&mut scores, None::<&Allocation<B>>, SEQ, NUM_HEADS, SUFFIX, &mut enc);+ for g in 0..NUM_GROUPS {+ matmul.encode(+ MatmulArguments {+ a: &scores, a_offset: g as usize * GQA as usize * SUFFIX as usize * s_row,+ b: &va, b_offset: g as usize * head_stride * dt,+ b_leading_dimension: Some(HEAD_DIM), b_transpose: false,+ ab_scale: 1.0, c: MatmulArgumentC::None, d: &mut grp_o,+ batch_dim: GQA * SUFFIX, input_dim: SEQ, output_dim: HEAD_DIM,+ },+ &mut enc,+ );+ scatter_values.encode(+ &grp_o, &mut out, g, GQA, SUFFIX, NUM_HEADS, HEAD_DIM,+ GQA * SUFFIX * HEAD_DIM, &mut enc,+ );+ }+ enc.end_encoding().submit().wait_until_completed().unwrap();+ allocation_to_vec::<B, bf16>(&out)+}++#[uzu_test]+fn test_head_dim_512_bf16() {+ let (q, k, v) = make_inputs();+ let scale = 1.0 / (HEAD_DIM as f32).sqrt();+ let expected = reference(&q, &k, &v, scale);+ for_each_non_cpu_backend!(|B| {+ let output = pipeline_output::<B>(&q, &k, &v, scale);+ assert_eq_float::<bf16>(&expected, &output, 1e-2, "AttentionFallback bf16");+ });+}diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rsindex ac8e23339..951c29f92 100644--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs@@ -187,8 +187,8 @@ fn test_gqa<T: ArrayElement + Float + Debug + Display>() {test_internal(&input, &expected);}-fn test_head_dim_128<T: ArrayElement + Float + Debug + Display>() {- let input = get_input::<T>(4, 4, 8, 2, 128, true);+fn test_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {+ let input = get_input::<T>(4, 4, 8, 2, head_dim, true);let expected = get_output::<T, Cpu>(&input);test_internal(&input, &expected);}@@ -244,15 +244,30 @@ fn test_gqa_bf16() {// Head dim 128#[uzu_test]fn test_head_dim_128_f32() {- test_head_dim_128::<f32>();+ test_head_dim::<f32>(128);}#[uzu_test]fn test_head_dim_128_f16() {- test_head_dim_128::<f16>();+ test_head_dim::<f16>(128);}#[uzu_test]fn test_head_dim_128_bf16() {- test_head_dim_128::<bf16>();+ test_head_dim::<bf16>(128);+}++#[uzu_test]+fn test_head_dim_512_f32() {+ test_head_dim::<f32>(512);+}++#[uzu_test]+fn test_head_dim_512_f16() {+ test_head_dim::<f16>(512);+}++#[uzu_test]+fn test_head_dim_512_bf16() {+ test_head_dim::<bf16>(512);}diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rsindex 64102c020..2f796c08d 100644--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs@@ -287,8 +287,8 @@ fn test_first_pass_gqa<T: ArrayElement + Float + Debug + Display>() {test_first_pass_internal(&input, &expected);}-fn test_first_pass_head_dim_128<T: ArrayElement + Float + Debug + Display>() {- let input = get_first_pass_input::<T>(4, 4, 8, 2, 128, true);+fn test_first_pass_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {+ let input = get_first_pass_input::<T>(4, 4, 8, 2, head_dim, true);let expected = get_first_pass_output::<T, Cpu>(&input);test_first_pass_internal(&input, &expected);}@@ -328,8 +328,8 @@ fn test_second_pass_basic<T: ArrayElement + Float + Debug + Display>() {test_second_pass_internal::<T>(&input, &expected);}-fn test_second_pass_head_dim_128<T: ArrayElement + Float + Debug + Display>() {- let input = get_second_pass_input(4, 2, 128);+fn test_second_pass_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {+ let input = get_second_pass_input(4, 2, head_dim);let expected = get_second_pass_output::<T, Cpu>(&input);test_second_pass_internal::<T>(&input, &expected);}@@ -383,17 +383,32 @@ fn test_first_pass_gqa_bf16() {#[uzu_test]fn test_first_pass_head_dim_128_f32() {- test_first_pass_head_dim_128::<f32>();+ test_first_pass_head_dim::<f32>(128);}#[uzu_test]fn test_first_pass_head_dim_128_f16() {- test_first_pass_head_dim_128::<f16>();+ test_first_pass_head_dim::<f16>(128);}#[uzu_test]fn test_first_pass_head_dim_128_bf16() {- test_first_pass_head_dim_128::<bf16>();+ test_first_pass_head_dim::<bf16>(128);+}++#[uzu_test]+fn test_first_pass_head_dim_512_f32() {+ test_first_pass_head_dim::<f32>(512);+}++#[uzu_test]+fn test_first_pass_head_dim_512_f16() {+ test_first_pass_head_dim::<f16>(512);+}++#[uzu_test]+fn test_first_pass_head_dim_512_bf16() {+ test_first_pass_head_dim::<bf16>(512);}// --- Second pass test entries ---@@ -415,15 +430,30 @@ fn test_second_pass_basic_bf16() {#[uzu_test]fn test_second_pass_head_dim_128_f32() {- test_second_pass_head_dim_128::<f32>();+ test_second_pass_head_dim::<f32>(128);}#[uzu_test]fn test_second_pass_head_dim_128_f16() {- test_second_pass_head_dim_128::<f16>();+ test_second_pass_head_dim::<f16>(128);}#[uzu_test]fn test_second_pass_head_dim_128_bf16() {- test_second_pass_head_dim_128::<bf16>();+ test_second_pass_head_dim::<bf16>(128);+}++#[uzu_test]+fn test_second_pass_head_dim_512_f32() {+ test_second_pass_head_dim::<f32>(512);+}++#[uzu_test]+fn test_second_pass_head_dim_512_f16() {+ test_second_pass_head_dim::<f16>(512);+}++#[uzu_test]+fn test_second_pass_head_dim_512_bf16() {+ test_second_pass_head_dim::<bf16>(512);}diff --git a/crates/backend-uzu/tests/unit/kernel/attention/mod.rs b/crates/backend-uzu/tests/unit/kernel/attention/mod.rsindex 661eb7fb1..796bfc81e 100644--- a/crates/backend-uzu/tests/unit/kernel/attention/mod.rs+++ b/crates/backend-uzu/tests/unit/kernel/attention/mod.rs@@ -1,3 +1,4 @@+mod attention_fallback_test;mod attention_gemm_test;mod attention_single_pass_test;mod attention_two_pass_test;diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rsindex 018aa3458..00204a6bd 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs@@ -98,6 +98,9 @@ fn get_cpu_output(input: &Input) -> Vec<bf16> {a: &left_allocation,a_offset: 0,b: right_array.allocation(),+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,ab_scale: input.ab_scale,c: argument_c,d: &mut destination_allocation,@@ -141,6 +144,9 @@ fn get_mpp_output(a: &left_allocation,a_offset: 0,b: right_array.allocation(),+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,ab_scale: input.ab_scale,c: argument_c,d: &mut destination_allocation,@@ -307,6 +313,9 @@ fn bench_gemm_mpp(criterion: &mut Criterion) {a: &left_allocation,a_offset: 0,b: right_array.allocation(),+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,ab_scale: 1.0,c: MatmulArgumentC::None,d: &mut destination_allocation,diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rsindex e1fa1aa86..3bd5359b6 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs@@ -82,6 +82,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>, ab_scale: fa: &a_allocation,a_offset: 0,b: b_array.allocation(),+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,ab_scale,c: MatmulArgumentC::None,d: &mut d_allocation,@@ -222,6 +225,9 @@ fn bench_gemm(criterion: &mut Criterion) {a: &left_allocation,a_offset: 0,b: right_array.allocation(),+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,ab_scale: 1.0,c: MatmulArgumentC::None,d: &mut destination_allocation,diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rsindex 351338060..d99850a4b 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs@@ -73,6 +73,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {a: &a_allocation,a_offset: 0,b: b_array.allocation(),+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,ab_scale: 1.0,c: MatmulArgumentC::None,d: &mut d_allocation,