Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Support Attention head_dim=512 (#412)

CCYeh5ff6f3a180e0
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul.rs b/crates/backend-uzu/src/backends/common/kernel/matmul.rs
index e8ef8c962..4986dbe47 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul.rs
@@ -21,13 +21,16 @@ pub enum MatmulArgumentC<'a, B: Backend> {
Bias(&'a Allocation<B>),
}
-// D = ab_scale * (A @ B.T) + C
+// D = ab_scale * (A @ op(B)) + C, where op(B) = B^T when b_transpose else B.
pub struct MatmulArguments<'a, B: Backend> {
/// A: [M, K]
pub a: &'a Allocation<B>,
pub a_offset: usize,
- /// B: [N, K]
+ /// B: [N, K] when b_transpose, else [K, N]
pub b: &'a Allocation<B>,
+ pub b_offset: usize,
+ pub b_leading_dimension: Option<u32>,
+ pub b_transpose: bool,
/// AB scale: also known as alpha
pub ab_scale: f32,
/// C: behavior depends on enum variant
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_fallback.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_fallback.rs
new file mode 100644
index 000000000..bbb825465
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_fallback.rs
@@ -0,0 +1,94 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::{
+ ArrayElement,
+ backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},
+};
+
+#[kernel(AttentionFallbackScatterScores)]
+#[variants(T, f32, f16, bf16)]
+pub fn attention_fallback_scatter_scores<T: ArrayElement + Float>(
+ group_scores: *const T,
+ scores: *mut T,
+ #[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,
+ #[optional(is_trie)] trie: Option<*const TrieNode>,
+ #[optional(is_sliding_window)] sliding_window_size: Option<u32>,
+ group_index: u32,
+ gqa_factor: u32,
+ sequence_length: u32,
+ suffix_length: u32,
+ #[allow(unused)] total_elements: u32,
+ #[specialize] is_kv_cache_ring: bool,
+ #[specialize] is_causal: bool,
+ #[specialize] is_trie: bool,
+ #[specialize] is_sliding_window: bool,
+) {
+ let prefix_length = sequence_length - suffix_length;
+ let suffix_position = ring_params.map(|params| params.ring_length).unwrap_or(prefix_length);
+
+ for head_in_group in 0..gqa_factor {
+ let head_index = group_index * gqa_factor + head_in_group;
+ for query_index in 0..suffix_length {
+ let query_position = if is_trie {
+ let trie_node = unsafe { &*trie.unwrap().add(query_index as usize) };
+ suffix_position + trie_node.height
+ } else {
+ suffix_position + query_index
+ };
+
+ for sequence_index in 0..sequence_length {
+ let source_index =
+ ((head_in_group * suffix_length + query_index) * sequence_length + sequence_index) as usize;
+ let destination_index =
+ ((head_index * suffix_length + query_index) * sequence_length + sequence_index) as usize;
+ let use_key = should_use_key(
+ ring_params,
+ trie,
+ sliding_window_size,
+ query_index,
+ prefix_length,
+ suffix_position,
+ query_position,
+ sequence_index,
+ is_causal,
+ );
+ unsafe {
+ *scores.add(destination_index) = if use_key {
+ *group_scores.add(source_index)
+ } else {
+ T::from(f32::NEG_INFINITY).unwrap()
+ };
+ }
+ }
+ }
+ }
+}
+
+#[kernel(AttentionFallbackScatterValues)]
+#[variants(T, f32, f16, bf16)]
+pub fn attention_fallback_scatter_values<T: ArrayElement + Float>(
+ group_output: *const T,
+ out: *mut T,
+ group_index: u32,
+ gqa_factor: u32,
+ suffix_length: u32,
+ num_heads: u32,
+ head_dim: u32,
+ #[allow(unused)] total_elements: u32,
+) {
+ for head_in_group in 0..gqa_factor as usize {
+ for query_index in 0..suffix_length as usize {
+ for dim_index in 0..head_dim as usize {
+ let source_index =
+ (head_in_group * suffix_length as usize + query_index) * head_dim as usize + dim_index;
+ let head_index = group_index as usize * gqa_factor as usize + head_in_group;
+ let destination_index = (query_index * num_heads as usize + head_index) * head_dim as usize + dim_index;
+ unsafe {
+ *out.add(destination_index) = *group_output.add(source_index);
+ }
+ }
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs
index 786a4a744..90282d1e9 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs
@@ -9,7 +9,7 @@ use crate::{
#[kernel(AttentionSinglePass)]
#[variants(T, f32, f16, bf16)]
-#[variants(HEAD_DIM, 64, 128, 256)]
+#[variants(HEAD_DIM, 64, 128, 256, 512)]
pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(
queries: *const T,
keys: *const T,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
index 5bbfb3148..da4a7695f 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
@@ -11,7 +11,7 @@ const TOTAL_BLOCKS_COUNT: u32 = 32;
#[kernel(AttentionTwoPass1)]
#[variants(T, f32, f16, bf16)]
-#[variants(HEAD_DIM, 64, 128, 256)]
+#[variants(HEAD_DIM, 64, 128, 256, 512)]
pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(
queries: *const T,
keys: *const T,
@@ -139,7 +139,7 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(
#[kernel(AttentionTwoPass2)]
#[variants(T, f32, f16, bf16)]
-#[variants(HEAD_DIM, 64, 128, 256)]
+#[variants(HEAD_DIM, 64, 128, 256, 512)]
pub fn attention_two_pass2<T: ArrayElement + Float, const HEAD_DIM: u32>(
partials: *const f32,
sums: *const f32,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs
index c862aeaf0..d84c4a40d 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs
@@ -1,3 +1,4 @@
+pub mod attention_fallback;
pub mod attention_gemm;
pub mod attention_single_pass;
pub mod attention_two_pass;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
index eb9fe033b..cbafef074 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
@@ -17,6 +17,7 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(
leading_dimension_a: usize,
leading_dimension_b: usize,
leading_dimension_d: usize,
+ b_transpose: bool,
is_accumulate: bool,
) {
unsafe {
@@ -24,7 +25,11 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(
let mut acc: f32 = 0.0;
for i in 0..k {
let a_val = *a.0.add(row * leading_dimension_a + i);
- let b_val = *b.0.add(col * leading_dimension_b + i);
+ let b_val = if b_transpose {
+ *b.0.add(col * leading_dimension_b + i)
+ } else {
+ *b.0.add(i * leading_dimension_b + col)
+ };
acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();
}
let d_element = d.0.add(row * leading_dimension_d + col);
@@ -48,6 +53,7 @@ pub fn matmul_gemm_impl<T: ArrayElement + Float>(
leading_dimension_a: usize,
leading_dimension_b: usize,
leading_dimension_d: usize,
+ b_transpose: bool,
is_accumulate: bool,
) {
let num_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
@@ -83,6 +89,7 @@ pub fn matmul_gemm_impl<T: ArrayElement + Float>(
leading_dimension_a,
leading_dimension_b,
leading_dimension_d,
+ b_transpose,
is_accumulate,
);
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
index fe73bacc4..3cc122c5d 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -40,6 +40,9 @@ impl MatmulKernel for MatmulCpuKernel {
a,
a_offset,
b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
ab_scale,
c,
d,
@@ -52,14 +55,18 @@ impl MatmulKernel for MatmulCpuKernel {
let n = output_dim as usize;
let k = input_dim as usize;
let lda = k;
- let ldb = k;
+ let ldb = b_leading_dimension.map(|n| n as usize).unwrap_or(if b_transpose {
+ k
+ } else {
+ n
+ });
let ldd = n;
let data_type = self.data_type;
let a_buffer_range = a.as_buffer_range_ref();
let b_buffer_range = b.as_buffer_range_ref();
let d_buffer_range = d.as_buffer_range_mut();
let a_offset = a_buffer_range.range().start + a_offset;
- let b_offset = b_buffer_range.range().start;
+ let b_offset = b_buffer_range.range().start + b_offset;
let d_offset = d_buffer_range.range().start;
let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_offset));
@@ -95,6 +102,7 @@ impl MatmulKernel for MatmulCpuKernel {
lda,
ldb,
ldd,
+ b_transpose,
is_accumulate,
);
if let Some(bias) = bias_ptr {
@@ -113,6 +121,7 @@ impl MatmulKernel for MatmulCpuKernel {
lda,
ldb,
ldd,
+ b_transpose,
is_accumulate,
);
if let Some(bias) = bias_ptr {
@@ -131,6 +140,7 @@ impl MatmulKernel for MatmulCpuKernel {
lda,
ldb,
ldd,
+ b_transpose,
is_accumulate,
);
if let Some(bias) = bias_ptr {
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
index 1e14e6cd4..421ec7c18 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
@@ -17,6 +17,7 @@ mod rms_norm;
mod rope;
mod sampling;
mod short_conv;
+mod softmax;
mod ssm;
mod tensor_add_bias;
mod tensor_add_scale;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/softmax/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/softmax/mod.rs
new file mode 100644
index 000000000..c85937510
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/softmax/mod.rs
@@ -0,0 +1 @@
+pub mod softmax;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rs b/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rs
new file mode 100644
index 000000000..3399ddf5e
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/softmax/softmax.rs
@@ -0,0 +1,57 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(Softmax)]
+#[variants(T, f32, f16, bf16)]
+pub fn softmax<T: ArrayElement + Float>(
+ values: *mut T,
+ #[optional(has_sinks)] sinks: Option<*const f32>,
+ row_length: u32,
+ outer_dim: u32,
+ batch_dim: u32,
+ #[specialize] has_sinks: bool,
+) {
+ for outer_index in 0..outer_dim {
+ for batch_index in 0..batch_dim {
+ let row_offset = ((outer_index * batch_dim + batch_index) * row_length) as usize;
+ let sink = if has_sinks {
+ unsafe { *sinks.unwrap().add(outer_index as usize) }
+ } else {
+ f32::NEG_INFINITY
+ };
+
+ let mut max_value = if has_sinks {
+ sink
+ } else {
+ f32::NEG_INFINITY
+ };
+ for i in 0..row_length as usize {
+ max_value = max_value.max(unsafe { *values.add(row_offset + i) }.to_f32().unwrap());
+ }
+
+ let mut sum_exp = if has_sinks {
+ (sink - max_value).exp()
+ } else {
+ 0.0
+ };
+ for i in 0..row_length as usize {
+ let p = (unsafe { *values.add(row_offset + i) }.to_f32().unwrap() - max_value).exp();
+ sum_exp += p;
+ unsafe {
+ *values.add(row_offset + i) = T::from(p).unwrap();
+ }
+ }
+
+ let inv = 1.0 / sum_exp;
+ for i in 0..row_length as usize {
+ unsafe {
+ let p = *values.add(row_offset + i);
+ *values.add(row_offset + i) = T::from(p.to_f32().unwrap() * inv).unwrap();
+ }
+ }
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metal
new file mode 100644
index 000000000..cccd17c89
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metal
@@ -0,0 +1,86 @@
+#include <metal_stdlib>
+#include <metal_simdgroup>
+#include "../common/defines.h"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../generated/ring.h"
+#include "../generated/trie.h"
+#include "mask.h"
+
+using namespace uzu::ring;
+using namespace uzu::trie;
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AttentionFallbackScatterScores)(
+ const device T* group_scores,
+ device T* scores,
+ const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),
+ const device TrieNode* trie OPTIONAL(is_trie),
+ const constant uint& sliding_window_size OPTIONAL(is_sliding_window),
+ const constant uint& group_index,
+ const constant uint& gqa_factor,
+ const constant uint& sequence_length,
+ const constant uint& suffix_length,
+ const constant uint& total_elements,
+ const bool is_kv_cache_ring SPECIALIZE,
+ const bool is_causal SPECIALIZE,
+ const bool is_trie SPECIALIZE,
+ const bool is_sliding_window SPECIALIZE,
+ const uint group_score_index AXIS(total_elements, 256)
+) {
+ const uint sequence_index = group_score_index % sequence_length;
+ const uint row_index = group_score_index / sequence_length;
+ const uint query_index = row_index % suffix_length;
+ const uint head_in_group = row_index / suffix_length;
+ const uint head_index = group_index * gqa_factor + head_in_group;
+
+ const uint prefix_length = sequence_length - suffix_length;
+ const uint suffix_position =
+ is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;
+ const uint query_position = is_trie
+ ? suffix_position + trie[query_index].height
+ : suffix_position + query_index;
+
+ const bool use_key = should_use_key(
+ ring_params,
+ trie,
+ sliding_window_size,
+ query_index,
+ prefix_length,
+ suffix_position,
+ query_position,
+ sequence_index,
+ is_kv_cache_ring,
+ is_causal,
+ is_trie,
+ is_sliding_window
+ );
+
+ scores
+ [(head_index * suffix_length + query_index) * sequence_length +
+ sequence_index] =
+ use_key ? group_scores[group_score_index] : T(-INFINITY);
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AttentionFallbackScatterValues)(
+ const device T* group_output,
+ device T* out,
+ const constant uint& group_index,
+ const constant uint& gqa_factor,
+ const constant uint& suffix_length,
+ const constant uint& num_heads,
+ const constant uint& head_dim,
+ const constant uint& total_elements,
+ const uint group_output_index AXIS(total_elements, 256)
+) {
+ const uint dim_index = group_output_index % head_dim;
+ const uint row_index = group_output_index / head_dim;
+ const uint query_index = row_index % suffix_length;
+ const uint head_in_group = row_index / suffix_length;
+ const uint head_index = group_index * gqa_factor + head_in_group;
+ out[(query_index * num_heads + head_index) * head_dim + dim_index] =
+ group_output[group_output_index];
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
index 5f75bbfde..368cc31e6 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -14,7 +14,7 @@ using namespace uzu::trie;
template <typename T, uint HEAD_DIM>
VARIANTS(T, float, half, bfloat)
-VARIANTS(HEAD_DIM, 64, 128, 256)
+VARIANTS(HEAD_DIM, 64, 128, 256, 512)
PUBLIC KERNEL(AttentionSinglePass)(
const device T* queries,
const device T* keys,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
index 9c3223e1a..3c36971e8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -19,7 +19,7 @@ using namespace uzu::trie;
// via integer division. Simdgroups sharing a KV head benefit from L1 cache.
template <typename T, uint HEAD_DIM>
VARIANTS(T, float, half, bfloat)
-VARIANTS(HEAD_DIM, 64, 128, 256)
+VARIANTS(HEAD_DIM, 64, 128, 256, 512)
PUBLIC KERNEL(AttentionTwoPass1)(
const device T* queries,
const device T* keys,
@@ -146,7 +146,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(
template <typename T, uint HEAD_DIM>
VARIANTS(T, float, half, bfloat)
-VARIANTS(HEAD_DIM, 64, 128, 256)
+VARIANTS(HEAD_DIM, 64, 128, 256, 512)
PUBLIC KERNEL(AttentionTwoPass2)(
const device float* partials,
const device float* sums,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal
index f790479f0..02ea47e3a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal
@@ -8,7 +8,7 @@ using namespace uzu::matmul;
#define GEMM_MAX_THREADGROUP_A 2560
#define GEMM_MAX_THREADGROUP_B 1536
-template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint BLOCK_DEPTH, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN, bool MN_ALIGNED, bool K_ALIGNED>
+template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint BLOCK_DEPTH, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN, bool MN_ALIGNED, bool K_ALIGNED, bool TRANSPOSE_B>
VARIANTS(T, float, half, bfloat)
VARIANTS(BLOCK_ROWS, 32, 64)
VARIANTS(BLOCK_COLS, 32, 64)
@@ -17,6 +17,7 @@ VARIANTS(SIMDGROUPS_PER_ROW, 1, 2, 4)
VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2)
VARIANTS(MN_ALIGNED, false, true)
VARIANTS(K_ALIGNED, false, true)
+VARIANTS(TRANSPOSE_B, false, true)
CONSTRAINT(max(BLOCK_ROWS, BLOCK_COLS) <= 32 * SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN)
KERNEL(MatmulGemm)(
const device T* a,
@@ -45,7 +46,7 @@ KERNEL(MatmulGemm)(
SIMDGROUPS_PER_ROW,
SIMDGROUPS_PER_COLUMN,
false, // transpose_a
- true, // transpose_b
+ TRANSPOSE_B,
MN_ALIGNED,
K_ALIGNED,
float>::
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs
index 3350dfe46..96a346750 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs
@@ -15,6 +15,7 @@ pub struct GemmSpecialization {
pub simdgroups_per_column: u32,
pub align_mn: bool,
pub align_k: bool,
+ pub transpose_b: bool,
pub is_accumulate: bool,
}
@@ -30,6 +31,7 @@ impl GemmSpecialization {
simdgroups_per_column: 2,
align_mn: false,
align_k: true,
+ transpose_b: true,
is_accumulate: false,
},
Self {
@@ -40,6 +42,7 @@ impl GemmSpecialization {
simdgroups_per_column: 2,
align_mn: true,
align_k: true,
+ transpose_b: true,
is_accumulate: false,
},
Self {
@@ -50,6 +53,7 @@ impl GemmSpecialization {
simdgroups_per_column: 2,
align_mn: false,
align_k: true,
+ transpose_b: true,
is_accumulate: false,
},
Self {
@@ -60,6 +64,7 @@ impl GemmSpecialization {
simdgroups_per_column: 2,
align_mn: true,
align_k: true,
+ transpose_b: true,
is_accumulate: false,
},
Self {
@@ -70,6 +75,7 @@ impl GemmSpecialization {
simdgroups_per_column: 2,
align_mn: true,
align_k: true,
+ transpose_b: true,
is_accumulate: false,
},
],
@@ -82,6 +88,7 @@ impl GemmSpecialization {
simdgroups_per_column: 2,
align_mn: true,
align_k: true,
+ transpose_b: true,
is_accumulate: false,
},
Self {
@@ -92,6 +99,7 @@ impl GemmSpecialization {
simdgroups_per_column: 2,
align_mn: false,
align_k: true,
+ transpose_b: true,
is_accumulate: false,
},
],
@@ -104,6 +112,7 @@ impl GemmSpecialization {
simdgroups_per_column: 2,
align_mn: false,
align_k: true,
+ transpose_b: true,
is_accumulate: false,
},
Self {
@@ -114,6 +123,7 @@ impl GemmSpecialization {
simdgroups_per_column: 2,
align_mn: true,
align_k: true,
+ transpose_b: true,
is_accumulate: false,
},
],
@@ -127,6 +137,7 @@ impl GemmSpecialization {
batch_dim: u32,
input_dim: u32,
output_dim: u32,
+ transpose_b: bool,
c: &MatmulArgumentC<Metal>,
) -> Self {
let overall_work_elements = batch_dim * output_dim;
@@ -158,6 +169,7 @@ impl GemmSpecialization {
simdgroups_per_column,
align_mn: (batch_dim % block_rows) == 0 && (output_dim % block_cols) == 0,
align_k: (input_dim % block_depth) == 0,
+ transpose_b,
is_accumulate: matches!(c, MatmulArgumentC::Accumulate),
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index bdb779ca4..1939c7c37 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -99,6 +99,7 @@ impl MatmulMetalKernel {
specialization.simdgroups_per_column as u32,
specialization.align_mn,
specialization.align_k,
+ specialization.transpose_b,
specialization.is_accumulate,
)
.map_err(MatmulError::BackendError)?;
@@ -143,6 +144,9 @@ impl MatmulMetalKernel {
a,
a_offset,
b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
ab_scale,
c,
d,
@@ -150,6 +154,7 @@ impl MatmulMetalKernel {
input_dim,
output_dim,
} = arguments;
+ assert!(b_transpose, "encode_gemv does not yet support b_transpose=false");
let (is_accumulate, output_bias) = match c {
MatmulArgumentC::Accumulate => (true, None),
@@ -161,13 +166,13 @@ impl MatmulMetalKernel {
gemv::GemvSpecialization::select(input_dim, output_dim, is_accumulate, output_bias.is_some());
self.get_or_create_gemv(encoder.context(), specialization)?.encode(
- b,
+ (b, b_offset),
(a, a_offset),
output_bias,
d,
input_dim,
output_dim,
- input_dim,
+ b_leading_dimension.unwrap_or(input_dim),
ab_scale,
batch_dim,
specialization.output_rows_per_threadgroup(),
@@ -186,6 +191,9 @@ impl MatmulMetalKernel {
a,
a_offset,
b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
ab_scale,
c,
d,
@@ -194,18 +202,30 @@ impl MatmulMetalKernel {
output_dim,
} = arguments;
- let specialization =
- gemm::GemmSpecialization::select(encoder.context(), self.data_type, batch_dim, input_dim, output_dim, &c);
+ let specialization = gemm::GemmSpecialization::select(
+ encoder.context(),
+ self.data_type,
+ batch_dim,
+ input_dim,
+ output_dim,
+ b_transpose,
+ &c,
+ );
let threadgroups_per_row = output_dim.div_ceil(specialization.block_cols);
let threadgroups_per_column = batch_dim.div_ceil(specialization.block_rows);
+ let default_ldb = if b_transpose {
+ input_dim
+ } else {
+ output_dim
+ };
let params = GemmParams {
M: batch_dim,
N: output_dim,
K: input_dim,
leading_dimension_a: input_dim,
- leading_dimension_b: input_dim,
+ leading_dimension_b: b_leading_dimension.unwrap_or(default_ldb),
leading_dimension_d: output_dim,
threadgroups_per_row,
threadgroups_per_column,
@@ -218,7 +238,7 @@ impl MatmulMetalKernel {
kernel.encode(
(a, a_offset),
- b,
+ (b, b_offset),
&mut *d,
std::slice::from_ref(&params),
threadgroups_per_row,
@@ -243,6 +263,9 @@ impl MatmulMetalKernel {
a,
a_offset,
b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
ab_scale,
c,
d,
@@ -250,6 +273,7 @@ impl MatmulMetalKernel {
input_dim,
output_dim,
} = arguments;
+ assert!(b_transpose, "encode_gemm_mpp does not yet support b_transpose=false");
let is_accumulate = matches!(c, MatmulArgumentC::Accumulate);
let specialization =
gemm_mpp::GemmMppSpecialization::select(batch_dim, output_dim, input_dim, is_accumulate, ab_scale != 1.0);
@@ -271,7 +295,7 @@ impl MatmulMetalKernel {
N: output_dim,
K: input_dim,
leading_dimension_a: input_dim,
- leading_dimension_b: input_dim,
+ leading_dimension_b: b_leading_dimension.unwrap_or(input_dim),
leading_dimension_d: output_dim,
threadgroups_per_row,
threadgroups_per_column,
@@ -289,7 +313,7 @@ impl MatmulMetalKernel {
let kernel = self.get_or_create_gemm_mpp(encoder.context(), specialization)?;
kernel.encode(
(a, a_offset),
- b,
+ (b, b_offset),
&mut *d,
std::slice::from_ref(&params),
group_count_x,
@@ -372,7 +396,9 @@ impl MatmulKernel for MatmulMetalKernel {
arguments: MatmulArguments<Metal>,
encoder: &mut Encoder<Metal>,
) {
- if arguments.batch_dim <= max_gemv_batch_threshold() {
+ if !arguments.b_transpose {
+ self.encode_gemm(encoder, arguments).expect("Failed to encode GEMM kernel");
+ } else if arguments.batch_dim <= max_gemv_batch_threshold() {
self.encode_gemv(encoder, arguments).expect("Failed to encode GEMV kernel");
} else if self.is_mpp_eligible(encoder.context()) {
self.encode_gemm_mpp(encoder, arguments).expect("Failed to encode GEMM MPP kernel");
diff --git a/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal b/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal
new file mode 100644
index 000000000..d53fdd198
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal
@@ -0,0 +1,98 @@
+#include <metal_stdlib>
+
+#include "../common/defines.h"
+#include "../common/dsl.h"
+
+// Online row-wise softmax (MLX-style: simd intrinsics + 2-pass online
+// algorithm). `has_sinks` seeds thread 0 with (sinks[outer_index], 1) as a
+// virtual element.
+
+#define SOFTMAX_THREADS 256
+// Number of consecutive elements each thread loads per iteration. Vectorizing
+// the per-thread loop this way improves memory coalescing on Apple Silicon.
+#define SOFTMAX_ELEMENTS_PER_THREAD 4
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(Softmax)(
+ device T* values,
+ const device float* sinks OPTIONAL(has_sinks),
+ const constant uint& row_length,
+ const constant uint& outer_dim,
+ const constant uint& batch_dim,
+ threadgroup float shared_max[METAL_SIMD_SIZE],
+ threadgroup float shared_norm[METAL_SIMD_SIZE],
+ const bool has_sinks SPECIALIZE,
+ const uint outer_index GROUPS(outer_dim),
+ const uint batch_index GROUPS(batch_dim),
+ const uint tid THREADS(SOFTMAX_THREADS)
+) {
+ const uint lane = tid & (METAL_SIMD_SIZE - 1);
+ const uint simdgroup_id = tid / METAL_SIMD_SIZE;
+ const uint stride = SOFTMAX_THREADS * SOFTMAX_ELEMENTS_PER_THREAD;
+ device T* row = values + (outer_index * batch_dim + batch_index) * row_length;
+
+ float maxval = (has_sinks && tid == 0) ? sinks[outer_index] : -FLT_MAX;
+ float norm = (has_sinks && tid == 0) ? 1.0f : 0.0f;
+
+ if (simdgroup_id == 0) {
+ shared_max[lane] = -FLT_MAX;
+ shared_norm[lane] = 0.0f;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Pass 1: online (max, normalizer) accumulation over the row.
+ for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length;
+ base += stride) {
+ float vals[SOFTMAX_ELEMENTS_PER_THREAD];
+ for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {
+ vals[i] = (base + i < row_length) ? float(row[base + i]) : -FLT_MAX;
+ }
+ float prev_max = maxval;
+ for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {
+ maxval = max(maxval, vals[i]);
+ }
+ norm *= fast::exp(prev_max - maxval);
+ for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {
+ norm += fast::exp(vals[i] - maxval);
+ }
+ }
+
+ // Combine across the simdgroup, then across simdgroups via shared memory.
+ float prev_max = maxval;
+ maxval = simd_max(maxval);
+ norm *= fast::exp(prev_max - maxval);
+ norm = simd_sum(norm);
+
+ if (lane == 0) {
+ shared_max[simdgroup_id] = maxval;
+ shared_norm[simdgroup_id] = norm;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (simdgroup_id == 0) {
+ float simdgroup_max = shared_max[lane];
+ float simdgroup_norm = shared_norm[lane];
+ float global_max = simd_max(simdgroup_max);
+ simdgroup_norm *= fast::exp(simdgroup_max - global_max);
+ float global_norm = simd_sum(simdgroup_norm);
+ if (lane == 0) {
+ shared_max[0] = global_max;
+ shared_norm[0] = global_norm;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const float final_max = shared_max[0];
+ const float inv_norm = 1.0f / shared_norm[0];
+
+ // Pass 2: normalize.
+ for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length;
+ base += stride) {
+ for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {
+ if (base + i < row_length) {
+ row[base + i] =
+ T(fast::exp(float(row[base + i]) - final_max) * inv_norm);
+ }
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rs
index c6906f248..b9152eff9 100644
--- a/crates/backend-uzu/src/encodable_block/attention.rs
+++ b/crates/backend-uzu/src/encodable_block/attention.rs
@@ -1,6 +1,6 @@
//! Attention kernel encodable.
-use std::collections::HashMap;
+use std::{cell::RefCell, collections::HashMap};
use itertools::iproduct;
@@ -11,9 +11,11 @@ use crate::{
Allocation, Backend, Encoder, Kernels,
gpu_types::ring::RingParams,
kernel::{
- AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,
- SigmoidGateKernel,
+ AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, AttentionSinglePassKernel,
+ AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel, ManualKernels,
+ SigmoidGateKernel, SoftmaxKernel,
attention::{AttentionGemmArguments, AttentionGemmBlock},
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
},
},
config::AttentionConfig,
@@ -40,6 +42,11 @@ pub struct Attention<B: Backend> {
single_pass_kernels: HashMap<KernelKey, <B::Kernels as Kernels>::AttentionSinglePassKernel>,
two_pass_1_kernels: HashMap<KernelKey, <B::Kernels as Kernels>::AttentionTwoPass1Kernel>,
two_pass_2_kernels: HashMap<u32, <B::Kernels as Kernels>::AttentionTwoPass2Kernel>,
+ softmax_kernel: <B::Kernels as Kernels>::SoftmaxKernel,
+ softmax_sinks_kernel: Option<<B::Kernels as Kernels>::SoftmaxKernel>,
+ fallback_scatter_scores_kernels: HashMap<bool, <B::Kernels as Kernels>::AttentionFallbackScatterScoresKernel>,
+ fallback_scatter_values_kernel: <B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel,
+ matmul_kernel: Option<RefCell<<B::Kernels as ManualKernels>::MatmulKernel>>,
update_kv_cache_kernel: <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel,
update_kv_cache_inplace_kernel: <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel,
gate_kernel: Option<<B::Kernels as Kernels>::SigmoidGateKernel>,
@@ -66,8 +73,11 @@ impl<B: Backend> Attention<B> {
let mut single_pass_kernels = HashMap::new();
let mut two_pass_1_kernels = HashMap::new();
let mut two_pass_2_kernels = HashMap::new();
+ let mut fallback_scatter_scores_kernels = HashMap::new();
- for (head_dim, is_trie, is_kv_cache_ring) in iproduct!([64u32, 128u32, 256u32], [false, true], [false, true]) {
+ for (head_dim, is_trie, is_kv_cache_ring) in
+ iproduct!([64u32, 128u32, 256u32, 512u32], [false, true], [false, true])
+ {
let key = KernelKey {
head_dim,
is_trie,
@@ -102,6 +112,28 @@ impl<B: Backend> Attention<B> {
two_pass_2_kernels.insert(head_dim, tp2_kernel);
}
+ for is_kv_cache_ring in [false, true] {
+ let scatter = <B::Kernels as Kernels>::AttentionFallbackScatterScoresKernel::new(
+ context,
+ data_type,
+ is_kv_cache_ring,
+ config.is_causal,
+ false,
+ config.sliding_window_size.is_some(),
+ )?;
+ fallback_scatter_scores_kernels.insert(is_kv_cache_ring, scatter);
+ }
+
+ let softmax_kernel = <B::Kernels as Kernels>::SoftmaxKernel::new(context, data_type, false)?;
+ let softmax_sinks_kernel = if config.has_sinks {
+ Some(<B::Kernels as Kernels>::SoftmaxKernel::new(context, data_type, true)?)
+ } else {
+ None
+ };
+ let fallback_scatter_values_kernel =
+ <B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(context, data_type)?;
+ let matmul_kernel =
+ <<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(context, data_type).ok().map(RefCell::new);
let update_kv_cache_kernel =
<B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type, false)?;
let update_kv_cache_inplace_kernel =
@@ -117,6 +149,11 @@ impl<B: Backend> Attention<B> {
single_pass_kernels,
two_pass_1_kernels,
two_pass_2_kernels,
+ softmax_kernel,
+ softmax_sinks_kernel,
+ fallback_scatter_scores_kernels,
+ fallback_scatter_values_kernel,
+ matmul_kernel,
update_kv_cache_kernel,
update_kv_cache_inplace_kernel,
gate_kernel,
@@ -137,8 +174,12 @@ impl<B: Backend> Attention<B> {
is_trie: bool,
is_kv_cache_ring: bool,
) -> KernelVariant {
- let use_gemm = gemm_enabled && suffix_length > 8 && matches!(head_dim, 64 | 128 | 256);
- if use_gemm {
+ // head_dim=512 doesn't fit the fused GEMM tile in threadgroup memory.
+ if head_dim == 512 && suffix_length > 8 && !is_trie {
+ return KernelVariant::Fallback;
+ }
+
+ if gemm_enabled && suffix_length > 8 && matches!(head_dim, 64 | 128 | 256) {
return KernelVariant::Gemm;
}
@@ -365,11 +406,31 @@ impl<B: Backend> Attention<B> {
};
self.gemm_block.encode(encoder, args)?;
},
+ KernelVariant::Fallback => {
+ self.encode_fallback(
+ kernel_key.is_kv_cache_ring,
+ queries,
+ keys,
+ values,
+ &mut attention_output,
+ sinks_allocation,
+ gqa_factor,
+ sequence_length,
+ k_head_stride,
+ k_seq_stride,
+ v_head_stride,
+ v_seq_stride,
+ ring_params,
+ scale,
+ num_heads,
+ num_groups,
+ suffix_length,
+ head_dim,
+ encoder,
+ )?;
+ },
KernelVariant::SinglePass => {
- let kernel = self
- .single_pass_kernels
- .get(kernel_key)
- .unwrap_or_else(|| panic!("Can not find AttentionSinglePassKernel for key {:?}", kernel_key));
+ let kernel = self.single_pass_kernels.get(kernel_key).expect("single_pass kernel missing");
kernel.encode(
queries,
keys,
@@ -392,14 +453,8 @@ impl<B: Backend> Attention<B> {
);
},
KernelVariant::TwoPass => {
- let kernel_pass1 = self
- .two_pass_1_kernels
- .get(kernel_key)
- .unwrap_or_else(|| panic!("Can not find AttentionTwoPass1Kernel for key {:?}", kernel_key));
- let kernel_pass2 = self
- .two_pass_2_kernels
- .get(&(head_dim as u32))
- .unwrap_or_else(|| panic!("Can not find AttentionTwoPass2Kernel for key {:?}", kernel_key));
+ let kernel_pass1 = self.two_pass_1_kernels.get(kernel_key).expect("two_pass_1 kernel missing");
+ let kernel_pass2 = self.two_pass_2_kernels.get(&(head_dim as u32)).expect("two_pass_2 kernel missing");
let mut partials = encoder
.allocate_scratch(size_for_shape(&[num_heads * suffix_length * 32 * head_dim], DataType::F32))?;
let mut sums =
@@ -442,11 +497,129 @@ impl<B: Backend> Attention<B> {
Ok(attention_output)
}
+
+ #[allow(clippy::too_many_arguments)]
+ fn encode_fallback(
+ &self,
+ is_kv_cache_ring: bool,
+ queries: &Allocation<B>,
+ keys: &Allocation<B>,
+ values: &Allocation<B>,
+ attention_output: &mut Allocation<B>,
+ sinks_allocation: Option<&Allocation<B>>,
+ gqa_factor: usize,
+ sequence_length: usize,
+ k_head_stride: u32,
+ k_seq_stride: u32,
+ v_head_stride: u32,
+ v_seq_stride: u32,
+ ring_params: Option<RingParams>,
+ scale: f32,
+ num_heads: usize,
+ num_groups: usize,
+ suffix_length: usize,
+ head_dim: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(), B::Error> {
+ let matmul = self.matmul_kernel.as_ref().expect("MatmulKernel required for head_dim=512 fallback");
+ let scatter_scores =
+ self.fallback_scatter_scores_kernels.get(&is_kv_cache_ring).expect("scatter_scores kernel missing");
+ let dt_bytes = self.data_type.size_in_bytes();
+
+ let mut scores =
+ encoder.allocate_scratch(size_for_shape(&[num_heads * suffix_length * sequence_length], self.data_type))?;
+ let mut group_scores =
+ encoder.allocate_scratch(size_for_shape(&[gqa_factor * suffix_length, sequence_length], self.data_type))?;
+
+ for group_index in 0..num_groups {
+ matmul.borrow_mut().encode(
+ MatmulArguments {
+ a: queries,
+ a_offset: group_index * gqa_factor * suffix_length * head_dim * dt_bytes,
+ b: keys,
+ b_offset: group_index * k_head_stride as usize * dt_bytes,
+ b_leading_dimension: Some(k_seq_stride),
+ b_transpose: true,
+ ab_scale: scale,
+ c: MatmulArgumentC::None,
+ d: &mut group_scores,
+ batch_dim: (gqa_factor * suffix_length) as u32,
+ input_dim: head_dim as u32,
+ output_dim: sequence_length as u32,
+ },
+ encoder,
+ );
+ scatter_scores.encode(
+ &group_scores,
+ &mut scores,
+ ring_params,
+ None::<&Allocation<B>>,
+ self.sliding_window_size.map(|s| s as u32),
+ group_index as u32,
+ gqa_factor as u32,
+ sequence_length as u32,
+ suffix_length as u32,
+ (gqa_factor * suffix_length * sequence_length) as u32,
+ encoder,
+ );
+ }
+
+ let softmax = if sinks_allocation.is_some() {
+ self.softmax_sinks_kernel.as_ref().expect("softmax_sinks_kernel missing but sinks provided")
+ } else {
+ &self.softmax_kernel
+ };
+ softmax.encode(
+ &mut scores,
+ sinks_allocation,
+ sequence_length as u32,
+ num_heads as u32,
+ suffix_length as u32,
+ encoder,
+ );
+
+ let mut group_output =
+ encoder.allocate_scratch(size_for_shape(&[gqa_factor * suffix_length, head_dim], self.data_type))?;
+
+ for group_index in 0..num_groups {
+ matmul.borrow_mut().encode(
+ MatmulArguments {
+ a: &scores,
+ a_offset: group_index * gqa_factor * suffix_length * sequence_length * dt_bytes,
+ b: values,
+ b_offset: group_index * v_head_stride as usize * dt_bytes,
+ b_leading_dimension: Some(v_seq_stride),
+ b_transpose: false,
+ ab_scale: 1.0,
+ c: MatmulArgumentC::None,
+ d: &mut group_output,
+ batch_dim: (gqa_factor * suffix_length) as u32,
+ input_dim: sequence_length as u32,
+ output_dim: head_dim as u32,
+ },
+ encoder,
+ );
+ self.fallback_scatter_values_kernel.encode(
+ &group_output,
+ &mut *attention_output,
+ group_index as u32,
+ gqa_factor as u32,
+ suffix_length as u32,
+ num_heads as u32,
+ head_dim as u32,
+ (gqa_factor * suffix_length * head_dim) as u32,
+ encoder,
+ );
+ }
+
+ Ok(())
+ }
}
#[derive(Clone, Copy)]
enum KernelVariant {
Gemm,
+ Fallback,
SinglePass,
TwoPass,
}
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index 3d9fa2414..0a7047244 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -563,6 +563,9 @@ impl<B: Backend> Embedding<B> {
a: input_allocation,
a_offset: 0,
b: weights,
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
ab_scale: 1.0,
c: MatmulArgumentC::None,
d: &mut output_allocation,
diff --git a/crates/backend-uzu/src/encodable_block/linear/full_precision.rs b/crates/backend-uzu/src/encodable_block/linear/full_precision.rs
index 32c2ec921..7c5e1063e 100644
--- a/crates/backend-uzu/src/encodable_block/linear/full_precision.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/full_precision.rs
@@ -133,6 +133,9 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {
a: &input,
a_offset: 0,
b: &self.weights,
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
ab_scale: 1.0,
c: match self.bias.as_ref() {
Some(bias) => MatmulArgumentC::Bias(bias),
diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
index 5d5d1048d..08b5516f7 100644
--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
@@ -106,6 +106,9 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
a: &input,
a_offset: 0,
b: &self.adapter_down,
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
ab_scale: 1.0,
c: MatmulArgumentC::None,
d: &mut intermediate,
@@ -123,6 +126,9 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
a: &intermediate,
a_offset: 0,
b: &self.adapter_up,
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
ab_scale: self.lora_scale,
c: MatmulArgumentC::Accumulate,
d: &mut output,
diff --git a/crates/backend-uzu/src/session/tts_session/fishaudio.rs b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
index 108248bc4..3bb6aac09 100644
--- a/crates/backend-uzu/src/session/tts_session/fishaudio.rs
+++ b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
@@ -519,6 +519,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
a: slow_hidden_capture,
a_offset: 0,
b: weights,
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
ab_scale: 1.0,
c: MatmulArgumentC::None,
d: output_embedding,
diff --git a/crates/backend-uzu/tests/performance/matmul/bench.rs b/crates/backend-uzu/tests/performance/matmul/bench.rs
index d21ef12ae..980660d38 100644
--- a/crates/backend-uzu/tests/performance/matmul/bench.rs
+++ b/crates/backend-uzu/tests/performance/matmul/bench.rs
@@ -53,6 +53,9 @@ fn encode_and_run(
a: a_allocation,
a_offset: 0,
b: b_allocation,
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
ab_scale: 1.0,
c: MatmulArgumentC::None,
d: d_allocation,
diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs
new file mode 100644
index 000000000..5e6969ae5
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_fallback_test.rs
@@ -0,0 +1,139 @@
+use backend_uzu::{
+ DataType,
+ backends::{
+ common::{
+ Allocation, Backend, Context, Encoder, Kernels,
+ kernel::{
+ AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, AttentionSinglePassKernel,
+ ManualKernels, SoftmaxKernel,
+ matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
+ },
+ },
+ cpu::Cpu,
+ },
+};
+use half::bf16;
+
+use crate::{
+ common::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ },
+ uzu_test,
+};
+
+const NUM_HEADS: u32 = 4;
+const NUM_GROUPS: u32 = 2;
+const SEQ: u32 = 16;
+const SUFFIX: u32 = 3;
+const HEAD_DIM: u32 = 512;
+const GQA: u32 = NUM_HEADS / NUM_GROUPS;
+
+fn make_inputs() -> (Box<[bf16]>, Box<[bf16]>, Box<[bf16]>) {
+ let q = (0..(NUM_HEADS * SUFFIX * HEAD_DIM) as usize)
+ .map(|i| bf16::from_f32((i as f32 * 0.013 + 0.5).sin() * 0.5))
+ .collect();
+ let k = (0..(NUM_GROUPS * SEQ * HEAD_DIM) as usize)
+ .map(|i| bf16::from_f32((i as f32 * 0.007 + 1.0).cos() * 0.5))
+ .collect();
+ let v = (0..(NUM_GROUPS * SEQ * HEAD_DIM) as usize)
+ .map(|i| bf16::from_f32((i as f32 * 0.011 + 2.0).sin() * 0.5))
+ .collect();
+ (q, k, v)
+}
+
+// Ground truth: fused single-pass attention on the CPU backend.
+fn reference(q: &[bf16], k: &[bf16], v: &[bf16], scale: f32) -> Vec<bf16> {
+ let ctx = <Cpu as Backend>::Context::new().unwrap();
+ let kernel = <<Cpu as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
+ &ctx, DataType::BF16, HEAD_DIM, false, false, true, false, false,
+ )
+ .unwrap();
+ let qa = alloc_allocation_with_data::<Cpu, bf16>(&ctx, q);
+ let ka = alloc_allocation_with_data::<Cpu, bf16>(&ctx, k);
+ let va = alloc_allocation_with_data::<Cpu, bf16>(&ctx, v);
+ let mut out = alloc_allocation::<Cpu, bf16>(&ctx, (NUM_HEADS * SUFFIX * HEAD_DIM) as usize);
+ let mut enc = Encoder::new(ctx.as_ref()).unwrap();
+ kernel.encode(
+ &qa, &ka, &va, &mut out, GQA, SEQ, SEQ * HEAD_DIM, HEAD_DIM, SEQ * HEAD_DIM, HEAD_DIM, None,
+ scale, None::<&Allocation<Cpu>>, None, None::<&Allocation<Cpu>>, NUM_HEADS, SUFFIX, &mut enc,
+ );
+ enc.end_encoding().submit().wait_until_completed().unwrap();
+ allocation_to_vec::<Cpu, bf16>(&out)
+}
+
+// Production fallback pipeline (matmul → scatter → softmax → matmul → scatter)
+// driven directly via the kernel-cache traits, for backends other than CPU.
+fn pipeline_output<B: Backend>(q: &[bf16], k: &[bf16], v: &[bf16], scale: f32) -> Vec<bf16> {
+ let ctx = B::Context::new().unwrap();
+ let softmax = <<B as Backend>::Kernels as Kernels>::SoftmaxKernel::new(&ctx, DataType::BF16, false).unwrap();
+ let scatter_scores =
+ <<B as Backend>::Kernels as Kernels>::AttentionFallbackScatterScoresKernel::new(
+ &ctx, DataType::BF16, false, true, false, false,
+ )
+ .unwrap();
+ let scatter_values =
+ <<B as Backend>::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(&ctx, DataType::BF16).unwrap();
+ let mut matmul = <<B as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&ctx, DataType::BF16).unwrap();
+
+ let qa = alloc_allocation_with_data::<B, bf16>(&ctx, q);
+ let ka = alloc_allocation_with_data::<B, bf16>(&ctx, k);
+ let va = alloc_allocation_with_data::<B, bf16>(&ctx, v);
+ let mut scores = alloc_allocation::<B, bf16>(&ctx, (NUM_HEADS * SUFFIX * SEQ) as usize);
+ let mut grp_s = alloc_allocation::<B, bf16>(&ctx, (GQA * SUFFIX * SEQ) as usize);
+ let mut grp_o = alloc_allocation::<B, bf16>(&ctx, (GQA * SUFFIX * HEAD_DIM) as usize);
+ let mut out = alloc_allocation::<B, bf16>(&ctx, (NUM_HEADS * SUFFIX * HEAD_DIM) as usize);
+
+ let dt = DataType::BF16.size_in_bytes();
+ let q_row = HEAD_DIM as usize * dt;
+ let s_row = SEQ as usize * dt;
+ let head_stride = (SEQ * HEAD_DIM) as usize;
+ let mut enc = Encoder::new(ctx.as_ref()).unwrap();
+
+ for g in 0..NUM_GROUPS {
+ matmul.encode(
+ MatmulArguments {
+ a: &qa, a_offset: g as usize * GQA as usize * SUFFIX as usize * q_row,
+ b: &ka, b_offset: g as usize * head_stride * dt,
+ b_leading_dimension: Some(HEAD_DIM), b_transpose: true,
+ ab_scale: scale, c: MatmulArgumentC::None, d: &mut grp_s,
+ batch_dim: GQA * SUFFIX, input_dim: HEAD_DIM, output_dim: SEQ,
+ },
+ &mut enc,
+ );
+ scatter_scores.encode(
+ &grp_s, &mut scores, None, None::<&Allocation<B>>, None,
+ g, GQA, SEQ, SUFFIX, GQA * SUFFIX * SEQ, &mut enc,
+ );
+ }
+ softmax.encode(&mut scores, None::<&Allocation<B>>, SEQ, NUM_HEADS, SUFFIX, &mut enc);
+ for g in 0..NUM_GROUPS {
+ matmul.encode(
+ MatmulArguments {
+ a: &scores, a_offset: g as usize * GQA as usize * SUFFIX as usize * s_row,
+ b: &va, b_offset: g as usize * head_stride * dt,
+ b_leading_dimension: Some(HEAD_DIM), b_transpose: false,
+ ab_scale: 1.0, c: MatmulArgumentC::None, d: &mut grp_o,
+ batch_dim: GQA * SUFFIX, input_dim: SEQ, output_dim: HEAD_DIM,
+ },
+ &mut enc,
+ );
+ scatter_values.encode(
+ &grp_o, &mut out, g, GQA, SUFFIX, NUM_HEADS, HEAD_DIM,
+ GQA * SUFFIX * HEAD_DIM, &mut enc,
+ );
+ }
+ enc.end_encoding().submit().wait_until_completed().unwrap();
+ allocation_to_vec::<B, bf16>(&out)
+}
+
+#[uzu_test]
+fn test_head_dim_512_bf16() {
+ let (q, k, v) = make_inputs();
+ let scale = 1.0 / (HEAD_DIM as f32).sqrt();
+ let expected = reference(&q, &k, &v, scale);
+ for_each_non_cpu_backend!(|B| {
+ let output = pipeline_output::<B>(&q, &k, &v, scale);
+ assert_eq_float::<bf16>(&expected, &output, 1e-2, "AttentionFallback bf16");
+ });
+}
diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs
index ac8e23339..951c29f92 100644
--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs
@@ -187,8 +187,8 @@ fn test_gqa<T: ArrayElement + Float + Debug + Display>() {
test_internal(&input, &expected);
}
-fn test_head_dim_128<T: ArrayElement + Float + Debug + Display>() {
- let input = get_input::<T>(4, 4, 8, 2, 128, true);
+fn test_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {
+ let input = get_input::<T>(4, 4, 8, 2, head_dim, true);
let expected = get_output::<T, Cpu>(&input);
test_internal(&input, &expected);
}
@@ -244,15 +244,30 @@ fn test_gqa_bf16() {
// Head dim 128
#[uzu_test]
fn test_head_dim_128_f32() {
- test_head_dim_128::<f32>();
+ test_head_dim::<f32>(128);
}
#[uzu_test]
fn test_head_dim_128_f16() {
- test_head_dim_128::<f16>();
+ test_head_dim::<f16>(128);
}
#[uzu_test]
fn test_head_dim_128_bf16() {
- test_head_dim_128::<bf16>();
+ test_head_dim::<bf16>(128);
+}
+
+#[uzu_test]
+fn test_head_dim_512_f32() {
+ test_head_dim::<f32>(512);
+}
+
+#[uzu_test]
+fn test_head_dim_512_f16() {
+ test_head_dim::<f16>(512);
+}
+
+#[uzu_test]
+fn test_head_dim_512_bf16() {
+ test_head_dim::<bf16>(512);
}
diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs
index 64102c020..2f796c08d 100644
--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs
@@ -287,8 +287,8 @@ fn test_first_pass_gqa<T: ArrayElement + Float + Debug + Display>() {
test_first_pass_internal(&input, &expected);
}
-fn test_first_pass_head_dim_128<T: ArrayElement + Float + Debug + Display>() {
- let input = get_first_pass_input::<T>(4, 4, 8, 2, 128, true);
+fn test_first_pass_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {
+ let input = get_first_pass_input::<T>(4, 4, 8, 2, head_dim, true);
let expected = get_first_pass_output::<T, Cpu>(&input);
test_first_pass_internal(&input, &expected);
}
@@ -328,8 +328,8 @@ fn test_second_pass_basic<T: ArrayElement + Float + Debug + Display>() {
test_second_pass_internal::<T>(&input, &expected);
}
-fn test_second_pass_head_dim_128<T: ArrayElement + Float + Debug + Display>() {
- let input = get_second_pass_input(4, 2, 128);
+fn test_second_pass_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {
+ let input = get_second_pass_input(4, 2, head_dim);
let expected = get_second_pass_output::<T, Cpu>(&input);
test_second_pass_internal::<T>(&input, &expected);
}
@@ -383,17 +383,32 @@ fn test_first_pass_gqa_bf16() {
#[uzu_test]
fn test_first_pass_head_dim_128_f32() {
- test_first_pass_head_dim_128::<f32>();
+ test_first_pass_head_dim::<f32>(128);
}
#[uzu_test]
fn test_first_pass_head_dim_128_f16() {
- test_first_pass_head_dim_128::<f16>();
+ test_first_pass_head_dim::<f16>(128);
}
#[uzu_test]
fn test_first_pass_head_dim_128_bf16() {
- test_first_pass_head_dim_128::<bf16>();
+ test_first_pass_head_dim::<bf16>(128);
+}
+
+#[uzu_test]
+fn test_first_pass_head_dim_512_f32() {
+ test_first_pass_head_dim::<f32>(512);
+}
+
+#[uzu_test]
+fn test_first_pass_head_dim_512_f16() {
+ test_first_pass_head_dim::<f16>(512);
+}
+
+#[uzu_test]
+fn test_first_pass_head_dim_512_bf16() {
+ test_first_pass_head_dim::<bf16>(512);
}
// --- Second pass test entries ---
@@ -415,15 +430,30 @@ fn test_second_pass_basic_bf16() {
#[uzu_test]
fn test_second_pass_head_dim_128_f32() {
- test_second_pass_head_dim_128::<f32>();
+ test_second_pass_head_dim::<f32>(128);
}
#[uzu_test]
fn test_second_pass_head_dim_128_f16() {
- test_second_pass_head_dim_128::<f16>();
+ test_second_pass_head_dim::<f16>(128);
}
#[uzu_test]
fn test_second_pass_head_dim_128_bf16() {
- test_second_pass_head_dim_128::<bf16>();
+ test_second_pass_head_dim::<bf16>(128);
+}
+
+#[uzu_test]
+fn test_second_pass_head_dim_512_f32() {
+ test_second_pass_head_dim::<f32>(512);
+}
+
+#[uzu_test]
+fn test_second_pass_head_dim_512_f16() {
+ test_second_pass_head_dim::<f16>(512);
+}
+
+#[uzu_test]
+fn test_second_pass_head_dim_512_bf16() {
+ test_second_pass_head_dim::<bf16>(512);
}
diff --git a/crates/backend-uzu/tests/unit/kernel/attention/mod.rs b/crates/backend-uzu/tests/unit/kernel/attention/mod.rs
index 661eb7fb1..796bfc81e 100644
--- a/crates/backend-uzu/tests/unit/kernel/attention/mod.rs
+++ b/crates/backend-uzu/tests/unit/kernel/attention/mod.rs
@@ -1,3 +1,4 @@
+mod attention_fallback_test;
mod attention_gemm_test;
mod attention_single_pass_test;
mod attention_two_pass_test;
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
index 018aa3458..00204a6bd 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
@@ -98,6 +98,9 @@ fn get_cpu_output(input: &Input) -> Vec<bf16> {
a: &left_allocation,
a_offset: 0,
b: right_array.allocation(),
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
ab_scale: input.ab_scale,
c: argument_c,
d: &mut destination_allocation,
@@ -141,6 +144,9 @@ fn get_mpp_output(
a: &left_allocation,
a_offset: 0,
b: right_array.allocation(),
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
ab_scale: input.ab_scale,
c: argument_c,
d: &mut destination_allocation,
@@ -307,6 +313,9 @@ fn bench_gemm_mpp(criterion: &mut Criterion) {
a: &left_allocation,
a_offset: 0,
b: right_array.allocation(),
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
ab_scale: 1.0,
c: MatmulArgumentC::None,
d: &mut destination_allocation,
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs
index e1fa1aa86..3bd5359b6 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs
@@ -82,6 +82,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>, ab_scale: f
a: &a_allocation,
a_offset: 0,
b: b_array.allocation(),
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
ab_scale,
c: MatmulArgumentC::None,
d: &mut d_allocation,
@@ -222,6 +225,9 @@ fn bench_gemm(criterion: &mut Criterion) {
a: &left_allocation,
a_offset: 0,
b: right_array.allocation(),
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
ab_scale: 1.0,
c: MatmulArgumentC::None,
d: &mut destination_allocation,
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
index 351338060..d99850a4b 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
@@ -73,6 +73,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
a: &a_allocation,
a_offset: 0,
b: b_array.allocation(),
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
ab_scale: 1.0,
c: MatmulArgumentC::None,
d: &mut d_allocation,