Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Add AttentionGemmGrouped (#765)

CCYeh8af3f886d60e
diff --git a/crates/uzu-engine/src/backends/common/kernel/attention.rs b/crates/uzu-engine/src/backends/common/kernel/attention.rs
new file mode 100644
index 000000000..6311c8a29
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/kernel/attention.rs
@@ -0,0 +1,46 @@
+use crate::{
+ backends::common::{Allocation, Backend, BufferArg, Encoder, Kernels},
+ data_type::DataType,
+ encodable_block::mixer::attention::KVCacheView,
+};
+
+pub trait AttentionKernel: Sized + Send + Sync {
+ type Backend: Backend<Kernels: Kernels<AttentionKernel = Self>>;
+
+ fn new(
+ context: &<Self::Backend as Backend>::Context,
+ config: AttentionKernelConfig,
+ ) -> Result<Self, <Self::Backend as Backend>::Error>;
+
+ fn encode<'a, KT, VT>(
+ &self,
+ arguments: AttentionArguments<'a, Self::Backend, KT, VT>,
+ encoder: &mut Encoder<Self::Backend>,
+ ) -> Result<Allocation<Self::Backend>, <Self::Backend as Backend>::Error>
+ where
+ KT: BufferArg<'a, Self::Backend>,
+ VT: BufferArg<'a, Self::Backend>;
+}
+
+#[derive(Clone, Copy)]
+pub struct AttentionKernelConfig {
+ pub head_dim: u32,
+ pub num_groups: u32,
+ pub num_q_heads: u32,
+ pub has_sinks: bool,
+ pub is_kv_cache_ring: bool,
+ pub is_causal: bool,
+ pub sliding_window_size: Option<u32>,
+ pub scale: Option<f32>,
+ pub data_type: DataType,
+}
+
+pub struct AttentionArguments<'a, B: Backend, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>> {
+ pub queries: &'a Allocation<B>,
+ pub keys: KT,
+ pub values: VT,
+ pub suffix_length: u32,
+ pub trie: Option<&'a Allocation<B>>,
+ pub sinks: Option<&'a Allocation<B>>,
+ pub cache: KVCacheView,
+}
diff --git a/crates/uzu-engine/src/backends/common/kernel/attention_gemm/kernel.rs b/crates/uzu-engine/src/backends/common/kernel/attention_gemm/kernel.rs
deleted file mode 100644
index ce758b8f1..000000000
--- a/crates/uzu-engine/src/backends/common/kernel/attention_gemm/kernel.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-use crate::{
- backends::common::{
- Allocation, Backend, BufferArg, Encoder,
- kernel::{Kernels, Unsupported},
- },
- encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},
-};
-
-pub trait AttentionGemmCore: Sized + Send + Sync {
- type Backend: Backend<Kernels: Kernels<AttentionGemmCore = Self>>;
-
- fn is_supported(
- arguments: &AttentionCoreNewArguments,
- context: &<Self::Backend as Backend>::Context,
- ) -> Result<bool, <Self::Backend as Backend>::Error>;
-
- fn new(
- context: &<Self::Backend as Backend>::Context,
- arguments: &AttentionCoreNewArguments,
- ) -> Result<Self, <Self::Backend as Backend>::Error>;
-
- fn encode<'a, KT: BufferArg<'a, Self::Backend>, VT: BufferArg<'a, Self::Backend>>(
- &self,
- arguments: AttentionCoreEncodeArguments<'a, Self::Backend, KT, VT>,
- encoder: &mut Encoder<Self::Backend>,
- ) -> Result<Allocation<Self::Backend>, <Self::Backend as Backend>::Error>;
-}
-
-impl<B: Backend<Kernels: Kernels<AttentionGemmCore = Unsupported<B>>>> AttentionGemmCore for Unsupported<B> {
- type Backend = B;
-
- fn is_supported(
- _arguments: &AttentionCoreNewArguments,
- _context: &B::Context,
- ) -> Result<bool, B::Error> {
- Ok(false)
- }
-
- fn new(
- _context: &B::Context,
- _arguments: &AttentionCoreNewArguments,
- ) -> Result<Self, B::Error> {
- unreachable!("unsupported attention core should not be constructed")
- }
-
- fn encode<'a, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>>(
- &self,
- _arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,
- _encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, B::Error> {
- match self.never {}
- }
-}
diff --git a/crates/uzu-engine/src/backends/common/kernel/attention_gemm/mod.rs b/crates/uzu-engine/src/backends/common/kernel/attention_gemm/mod.rs
deleted file mode 100644
index 608e559cc..000000000
--- a/crates/uzu-engine/src/backends/common/kernel/attention_gemm/mod.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-mod kernel;
-
-pub use kernel::AttentionGemmCore;
diff --git a/crates/uzu-engine/src/backends/common/kernel/mod.rs b/crates/uzu-engine/src/backends/common/kernel/mod.rs
index ea0b5ea2e..365a80af5 100644
--- a/crates/uzu-engine/src/backends/common/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/mod.rs
@@ -3,7 +3,7 @@ use std::{convert::Infallible, marker::PhantomData};
use crate::backends::common::Backend;
pub mod activation_transform;
-pub mod attention_gemm;
+pub mod attention;
pub mod delta_net_chunked_prefill;
pub mod delta_net_tree_verify;
pub mod gated_act_mul;
@@ -11,6 +11,7 @@ pub mod matmul;
pub mod radix_top_k_small;
pub use activation_transform::ActivationTransform;
+pub use attention::{AttentionArguments, AttentionKernel, AttentionKernelConfig};
pub use gated_act_mul::{GatedActMul, GatedActMulSettings};
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
@@ -19,7 +20,7 @@ pub trait Kernels: Sized {
type Backend: Backend<Kernels = Self>;
autogen_kernels!();
- type AttentionGemmCore: attention_gemm::AttentionGemmCore<Backend = Self::Backend>;
+ type AttentionKernel: attention::AttentionKernel<Backend = Self::Backend>;
type DeltaNetChunkedPrefill: delta_net_chunked_prefill::DeltaNetChunkedPrefill<Backend = Self::Backend>;
type DeltaNetTreeVerify: delta_net_tree_verify::DeltaNetTreeVerify<Backend = Self::Backend>;
type MatmulKernel: matmul::MatmulKernel<Backend = Self::Backend>;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_fallback.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_fallback.rs
deleted file mode 100644
index 842e3b020..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_fallback.rs
+++ /dev/null
@@ -1,108 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::{
- array::ArrayElement,
- backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},
-};
-
-#[kernel(AttentionFallbackScatterScores)]
-#[variants(T, f32, f16, bf16)]
-pub fn attention_fallback_scatter_scores<T: ArrayElement + Float>(
- group_scores: *const T,
- scores: *mut T,
- #[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,
- #[optional(is_trie)] trie: Option<*const TrieNode>,
- #[optional(is_sliding_window)] sliding_window_size: Option<u32>,
- group_index: u32,
- gqa_factor: u32,
- sequence_length: u32,
- suffix_length: u32,
- total_elements: u32,
- #[specialize] is_kv_cache_ring: bool,
- #[specialize] is_causal: bool,
- #[specialize] is_trie: bool,
- #[specialize] is_sliding_window: bool,
-) {
- assert_eq!(ring_params.is_some(), is_kv_cache_ring);
- assert_eq!(sliding_window_size.is_some(), is_sliding_window);
- let expected_total_elements = gqa_factor
- .checked_mul(suffix_length)
- .and_then(|elements| elements.checked_mul(sequence_length))
- .expect("scatter scores element count overflow");
- assert_eq!(total_elements, expected_total_elements);
-
- let prefix_length = sequence_length - suffix_length;
- let suffix_position = ring_params.map(|params| params.ring_length).unwrap_or(prefix_length);
-
- for head_in_group in 0..gqa_factor {
- let head_index = group_index * gqa_factor + head_in_group;
- for query_index in 0..suffix_length {
- let query_position = if is_trie {
- let trie_node = unsafe { &*trie.unwrap().add(query_index as usize) };
- suffix_position + trie_node.height
- } else {
- suffix_position + query_index
- };
-
- for sequence_index in 0..sequence_length {
- let source_index =
- ((head_in_group * suffix_length + query_index) * sequence_length + sequence_index) as usize;
- let destination_index =
- ((head_index * suffix_length + query_index) * sequence_length + sequence_index) as usize;
- let use_key = should_use_key(
- ring_params,
- trie,
- sliding_window_size,
- query_index,
- prefix_length,
- suffix_position,
- query_position,
- sequence_index,
- is_causal,
- );
- unsafe {
- *scores.add(destination_index) = if use_key {
- *group_scores.add(source_index)
- } else {
- T::from(f32::NEG_INFINITY).unwrap()
- };
- }
- }
- }
- }
-}
-
-#[kernel(AttentionFallbackScatterValues)]
-#[variants(T, f32, f16, bf16)]
-pub fn attention_fallback_scatter_values<T: ArrayElement + Float>(
- group_output: *const T,
- out: *mut T,
- group_index: u32,
- gqa_factor: u32,
- suffix_length: u32,
- num_heads: u32,
- head_dim: u32,
- total_elements: u32,
-) {
- let expected_total_elements = gqa_factor
- .checked_mul(suffix_length)
- .and_then(|elements| elements.checked_mul(head_dim))
- .expect("scatter values element count overflow");
- assert_eq!(total_elements, expected_total_elements);
-
- for head_in_group in 0..gqa_factor as usize {
- for query_index in 0..suffix_length as usize {
- for dim_index in 0..head_dim as usize {
- let source_index =
- (head_in_group * suffix_length as usize + query_index) * head_dim as usize + dim_index;
- let head_index = group_index as usize * gqa_factor as usize + head_in_group;
- let destination_index = (query_index * num_heads as usize + head_index) * head_dim as usize + dim_index;
- unsafe {
- *out.add(destination_index) = *group_output.add(source_index);
- }
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs
index 75eb1a955..dea58b22b 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs
@@ -1,4 +1,4 @@
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
@@ -8,7 +8,7 @@ use crate::{
};
#[kernel(AttentionSinglePass)]
-#[variants(T, f32, f16, bf16)]
+#[variants(T, f32, bf16)]
#[variants(HEAD_DIM, 64, 128, 256, 512)]
pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(
queries: *const T,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_two_pass.rs
deleted file mode 100644
index e696d7cdc..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_two_pass.rs
+++ /dev/null
@@ -1,190 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::{
- array::ArrayElement,
- backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},
-};
-
-const TOTAL_BLOCKS_COUNT: u32 = 32;
-
-#[kernel(AttentionTwoPass1)]
-#[variants(T, f32, f16, bf16)]
-#[variants(HEAD_DIM, 64, 128, 256, 512)]
-pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(
- queries: *const T,
- keys: *const T,
- values: *const T,
- out: *mut f32,
- sums: *mut f32,
- maxs: *mut f32,
- gqa_factor: u32,
- sequence_length: u32,
- k_head_stride: u32,
- k_seq_stride: u32,
- v_head_stride: u32,
- v_seq_stride: u32,
- #[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,
- scale: f32,
- num_heads: u32,
- suffix_length: u32,
- #[optional(is_trie)] trie: Option<*const TrieNode>,
- #[optional(is_sliding_window)] sliding_window_size: Option<u32>,
- #[optional(has_sinks)] sinks: Option<*const T>,
- #[specialize] has_sinks: bool,
- #[specialize] is_kv_cache_ring: bool,
- #[specialize] is_causal: bool,
- #[specialize] is_trie: bool,
- #[specialize] is_sliding_window: bool,
-) {
- assert_eq!(ring_params.is_some(), is_kv_cache_ring);
- assert_eq!(sliding_window_size.is_some(), is_sliding_window);
-
- let value_dim = HEAD_DIM;
- let mut q = vec![0.0f32; HEAD_DIM as usize];
- let mut o = vec![0.0f32; HEAD_DIM as usize];
-
- let prefix_length = sequence_length - suffix_length;
- let suffix_position = if let Some(ring_params) = ring_params {
- ring_params.ring_length
- } else {
- prefix_length
- };
-
- for head_idx in 0..num_heads {
- for q_seq_idx in 0..suffix_length {
- let query_position = if is_trie {
- let trie_node = unsafe { &*trie.unwrap().add(q_seq_idx as usize) };
- suffix_position + trie_node.height
- } else {
- suffix_position + q_seq_idx
- };
-
- for block_idx in 0..TOTAL_BLOCKS_COUNT {
- let o_offset = q_seq_idx * num_heads + head_idx;
- let q_offset = head_idx * suffix_length + q_seq_idx;
- let kv_head_idx = head_idx / gqa_factor;
-
- let queries_base: *const T = unsafe { queries.add((q_offset * HEAD_DIM) as usize) };
- let keys_base: *const T = unsafe { keys.add((kv_head_idx * k_head_stride) as usize) };
- let values_base: *const T = unsafe { values.add((kv_head_idx * v_head_stride) as usize) };
- let out_base: *mut f32 =
- unsafe { out.add((o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim) as usize) };
-
- // Read the query and scale
- for j in 0..HEAD_DIM as usize {
- q[j] = scale * unsafe { *queries_base.add(j) }.to_f32().unwrap();
- }
- o.fill(0f32);
-
- let mut max_score = -1e9f32;
- let mut sum_exp_score = 0.0f32;
-
- if has_sinks && block_idx == 0 {
- max_score = unsafe { *sinks.unwrap().add(head_idx as usize) }.to_f32().unwrap();
- sum_exp_score = 1.0;
- }
-
- // Iterate over sequence positions assigned to this block
- let mut i = block_idx;
- while i < sequence_length {
- if should_use_key(
- ring_params,
- trie,
- sliding_window_size,
- q_seq_idx,
- prefix_length,
- suffix_position,
- query_position,
- i,
- is_causal,
- ) {
- let keys_ptr = unsafe { keys_base.add((i * k_seq_stride) as usize) };
-
- // Compute the i-th score
- let mut score = 0.0f32;
- for j in 0..HEAD_DIM as usize {
- score += q[j] * unsafe { *keys_ptr.add(j) }.to_f32().unwrap();
- }
-
- // Update the accumulators
- let new_max = f32::max(max_score, score);
- let factor = (max_score - new_max).exp();
- let exp_score = (score - new_max).exp();
-
- max_score = new_max;
- sum_exp_score = sum_exp_score * factor + exp_score;
-
- // Update the output accumulator
- let values_ptr = unsafe { values_base.add((i * v_seq_stride) as usize) };
- for j in 0..HEAD_DIM as usize {
- o[j] = o[j] * factor + exp_score * unsafe { *values_ptr.add(j) }.to_f32().unwrap();
- }
- }
- i += TOTAL_BLOCKS_COUNT;
- }
-
- // Write partial output, sum, and max for this block
- for j in 0..HEAD_DIM as usize {
- unsafe {
- *out_base.add(j) = o[j];
- }
- }
- unsafe {
- *sums.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) = sum_exp_score;
- *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) = max_score;
- }
- }
- }
- }
-}
-
-#[kernel(AttentionTwoPass2)]
-#[variants(T, f32, f16, bf16)]
-#[variants(HEAD_DIM, 64, 128, 256, 512)]
-pub fn attention_two_pass2<T: ArrayElement + Float, const HEAD_DIM: u32>(
- partials: *const f32,
- sums: *const f32,
- maxs: *const f32,
- out: *mut T,
- num_heads: u32,
- suffix_length: u32,
-) {
- for head_idx in 0..num_heads {
- for q_seq_idx in 0..suffix_length {
- let o_offset = q_seq_idx * num_heads + head_idx;
-
- // Find global max across all blocks
- let mut global_max = f32::NEG_INFINITY;
- for block_idx in 0..TOTAL_BLOCKS_COUNT {
- let max_val = unsafe { *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };
- global_max = f32::max(global_max, max_val);
- }
-
- // Compute global sum of exponentials with correction
- let mut global_sum = 0.0f32;
- for block_idx in 0..TOTAL_BLOCKS_COUNT {
- let sum_val = unsafe { *sums.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };
- let max_val = unsafe { *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };
- global_sum += sum_val * (max_val - global_max).exp();
- }
-
- // Combine partial outputs
- let out_ptr = unsafe { out.add((o_offset * HEAD_DIM) as usize) };
- for j in 0..HEAD_DIM as usize {
- let mut val = 0.0f32;
- for block_idx in 0..TOTAL_BLOCKS_COUNT {
- let partial_val = unsafe {
- *partials.add((o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM + block_idx * HEAD_DIM) as usize + j)
- };
- let max_val = unsafe { *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };
- val += partial_val * (max_val - global_max).exp();
- }
- unsafe {
- *out_ptr.add(j) = T::from(val / global_sum).unwrap();
- }
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs
index 026a4486b..736c515c3 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs
@@ -1,11 +1,11 @@
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
use crate::array::ArrayElement;
#[kernel(KVCacheUpdate)]
-#[variants(T, f32, bf16, f16)]
+#[variants(T, f32, bf16)]
pub fn kv_cache_update<T: ArrayElement + Float>(
in_place_keys: *mut T,
in_place_values: *mut T,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
index 052bec54a..dc9d18668 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs
@@ -1,10 +1,83 @@
pub mod ancestor_attention;
-pub mod attention_fallback;
pub mod attention_prepare;
pub mod attention_single_pass;
-pub mod attention_two_pass;
pub mod kv_cache_update;
pub mod qkv_norm;
pub mod sigmoid_gate;
mod mask;
+
+use crate::backends::{
+ common::{
+ Allocation, BufferArg, Encoder,
+ kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig, AttentionSinglePassKernel},
+ },
+ cpu::{Cpu, context::CpuContext, error::CpuError},
+};
+
+pub struct AttentionCpuKernel {
+ config: AttentionKernelConfig,
+}
+
+impl AttentionKernel for AttentionCpuKernel {
+ type Backend = Cpu;
+
+ fn new(
+ _context: &CpuContext,
+ config: AttentionKernelConfig,
+ ) -> Result<Self, CpuError> {
+ Ok(Self {
+ config,
+ })
+ }
+
+ fn encode<'a, KT, VT>(
+ &self,
+ arguments: AttentionArguments<'a, Cpu, KT, VT>,
+ encoder: &mut Encoder<Cpu>,
+ ) -> Result<Allocation<Cpu>, CpuError>
+ where
+ KT: BufferArg<'a, Cpu>,
+ VT: BufferArg<'a, Cpu>,
+ {
+ let config = self.config;
+ let mut output = encoder.allocate_constant_for_shape(
+ &[arguments.suffix_length, config.num_q_heads, config.head_dim],
+ config.data_type,
+ )?;
+
+ let single_pass = <attention_single_pass::AttentionSinglePassCpuKernel as AttentionSinglePassKernel>::new(
+ encoder.context(),
+ config.data_type,
+ config.head_dim,
+ config.has_sinks,
+ config.is_kv_cache_ring,
+ config.is_causal,
+ arguments.trie.is_some(),
+ config.sliding_window_size.is_some(),
+ )?;
+ encoder.push_debug_group("attention core");
+ single_pass.encode(
+ arguments.queries,
+ arguments.keys,
+ arguments.values,
+ &mut output,
+ config.num_q_heads / config.num_groups,
+ arguments.cache.prefix_len() + arguments.suffix_length,
+ config.head_dim,
+ config.num_groups * config.head_dim,
+ config.head_dim,
+ config.num_groups * config.head_dim,
+ arguments.cache.ring_params(),
+ config.scale.unwrap_or(1.0 / (config.head_dim as f32).sqrt()),
+ arguments.trie,
+ config.sliding_window_size,
+ arguments.sinks,
+ config.num_q_heads,
+ arguments.suffix_length,
+ encoder,
+ );
+ encoder.pop_debug_group();
+ Ok(output)
+ }
+}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs
index 4a37c2061..007fd160a 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs
@@ -1,14 +1,14 @@
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
use crate::array::ArrayElement;
#[kernel(QKVNorm)]
-#[variants(InputT, f32, f16, bf16)]
-#[variants(ScaleT, f32, f16, bf16)]
-#[variants(OutputT, f32, f16, bf16)]
-#[variants(AccumT, f32, f16)]
+#[variants(InputT, f32, bf16)]
+#[variants(ScaleT, f32, bf16)]
+#[variants(OutputT, f32, bf16)]
+#[variants(AccumT, f32)]
pub fn qkv_norm<
InputT: ArrayElement + Float,
ScaleT: ArrayElement + Float,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs
index 6bf4883fc..dd3a07e25 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs
@@ -1,11 +1,11 @@
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use uzu_engine_macros::kernel;
use crate::array::ArrayElement;
#[kernel(SigmoidGate)]
-#[variants(T, f32, f16, bf16)]
+#[variants(T, f32, bf16)]
pub fn sigmoid_gate<T: ArrayElement + Float>(
gate: *const T,
output: *mut T,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
index fe72506a6..5cbd2a504 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
@@ -33,7 +33,7 @@ impl Kernels for CpuKernels {
type Backend = Cpu;
autogen_kernels!();
- type AttentionGemmCore = Unsupported<Cpu>;
+ type AttentionKernel = attention::AttentionCpuKernel;
type DeltaNetChunkedPrefill = Unsupported<Cpu>;
type DeltaNetTreeVerify = Unsupported<Cpu>;
type MatmulKernel = matmul::MatmulCpuKernel;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal
index 14e2e12b2..dbb45dfe0 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal
@@ -11,8 +11,8 @@ using namespace uzu::ring;
using namespace uzu::trie;
template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AttentionFallbackScatterScores)(
+VARIANTS(T, float, bfloat)
+KERNEL(AttentionFallbackScatterScores)(
const device T* group_scores,
device T* scores,
const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),
@@ -59,8 +59,8 @@ PUBLIC KERNEL(AttentionFallbackScatterScores)(
}
template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AttentionFallbackScatterValues)(
+VARIANTS(T, float, bfloat)
+KERNEL(AttentionFallbackScatterValues)(
const device T* group_output,
device T* out,
const constant uint& group_index,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal
index 3010a678a..b31d7b054 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -37,7 +37,7 @@ struct AttentionGemmLayout {
};
template <typename T, uint BK, uint BD, bool USE_MXU>
-VARIANTS(T, float, half, bfloat)
+VARIANTS(T, float, bfloat)
VARIANTS(BK, 16, 32)
VARIANTS(BD, 64, 128, 256)
VARIANTS(USE_MXU, false, true)
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal
index 923a9a135..42869cc1d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -13,7 +13,7 @@ using namespace uzu::ring;
using namespace uzu::trie;
template <typename T, uint HEAD_DIM>
-VARIANTS(T, float, half, bfloat)
+VARIANTS(T, float, bfloat)
VARIANTS(HEAD_DIM, 64, 128, 256, 512)
PUBLIC KERNEL(AttentionSinglePass)(
const device T* queries,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal
index a88f9433e..3594bcbce 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -18,9 +18,9 @@ using namespace uzu::trie;
// Each simdgroup independently handles one query head and derives its KV head
// via integer division. Simdgroups sharing a KV head benefit from L1 cache.
template <typename T, uint HEAD_DIM>
-VARIANTS(T, float, half, bfloat)
+VARIANTS(T, float, bfloat)
VARIANTS(HEAD_DIM, 64, 128, 256, 512)
-PUBLIC KERNEL(AttentionTwoPass1)(
+KERNEL(AttentionTwoPass1)(
const device T* queries,
const device T* keys,
const device T* values,
@@ -139,9 +139,9 @@ PUBLIC KERNEL(AttentionTwoPass1)(
}
template <typename T, uint HEAD_DIM>
-VARIANTS(T, float, half, bfloat)
+VARIANTS(T, float, bfloat)
VARIANTS(HEAD_DIM, 64, 128, 256, 512)
-PUBLIC KERNEL(AttentionTwoPass2)(
+KERNEL(AttentionTwoPass2)(
const device float* partials,
const device float* sums,
const device float* maxs,
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
similarity index 62%
rename from crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rs
rename to crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
index bb97d7057..589d67806 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
@@ -1,62 +1,77 @@
use parking_lot::Mutex;
use crate::{
- backends::common::{
- Allocation, Backend, BufferArg, Encoder, Kernels,
- kernel::{
- AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, SoftmaxKernel,
- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
+ backends::{
+ common::{
+ Allocation, BufferArg, Encoder, Kernels,
+ kernel::{
+ AttentionArguments, AttentionKernelConfig, SoftmaxKernel,
+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
+ },
+ },
+ metal::{
+ Metal,
+ context::MetalContext,
+ error::MetalError,
+ kernel::{
+ AttentionFallbackScatterScoresMetalKernel, AttentionFallbackScatterValuesMetalKernel, MetalKernels,
+ },
},
},
data_type::DataType,
- encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},
};
-pub struct AttentionFallbackCore<B: Backend> {
+const HEAD_DIM: u32 = 512;
+
+pub struct AttentionFallback {
head_dim: u32,
num_groups: u32,
num_q_heads: u32,
sliding_window_size: Option<u32>,
scale: Option<f32>,
data_type: DataType,
- scatter_scores: <B::Kernels as Kernels>::AttentionFallbackScatterScoresKernel,
- scatter_values: <B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel,
- softmax: <B::Kernels as Kernels>::SoftmaxKernel,
- matmul: Mutex<<B::Kernels as Kernels>::MatmulKernel>,
+ scatter_scores: AttentionFallbackScatterScoresMetalKernel,
+ scatter_values: AttentionFallbackScatterValuesMetalKernel,
+ softmax: <MetalKernels as Kernels>::SoftmaxKernel,
+ matmul: Mutex<<MetalKernels as Kernels>::MatmulKernel>,
}
-impl<B: Backend> AttentionFallbackCore<B> {
- pub fn new(
- arguments: &AttentionCoreNewArguments,
- context: &B::Context,
- ) -> Result<Self, B::Error> {
- assert!(!arguments.is_trie, "trie not supported by attention fallback"); // Is it?
+impl AttentionFallback {
+ pub fn is_supported(config: &AttentionKernelConfig) -> bool {
+ config.head_dim == HEAD_DIM && matches!(config.data_type, DataType::BF16 | DataType::F32)
+ }
- let scatter_scores = <B::Kernels as Kernels>::AttentionFallbackScatterScoresKernel::new(
+ pub fn new(
+ config: &AttentionKernelConfig,
+ context: &MetalContext,
+ ) -> Result<Self, MetalError> {
+ let scatter_scores = AttentionFallbackScatterScoresMetalKernel::new(
+ context,
+ config.data_type,
+ config.is_kv_cache_ring,
+ config.is_causal,
+ false,
+ config.sliding_window_size.is_some(),
+ )?;
+ let scatter_values = AttentionFallbackScatterValuesMetalKernel::new(context, config.data_type)?;
+ let softmax = <<MetalKernels as Kernels>::SoftmaxKernel as SoftmaxKernel>::new(
context,
- arguments.data_type,
- arguments.is_kv_cache_ring,
- arguments.is_causal,
- arguments.is_trie,
- arguments.sliding_window_size.is_some(),
+ config.data_type,
+ config.has_sinks,
)?;
- let scatter_values =
- <B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(context, arguments.data_type)?;
- let softmax = <B::Kernels as Kernels>::SoftmaxKernel::new(context, arguments.data_type, arguments.has_sinks)?;
- let matmul = Mutex::new(<B::Kernels as Kernels>::MatmulKernel::new(
+ let matmul = Mutex::new(<<MetalKernels as Kernels>::MatmulKernel as MatmulKernel>::new(
context,
- arguments.data_type,
- arguments.data_type,
- arguments.data_type,
+ config.data_type,
+ config.data_type,
+ config.data_type,
)?);
-
Ok(Self {
- head_dim: arguments.head_dim,
- num_groups: arguments.num_groups,
- num_q_heads: arguments.num_q_heads,
- sliding_window_size: arguments.sliding_window_size,
- scale: arguments.scale,
- data_type: arguments.data_type,
+ head_dim: config.head_dim,
+ num_groups: config.num_groups,
+ num_q_heads: config.num_q_heads,
+ sliding_window_size: config.sliding_window_size,
+ scale: config.scale,
+ data_type: config.data_type,
scatter_scores,
scatter_values,
softmax,
@@ -64,20 +79,19 @@ impl<B: Backend> AttentionFallbackCore<B> {
})
}
- pub fn encode<'a, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>>(
+ pub fn encode<'a, KT: BufferArg<'a, Metal>, VT: BufferArg<'a, Metal>>(
&self,
- arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,
- encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, B::Error> {
+ arguments: AttentionArguments<'a, Metal, KT, VT>,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<Allocation<Metal>, MetalError> {
+ assert!(arguments.trie.is_none(), "fallback does not support trie");
let suffix_length = arguments.suffix_length;
let sequence_length = arguments.cache.prefix_len() + suffix_length;
let gqa_factor = self.num_q_heads / self.num_groups;
- let scale = self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt());
-
+ let scale = self.scale.unwrap_or(1.0 / (self.head_dim as f32).sqrt());
let dt_bytes = self.data_type.size_in_bytes();
let head_dim_bytes = self.head_dim as usize * dt_bytes;
let group_rows = (gqa_factor * suffix_length) as usize;
-
let mut output =
encoder.allocate_constant_for_shape(&[suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;
let mut scores =
@@ -113,7 +127,7 @@ impl<B: Backend> AttentionFallbackCore<B> {
&group_scores,
&mut scores,
arguments.cache.ring_params(),
- None::<&Allocation<B>>,
+ None::<&Allocation<Metal>>,
self.sliding_window_size,
group_index,
gqa_factor,
@@ -125,10 +139,8 @@ impl<B: Backend> AttentionFallbackCore<B> {
}
self.softmax.encode(&mut scores, arguments.sinks, sequence_length, self.num_q_heads, suffix_length, encoder);
-
let mut group_output =
encoder.allocate_scratch_for_shape(&[gqa_factor * suffix_length, self.head_dim], self.data_type)?;
-
for group_index in 0..self.num_groups {
self.matmul.lock().encode(
MatmulArguments {
@@ -162,7 +174,6 @@ impl<B: Backend> AttentionFallbackCore<B> {
encoder,
);
}
-
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs
similarity index 71%
rename from crates/uzu-engine/src/backends/metal/kernel/attention.rs
rename to crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs
index dfb95d2b6..3501e1a6d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs
@@ -4,14 +4,17 @@ use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};
use crate::{
backends::{
- common::{Allocation, BufferArg, Encoder, gpu_types::AttnParams, kernel::attention_gemm::AttentionGemmCore},
+ common::{
+ Allocation, BufferArg, Encoder,
+ gpu_types::AttnParams,
+ kernel::{AttentionArguments, AttentionKernelConfig},
+ },
metal::{Metal, context::MetalContext, error::MetalError, kernel::AttentionGemmMetalKernel},
},
data_type::DataType,
- encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},
};
-pub struct AttentionGemmMetalCore {
+pub struct AttentionGemm {
kernels: Mutex<HashMap<AttentionGemmKey, AttentionGemmMetalKernel>>,
head_dim: u32,
num_groups: u32,
@@ -22,7 +25,6 @@ pub struct AttentionGemmMetalCore {
simd_bk: u32,
is_kv_cache_ring: bool,
is_causal: bool,
- is_trie: bool,
is_sliding_window: bool,
has_sinks: bool,
}
@@ -32,6 +34,7 @@ struct AttentionGemmKey {
use_mxu: bool,
align_q: bool,
align_k: bool,
+ is_trie: bool,
}
fn retile_params(
@@ -47,7 +50,33 @@ fn retile_params(
params
}
-impl AttentionGemmMetalCore {
+impl AttentionGemm {
+ pub fn is_supported(config: &AttentionKernelConfig) -> bool {
+ matches!(config.head_dim, 64 | 128 | 256) && matches!(config.data_type, DataType::BF16 | DataType::F32)
+ }
+
+ pub fn new(config: &AttentionKernelConfig) -> Self {
+ let simd_bk = if config.head_dim < 128 {
+ 32
+ } else {
+ 16
+ };
+ Self {
+ kernels: Mutex::new(HashMap::new()),
+ head_dim: config.head_dim,
+ num_groups: config.num_groups,
+ num_q_heads: config.num_q_heads,
+ sliding_window_size: config.sliding_window_size,
+ scale: config.scale,
+ data_type: config.data_type,
+ simd_bk,
+ is_kv_cache_ring: config.is_kv_cache_ring,
+ is_causal: config.is_causal,
+ is_sliding_window: config.sliding_window_size.is_some(),
+ has_sinks: config.has_sinks,
+ }
+ }
+
fn get_or_create(
&self,
context: &MetalContext,
@@ -70,7 +99,7 @@ impl AttentionGemmMetalCore {
key.align_k,
self.is_kv_cache_ring,
self.is_causal,
- self.is_trie,
+ key.is_trie,
self.is_sliding_window,
self.has_sinks,
)?;
@@ -78,48 +107,10 @@ impl AttentionGemmMetalCore {
}
Ok(MutexGuard::map(kernels, |kernels| kernels.get_mut(&key).expect("kernel was just initialized")))
}
-}
-impl AttentionGemmCore for AttentionGemmMetalCore {
- type Backend = Metal;
-
- fn is_supported(
- arguments: &AttentionCoreNewArguments,
- _context: &MetalContext,
- ) -> Result<bool, MetalError> {
- Ok(matches!(arguments.head_dim, 64 | 128 | 256))
- }
-
- fn new(
- _context: &MetalContext,
- arguments: &AttentionCoreNewArguments,
- ) -> Result<Self, MetalError> {
- let simd_bk = if arguments.head_dim < 128 {
- 32
- } else {
- 16
- };
-
- Ok(Self {
- kernels: Mutex::new(HashMap::new()),
- head_dim: arguments.head_dim,
- num_groups: arguments.num_groups,
- num_q_heads: arguments.num_q_heads,
- sliding_window_size: arguments.sliding_window_size,
- scale: arguments.scale,
- data_type: arguments.data_type,
- simd_bk,
- is_kv_cache_ring: arguments.is_kv_cache_ring,
- is_causal: arguments.is_causal,
- is_trie: arguments.is_trie,
- is_sliding_window: arguments.sliding_window_size.is_some(),
- has_sinks: arguments.has_sinks,
- })
- }
-
- fn encode<'a, KT: BufferArg<'a, Metal>, VT: BufferArg<'a, Metal>>(
+ pub fn encode<'a, KT: BufferArg<'a, Metal>, VT: BufferArg<'a, Metal>>(
&self,
- arguments: AttentionCoreEncodeArguments<'a, Metal, KT, VT>,
+ arguments: AttentionArguments<'a, Metal, KT, VT>,
encoder: &mut Encoder<Metal>,
) -> Result<Allocation<Metal>, MetalError> {
let mut output = encoder
@@ -127,7 +118,7 @@ impl AttentionGemmCore for AttentionGemmMetalCore {
let use_mxu = arguments.suffix_length >= 64
&& encoder.context().supports_mxu()
- && matches!(self.data_type, DataType::BF16 | DataType::F16)
+ && matches!(self.data_type, DataType::BF16)
&& matches!(self.head_dim, 64 | 128);
let (bq, bk) = if use_mxu {
(64, 32)
@@ -141,7 +132,7 @@ impl AttentionGemmCore for AttentionGemmMetalCore {
v_strides: [0, self.head_dim, self.num_groups * self.head_dim],
o_strides: [0, self.head_dim, self.num_q_heads * self.head_dim],
gqa_factor: self.num_q_heads / self.num_groups,
- scale: self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),
+ scale: self.scale.unwrap_or(1.0 / (self.head_dim as f32).sqrt()),
q_len: arguments.suffix_length,
k_len: arguments.cache.prefix_len() + arguments.suffix_length,
q_off: arguments.cache.prefix_len(),
@@ -158,9 +149,9 @@ impl AttentionGemmCore for AttentionGemmMetalCore {
use_mxu,
align_q: params.q_rem == 0,
align_k: params.k_rem == 0,
+ is_trie: arguments.trie.is_some(),
};
let kernel = self.get_or_create(encoder.context(), key)?;
-
kernel.encode(
arguments.queries,
arguments.keys,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metal
new file mode 100644
index 000000000..8478518d5
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metal
@@ -0,0 +1,452 @@
+#include <metal_stdlib>
+#include "../../common/dsl.h"
+#include "../../common/integral_constant.h"
+#include "../../common/thread_context.h"
+#include "../../matmul/common/fragment.h"
+#include "../../matmul/common/loader.h"
+#include "../../matmul/common/mxu_fragment/ops.h"
+#include "../../generated/attention.h"
+#include "../../generated/trie.h"
+
+using namespace metal;
+using namespace uzu::matmul;
+using namespace uzu::attention;
+using namespace uzu::trie;
+
+#define ATTENTION_GQA_SIMDGROUPS 8
+
+template <typename T, uint BK, uint BD, uint D_SLICES>
+struct AttentionGemmGroupedLayout {
+ using Ops = MxuFragmentOps<>;
+
+ UZU_CONST int SIMDGROUPS = ATTENTION_GQA_SIMDGROUPS;
+ UZU_CONST int TILE_ROWS = Ops::FRAGMENT_ROWS;
+ UZU_CONST int SLICE_COLS = int(BD) / int(D_SLICES);
+ UZU_CONST int BANDS = SIMDGROUPS / int(D_SLICES);
+ UZU_CONST int BLOCK_ROWS = BANDS * TILE_ROWS;
+ UZU_CONST int SCORE_TILE_ELEMENTS = TILE_ROWS * int(BK);
+ UZU_CONST int SCORE_EXCHANGE_SIZE = SIMDGROUPS * SCORE_TILE_ELEMENTS;
+};
+
+template <class FragmentType, class Fn>
+METAL_FUNC void map_lane_rows(thread FragmentType& tile, const short2 position, Fn fn) {
+ using Ops = typename FragmentType::FragmentOpsType;
+ using Element = typename FragmentType::ElementType;
+ static_assert(FragmentType::ROW_FRAGMENTS == 1, "requires one row fragment");
+ static_assert(
+ Ops::THREAD_ELEMENT_ROWS * Ops::THREAD_ELEMENT_COLS == Ops::ELEMENTS_PER_THREAD,
+ "requires row-major thread elements"
+ );
+
+ thread Element* data = tile.elements();
+ METAL_PRAGMA_UNROLL
+ for (ushort fragment_col = 0; fragment_col < FragmentType::COL_FRAGMENTS; ++fragment_col) {
+ const ushort fragment_base = fragment_col * Ops::ELEMENTS_PER_THREAD;
+ const short col_base = position.x + short(fragment_col) * short(Ops::FRAGMENT_COLS);
+ METAL_PRAGMA_UNROLL
+ for (ushort row = 0; row < Ops::THREAD_ELEMENT_ROWS; ++row) {
+ METAL_PRAGMA_UNROLL
+ for (ushort col = 0; col < Ops::THREAD_ELEMENT_COLS; ++col) {
+ const ushort element = fragment_base + row * Ops::THREAD_ELEMENT_COLS + col;
+ data[element] = Element(fn(row, col_base + short(col), data[element]));
+ }
+ }
+ }
+}
+
+// TODO: support more BD and should tune more to replace AttentionGEMM
+template <typename T, uint BK, uint BD, uint D_SLICES>
+VARIANTS(T, bfloat)
+VARIANTS(BK, 32)
+VARIANTS(BD, 128, 256)
+VARIANTS(D_SLICES, 2, 4)
+CONSTRAINT(D_SLICES == BD / 64)
+KERNEL(AttentionGemmGrouped)(
+ const device T* q,
+ const device T* k,
+ const device T* v,
+ device T* o OPTIONAL(!split_kv),
+ device float* partials OPTIONAL(split_kv),
+ device float* partial_maxs OPTIONAL(split_kv),
+ device float* partial_sums OPTIONAL(split_kv),
+ const device TrieNode* trie OPTIONAL(is_trie),
+ const constant AttnParams& params,
+ const constant uint& m_tiles,
+ const constant uint& kv_heads,
+ const constant uint& batch_count,
+ const constant uint& num_splits,
+ const bool split_kv SPECIALIZE,
+ const bool is_causal SPECIALIZE,
+ const bool is_trie SPECIALIZE,
+ threadgroup float score_exchange[AttentionGemmGroupedLayout<T, BK, BD, D_SLICES>::SCORE_EXCHANGE_SIZE],
+ const ThreadContext thread_context,
+ const uint tile_split_flat GROUPS(m_tiles * num_splits),
+ const uint kv_head_idx GROUPS(kv_heads),
+ const uint batch_idx GROUPS(batch_count),
+ const uint lid THREADS(ATTENTION_GQA_SIMDGROUPS * METAL_SIMD_SIZE)
+) {
+ using Layout = AttentionGemmGroupedLayout<T, BK, BD, D_SLICES>;
+ using Ops = typename Layout::Ops;
+ using AccumType = float;
+
+ constexpr int TILE_ROWS = Layout::TILE_ROWS;
+ constexpr int SLICE_COLS = Layout::SLICE_COLS;
+ constexpr int BLOCK_ROWS = Layout::BLOCK_ROWS;
+ constexpr int SCORE_TILE_ELEMENTS = Layout::SCORE_TILE_ELEMENTS;
+ constexpr int KEY_COL_FRAGMENTS = int(BK) / Ops::FRAGMENT_COLS;
+ constexpr int HEAD_DIM_FRAGMENTS_PER_SLICE = SLICE_COLS / Ops::FRAGMENT_ROWS;
+ constexpr int VALUE_COL_FRAGMENTS = SLICE_COLS / Ops::FRAGMENT_COLS;
+ constexpr int ROWS_PER_LANE = Ops::THREAD_ELEMENT_ROWS;
+
+ static_assert(Layout::SIMDGROUPS == 8, "requires eight simdgroups");
+ static_assert(KEY_COL_FRAGMENTS % 2 == 0, "QK fragments must be even");
+ static_assert(VALUE_COL_FRAGMENTS % 2 == 0, "PV fragments must be even");
+
+ using QueryFragment = OperandFragment<T, 1, 1, Ops>;
+ using KeyFragment = OperandFragment<T, 1, KEY_COL_FRAGMENTS, Ops, ReadTranspose>;
+ using ValueFragment = OperandFragment<T, KEY_COL_FRAGMENTS, VALUE_COL_FRAGMENTS, Ops>;
+ using ScoreFragment = Fragment<AccumType, 1, KEY_COL_FRAGMENTS, Ops>;
+ using OutputFragment = Fragment<AccumType, 1, VALUE_COL_FRAGMENTS, Ops>;
+
+ const ushort lane = ushort(thread_context.simd_lane_id);
+ const ushort simdgroup_index = ushort(thread_context.simdgroup_index);
+ const ushort band = simdgroup_index / ushort(D_SLICES);
+ const ushort slice = simdgroup_index % ushort(D_SLICES);
+
+ const uint m_tile_idx = split_kv ? (tile_split_flat % m_tiles) : tile_split_flat;
+ const uint split_idx = split_kv ? (tile_split_flat / m_tiles) : 0u;
+
+ const uint grouped_rows = params.gqa_factor * params.q_len;
+ const uint row_base = m_tile_idx * uint(BLOCK_ROWS) + uint(band) * uint(TILE_ROWS);
+ const uint head_in_group = row_base / params.q_len;
+ const uint query_base = row_base - head_in_group * params.q_len;
+ const uint q_head_idx = kv_head_idx * params.gqa_factor + head_in_group;
+
+ const uint query_row_stride = params.q_strides[2];
+ const uint key_row_stride = params.k_strides[2];
+ const uint value_row_stride = params.v_strides[2];
+ const uint output_row_stride = params.o_strides[2];
+ const uint slice_offset = uint(slice) * uint(SLICE_COLS);
+
+ q += size_t(batch_idx) * params.q_strides[0] + size_t(q_head_idx) * params.q_strides[1] +
+ size_t(query_base) * query_row_stride + slice_offset;
+ k += size_t(batch_idx) * params.k_strides[0] + size_t(kv_head_idx) * params.k_strides[1] + slice_offset;
+ v += size_t(batch_idx) * params.v_strides[0] + size_t(kv_head_idx) * params.v_strides[1] + slice_offset;
+
+ QueryFragment query_frags[HEAD_DIM_FRAGMENTS_PER_SLICE];
+ {
+ auto query_source = fragment_source(q, int(query_row_stride));
+ const uint valid_rows = row_base < grouped_rows ? grouped_rows - row_base : 0u;
+ if (valid_rows < uint(TILE_ROWS)) {
+ query_source = query_source.bounded(short(valid_rows), short(Ops::FRAGMENT_COLS));
+ }
+ METAL_PRAGMA_UNROLL
+ for (short head_dim_fragment_idx = 0; head_dim_fragment_idx < HEAD_DIM_FRAGMENTS_PER_SLICE;
+ ++head_dim_fragment_idx) {
+ query_frags[head_dim_fragment_idx].load_from(
+ lane,
+ query_source.advanced(head_dim_fragment_idx * Ops::FRAGMENT_ROWS)
+ );
+ }
+ }
+
+ OutputFragment output;
+ output.clear();
+
+ AccumType max_score[ROWS_PER_LANE];
+ AccumType sum_score[ROWS_PER_LANE];
+ METAL_PRAGMA_UNROLL
+ for (int r = 0; r < ROWS_PER_LANE; ++r) {
+ max_score[r] = -INFINITY;
+ sum_score[r] = AccumType(0);
+ }
+
+ const uint prefix_length = params.q_off;
+ const uint key_length = params.k_len;
+ const short2 score_position = ScoreFragment::get_position(lane);
+ uint last_visible_key[ROWS_PER_LANE];
+ ulong row_trie_mask[ROWS_PER_LANE];
+ METAL_PRAGMA_UNROLL
+ for (ushort r = 0; r < ROWS_PER_LANE; ++r) {
+ last_visible_key[r] = 0;
+ row_trie_mask[r] = 0ul;
+ }
+ if (is_causal || is_trie) {
+ METAL_PRAGMA_UNROLL
+ for (ushort r = 0; r < ROWS_PER_LANE; ++r) {
+ const uint row = row_base + uint(score_position.y) + uint(r) * uint(Ops::THREAD_ELEMENT_ROW_STRIDE);
+ last_visible_key[r] = prefix_length + row % params.q_len;
+ }
+ if (is_trie) {
+ const ulong all_suffix_keys = (params.q_len >= 64u) ? ~0ul : ((1ul << params.q_len) - 1ul);
+ METAL_PRAGMA_UNROLL
+ for (ushort r = 0; r < ROWS_PER_LANE; ++r) {
+ row_trie_mask[r] = is_causal ? 0ul : all_suffix_keys;
+ }
+ if (is_causal) {
+ for (uint node_index = 0; node_index < params.q_len; ++node_index) {
+ const TrieNode node = trie[node_index];
+ METAL_PRAGMA_UNROLL
+ for (ushort r = 0; r < ROWS_PER_LANE; ++r) {
+ const uint query_index = last_visible_key[r] - prefix_length;
+ if (query_index >= node.trie_start && query_index <= node.trie_end) {
+ row_trie_mask[r] |= (1ul << node_index);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ threadgroup float* exchange_slot = score_exchange + simdgroup_index * SCORE_TILE_ELEMENTS;
+ const ushort band_exchange_base = band * ushort(D_SLICES);
+ const uint key_block_stride = uint(BK) * key_row_stride;
+ const uint value_block_stride = uint(BK) * value_row_stride;
+
+ const uint block_begin = split_kv ? (split_idx * params.nk) / num_splits : 0u;
+ uint block_end = split_kv ? ((split_idx + 1u) * params.nk) / num_splits : params.nk;
+
+ if (is_causal) {
+ const uint tile_first_row = m_tile_idx * uint(BLOCK_ROWS);
+ const uint tile_last_row = min(tile_first_row + uint(BLOCK_ROWS) - 1u, grouped_rows - 1u);
+ const uint max_query_index = (tile_first_row / params.q_len == tile_last_row / params.q_len)
+ ? (tile_last_row % params.q_len)
+ : (params.q_len - 1u);
+ const uint causal_blocks = (uint(prefix_length) + max_query_index + uint(BK)) / uint(BK);
+ block_end = min(block_end, causal_blocks);
+ }
+
+ const uint tail_keys = params.k_rem;
+ const uint block_count = block_end > block_begin ? block_end - block_begin : 0u;
+ const bool owns_tail_block = (tail_keys != 0u) && (block_count > 0u) && (block_end == params.nk);
+ const uint full_blocks = owns_tail_block ? block_count - 1u : block_count;
+
+ k += size_t(block_begin) * key_block_stride;
+ v += size_t(block_begin) * value_block_stride;
+
+ const AccumType score_scale = AccumType(params.scale) * AccumType(M_LOG2E_F);
+ // Masked scores are multiplied by score_scale below, so pre-divide this value.
+ const AccumType masked_score_pre_scale = (static_cast<AccumType>(-1e9f) * M_LOG2E_F) / score_scale;
+ const uint first_key = block_begin * uint(BK);
+
+ auto accumulate_kv_block = [&](const uint kb, auto tail_flag) {
+ constexpr bool IS_TAIL = decltype(tail_flag)::value;
+
+ ScoreFragment score;
+ score.clear();
+ {
+ auto key_source = fragment_source(k + size_t(kb) * key_block_stride, int(key_row_stride));
+ if constexpr (IS_TAIL) {
+ key_source = key_source.bounded(short(tail_keys), short(Ops::FRAGMENT_COLS));
+ }
+ METAL_PRAGMA_UNROLL
+ for (short head_dim_fragment_idx = 0; head_dim_fragment_idx < HEAD_DIM_FRAGMENTS_PER_SLICE;
+ ++head_dim_fragment_idx) {
+ KeyFragment key_fragment;
+ key_fragment.load_from(lane, key_source.advanced(head_dim_fragment_idx * Ops::FRAGMENT_ROWS));
+ simdgroup_barrier(mem_flags::mem_none);
+ fragment_mma(score, query_frags[head_dim_fragment_idx], key_fragment);
+ }
+ }
+
+ if constexpr (D_SLICES > 1) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ score.store(lane, exchange_slot, int(BK));
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ thread AccumType* score_elements = score.elements();
+ METAL_PRAGMA_UNROLL
+ for (ushort j = 1; j < ushort(D_SLICES); ++j) {
+ const ushort other = (slice + j) % ushort(D_SLICES);
+ ScoreFragment partial;
+ partial.load_from(
+ lane,
+ fragment_source(score_exchange + (band_exchange_base + other) * SCORE_TILE_ELEMENTS, int(BK))
+ );
+ thread AccumType* partial_elements = partial.elements();
+ METAL_PRAGMA_UNROLL
+ for (ushort e = 0; e < ScoreFragment::ELEMENTS_PER_FRAGMENT; ++e) {
+ score_elements[e] += partial_elements[e];
+ }
+ }
+ }
+
+ const uint key_base = first_key + kb * uint(BK);
+ const bool masked_block = IS_TAIL || ((is_causal || is_trie) && (key_base + uint(BK) > prefix_length));
+ if (masked_block) {
+ map_lane_rows(score, score_position, [&](ushort row, short col, AccumType value) {
+ const uint key = key_base + uint(col);
+ bool visible = key < key_length;
+ if (is_trie) {
+ visible =
+ visible && (key < prefix_length || ((row_trie_mask[row] >> ulong(key - prefix_length)) & 1ul) != 0ul);
+ } else if (is_causal) {
+ visible = visible && (key <= last_visible_key[row]);
+ }
+ return visible ? value : masked_score_pre_scale;
+ });
+ }
+
+ AccumType block_max[ROWS_PER_LANE];
+ score.row_reduce(block_max, -INFINITY, [](AccumType a, AccumType b) { return metal::max(a, b); });
+ METAL_PRAGMA_UNROLL
+ for (int r = 0; r < ROWS_PER_LANE; ++r) {
+ block_max[r] *= score_scale;
+ }
+
+ AccumType new_max[ROWS_PER_LANE];
+ AccumType rescale[ROWS_PER_LANE];
+ METAL_PRAGMA_UNROLL
+ for (int r = 0; r < ROWS_PER_LANE; ++r) {
+ new_max[r] = metal::max(max_score[r], block_max[r]);
+ rescale[r] = fast::exp2(max_score[r] - new_max[r]);
+ max_score[r] = new_max[r];
+ sum_score[r] *= rescale[r];
+ }
+
+ score.map_rows(new_max, [score_scale](AccumType value, AccumType m) {
+ return fast::exp2(value * score_scale - m);
+ });
+
+ AccumType block_sum[ROWS_PER_LANE];
+ score.row_reduce(block_sum, AccumType(0), [](AccumType a, AccumType b) { return a + b; });
+ METAL_PRAGMA_UNROLL
+ for (int r = 0; r < ROWS_PER_LANE; ++r) {
+ sum_score[r] += block_sum[r];
+ }
+
+ output.map_rows(rescale, [](AccumType value, AccumType f) { return value * f; });
+
+ {
+ auto value_source = fragment_source(v + size_t(kb) * value_block_stride, int(value_row_stride));
+ if constexpr (IS_TAIL) {
+ value_source = value_source.bounded(short(tail_keys), short(SLICE_COLS));
+ }
+ ValueFragment value_fragment;
+ value_fragment.load_from(lane, value_source);
+ simdgroup_barrier(mem_flags::mem_none);
+ fragment_mma(output, score, value_fragment);
+ }
+ };
+
+ for (uint kb = 0; kb < full_blocks; ++kb) {
+ accumulate_kv_block(kb, uzu::false_type{});
+ }
+ if (owns_tail_block) {
+ accumulate_kv_block(full_blocks, uzu::true_type{});
+ }
+
+ if (split_kv) {
+ const uint tile_index = ((batch_idx * kv_heads + kv_head_idx) * m_tiles + m_tile_idx) * num_splits + split_idx;
+ device float* partial_tile = partials + size_t(tile_index) * BLOCK_ROWS * BD + size_t(band) * TILE_ROWS * BD;
+ output.store(lane, partial_tile + slice_offset, int(BD));
+
+ if (slice == 0) {
+ const short2 position = OutputFragment::get_position(lane);
+ if (position.x == 0) {
+ const uint row_base_in_tile = uint(tile_index) * uint(BLOCK_ROWS) + uint(band) * uint(TILE_ROWS);
+ METAL_PRAGMA_UNROLL
+ for (int r = 0; r < ROWS_PER_LANE; ++r) {
+ const uint row = row_base_in_tile + uint(position.y + r * Ops::THREAD_ELEMENT_ROW_STRIDE);
+ partial_maxs[row] = max_score[r];
+ partial_sums[row] = sum_score[r];
+ }
+ }
+ }
+ } else {
+ AccumType inverse_sum[ROWS_PER_LANE];
+ METAL_PRAGMA_UNROLL
+ for (int r = 0; r < ROWS_PER_LANE; ++r) {
+ inverse_sum[r] = AccumType(1) / sum_score[r];
+ }
+ output.map_rows(inverse_sum, [](AccumType value, AccumType s) { return value * s; });
+
+ device T* output_base = o + size_t(batch_idx) * params.o_strides[0];
+ if (params.q_len % uint(TILE_ROWS) == 0u) {
+ if (row_base < grouped_rows) {
+ device T* destination = output_base + size_t(q_head_idx) * params.o_strides[1] +
+ size_t(query_base) * output_row_stride + slice_offset;
+ output.store(lane, destination, int(output_row_stride));
+ }
+ } else {
+ const short2 position = OutputFragment::get_position(lane);
+ thread AccumType* output_elements = output.elements();
+ METAL_PRAGMA_UNROLL
+ for (ushort r = 0; r < ROWS_PER_LANE; ++r) {
+ const uint row = row_base + uint(position.y) + uint(r) * uint(Ops::THREAD_ELEMENT_ROW_STRIDE);
+ if (row < grouped_rows) {
+ const uint row_head = row / params.q_len;
+ const uint row_position = row - row_head * params.q_len;
+ device T* destination = output_base +
+ (size_t(kv_head_idx) * params.gqa_factor + row_head) * params.o_strides[1] +
+ size_t(row_position) * output_row_stride + slice_offset + position.x;
+ METAL_PRAGMA_UNROLL
+ for (ushort fragment_col = 0; fragment_col < VALUE_COL_FRAGMENTS; ++fragment_col) {
+ METAL_PRAGMA_UNROLL
+ for (ushort col = 0; col < Ops::THREAD_ELEMENT_COLS; ++col) {
+ const ushort element = fragment_col * Ops::ELEMENTS_PER_THREAD + r * Ops::THREAD_ELEMENT_COLS + col;
+ destination[fragment_col * Ops::FRAGMENT_COLS + col] = static_cast<T>(output_elements[element]);
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+template <typename T, uint BD>
+VARIANTS(T, bfloat)
+VARIANTS(BD, 128, 256)
+KERNEL(AttentionGemmGroupedCombine)(
+ const device float* partials,
+ const device float* partial_maxs,
+ const device float* partial_sums,
+ device T* o,
+ const constant AttnParams& params,
+ const constant uint& grouped_rows,
+ const constant uint& block_rows,
+ const constant uint& m_tiles,
+ const constant uint& kv_heads,
+ const constant uint& batch_count,
+ const constant uint& num_splits,
+ const uint grouped_row_idx GROUPS(grouped_rows),
+ const uint kv_head_idx GROUPS(kv_heads),
+ const uint batch_idx GROUPS(batch_count),
+ const uint column THREADS(BD)
+) {
+ const uint m_tile_idx = grouped_row_idx / block_rows;
+ const uint row_in_tile = grouped_row_idx - m_tile_idx * block_rows;
+ const uint tile_base = ((batch_idx * kv_heads + kv_head_idx) * m_tiles + m_tile_idx) * num_splits;
+
+ const device float* row_maxs = partial_maxs + size_t(tile_base) * block_rows + row_in_tile;
+ const device float* row_sums = partial_sums + size_t(tile_base) * block_rows + row_in_tile;
+ const device float* row_partials = partials + size_t(tile_base) * block_rows * BD + size_t(row_in_tile) * BD + column;
+
+ float global_max = -INFINITY;
+ for (uint s = 0; s < num_splits; ++s) {
+ if (row_sums[s * block_rows] > 0.0f) {
+ global_max = metal::max(global_max, row_maxs[s * block_rows]);
+ }
+ }
+
+ float numerator = 0.0f;
+ float denominator = 0.0f;
+ if (global_max > -INFINITY) {
+ for (uint s = 0; s < num_splits; ++s) {
+ const float split_sum = row_sums[s * block_rows];
+ if (split_sum <= 0.0f) {
+ continue;
+ }
+ const float rescale = fast::exp2(row_maxs[s * block_rows] - global_max);
+ denominator += rescale * split_sum;
+ numerator += rescale * row_partials[size_t(s) * block_rows * BD];
+ }
+ }
+
+ const uint head_in_group = grouped_row_idx / params.q_len;
+ const uint query_position = grouped_row_idx - head_in_group * params.q_len;
+ const uint q_head_idx = kv_head_idx * params.gqa_factor + head_in_group;
+ o += size_t(batch_idx) * params.o_strides[0] + size_t(q_head_idx) * params.o_strides[1] +
+ size_t(query_position) * params.o_strides[2] + column;
+ o[0] = static_cast<T>(denominator > 0.0f ? numerator / denominator : 0.0f);
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
new file mode 100644
index 000000000..14fd5ebbc
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
@@ -0,0 +1,289 @@
+use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};
+
+use crate::{
+ backends::{
+ common::{
+ Allocation, BufferArg, BufferArgMut, Encoder,
+ gpu_types::AttnParams,
+ kernel::{AttentionArguments, AttentionKernelConfig},
+ },
+ metal::{
+ Metal,
+ context::MetalContext,
+ error::MetalError,
+ kernel::{AttentionGemmGroupedCombineMetalKernel, AttentionGemmGroupedMetalKernel},
+ },
+ },
+ data_type::DataType,
+};
+
+mod policy;
+use policy::{MAX_TRIE_SUFFIX, choose_splits};
+
+use super::MaskKind;
+
+const SIMDGROUPS_PER_THREADGROUP: u32 = 8;
+const SLICE_COLS: u32 = 64;
+const TILE_ROWS: u32 = 16;
+const BLOCK_K: u32 = 32;
+
+struct AttentionGemmGroupedMetal {
+ kernel: AttentionGemmGroupedMetalKernel,
+ split_kernel: AttentionGemmGroupedMetalKernel,
+ combine: AttentionGemmGroupedCombineMetalKernel,
+ head_dim: u32,
+ num_groups: u32,
+ num_q_heads: u32,
+ scale: Option<f32>,
+ block_rows: u32,
+ mask: MaskKind,
+ gpu_core_count: u32,
+}
+
+impl AttentionGemmGroupedMetal {
+ fn new_fixed(
+ context: &MetalContext,
+ head_dim: u32,
+ num_groups: u32,
+ num_q_heads: u32,
+ scale: Option<f32>,
+ mask: MaskKind,
+ ) -> Result<Self, MetalError> {
+ assert!(matches!(head_dim, 128 | 256), "head_dim must be 128 or 256");
+ assert_eq!(num_q_heads % num_groups, 0, "num_q_heads must be divisible by num_groups");
+ let slices = head_dim / SLICE_COLS;
+ let block_rows = SIMDGROUPS_PER_THREADGROUP / slices * TILE_ROWS;
+ let new_kernel = |split| {
+ AttentionGemmGroupedMetalKernel::new(
+ context,
+ DataType::BF16,
+ BLOCK_K,
+ head_dim,
+ slices,
+ split,
+ mask.is_causal(),
+ mask.is_trie(),
+ )
+ };
+ let combine = AttentionGemmGroupedCombineMetalKernel::new(context, DataType::BF16, head_dim)?;
+ Ok(Self {
+ kernel: new_kernel(false)?,
+ split_kernel: new_kernel(true)?,
+ combine,
+ head_dim,
+ num_groups,
+ num_q_heads,
+ scale,
+ block_rows,
+ mask,
+ gpu_core_count: context.device_profile().gpu_core_count(),
+ })
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ fn encode<'q, 'k, 'v, 'o, 't>(
+ &self,
+ queries: impl BufferArg<'q, Metal>,
+ keys: impl BufferArg<'k, Metal>,
+ values: impl BufferArg<'v, Metal>,
+ output: impl BufferArgMut<'o, Metal>,
+ trie: Option<impl BufferArg<'t, Metal>>,
+ suffix_length: u32,
+ kv_length: u32,
+ q_replicas: u32,
+ num_splits: u32,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<(), MetalError> {
+ let params = self.params(suffix_length, kv_length);
+ let grouped_rows = self.num_q_heads / self.num_groups * suffix_length;
+ let m_tiles = grouped_rows.div_ceil(self.block_rows);
+ assert_eq!(trie.is_some(), self.mask.is_trie(), "trie presence must match mask");
+ assert!(num_splits >= 1, "num_splits must be at least one");
+ assert!(num_splits <= params.nk, "num_splits must not exceed KV blocks");
+
+ if num_splits == 1 {
+ self.kernel.encode(
+ queries,
+ keys,
+ values,
+ Some(output),
+ None::<&mut Allocation<Metal>>,
+ None::<&mut Allocation<Metal>>,
+ None::<&mut Allocation<Metal>>,
+ trie,
+ params,
+ m_tiles,
+ self.num_groups,
+ q_replicas,
+ 1,
+ encoder,
+ );
+ return Ok(());
+ }
+
+ let partial_rows = q_replicas * self.num_groups * m_tiles * num_splits * self.block_rows;
+ let mut partials = encoder.allocate_scratch_for_shape(&[partial_rows, self.head_dim], DataType::F32)?;
+ let mut partial_maxs = encoder.allocate_scratch_for_shape(&[partial_rows], DataType::F32)?;
+ let mut partial_sums = encoder.allocate_scratch_for_shape(&[partial_rows], DataType::F32)?;
+
+ self.split_kernel.encode(
+ queries,
+ keys,
+ values,
+ None::<&mut Allocation<Metal>>,
+ Some(&mut partials),
+ Some(&mut partial_maxs),
+ Some(&mut partial_sums),
+ trie,
+ params,
+ m_tiles,
+ self.num_groups,
+ q_replicas,
+ num_splits,
+ encoder,
+ );
+
+ self.combine.encode(
+ &partials,
+ &partial_maxs,
+ &partial_sums,
+ output,
+ params,
+ grouped_rows,
+ self.block_rows,
+ m_tiles,
+ self.num_groups,
+ q_replicas,
+ num_splits,
+ encoder,
+ );
+ Ok(())
+ }
+
+ fn params(
+ &self,
+ suffix_length: u32,
+ kv_length: u32,
+ ) -> AttnParams {
+ assert!(suffix_length > 0 && kv_length > 0, "lengths must be positive");
+ assert!(kv_length >= suffix_length, "kv_length must be at least suffix_length");
+ assert!(!self.mask.is_trie() || suffix_length <= MAX_TRIE_SUFFIX, "trie suffix is too long");
+
+ let head_dim = self.head_dim;
+ AttnParams {
+ q_strides: [self.num_q_heads * suffix_length * head_dim, suffix_length * head_dim, head_dim],
+ k_strides: [0, head_dim, self.num_groups * head_dim],
+ v_strides: [0, head_dim, self.num_groups * head_dim],
+ o_strides: [suffix_length * self.num_q_heads * head_dim, head_dim, self.num_q_heads * head_dim],
+ gqa_factor: self.num_q_heads / self.num_groups,
+ scale: self.scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt()),
+ q_len: suffix_length,
+ k_len: kv_length,
+ q_off: kv_length - suffix_length,
+ nq_aligned: 0,
+ q_rem: 0,
+ nk: kv_length.div_ceil(BLOCK_K),
+ nk_aligned: kv_length / BLOCK_K,
+ k_rem: kv_length % BLOCK_K,
+ }
+ }
+}
+
+pub struct AttentionGemmGrouped {
+ non_trie: Mutex<Option<AttentionGemmGroupedMetal>>,
+ trie: Mutex<Option<AttentionGemmGroupedMetal>>,
+ head_dim: u32,
+ num_groups: u32,
+ num_q_heads: u32,
+ scale: Option<f32>,
+}
+
+impl AttentionGemmGrouped {
+ fn get_or_create(
+ &self,
+ context: &MetalContext,
+ mask: MaskKind,
+ ) -> Result<MappedMutexGuard<'_, AttentionGemmGroupedMetal>, MetalError> {
+ let cache = if mask.is_trie() {
+ &self.trie
+ } else {
+ &self.non_trie
+ };
+ let mut cache = cache.lock();
+ if cache.is_none() {
+ *cache = Some(AttentionGemmGroupedMetal::new_fixed(
+ context,
+ self.head_dim,
+ self.num_groups,
+ self.num_q_heads,
+ self.scale,
+ mask,
+ )?);
+ }
+ Ok(MutexGuard::map(cache, |cache| cache.as_mut().expect("attention pipeline was just initialized")))
+ }
+
+ pub fn is_supported(
+ config: &AttentionKernelConfig,
+ context: &MetalContext,
+ ) -> bool {
+ policy::is_supported(config, context)
+ }
+
+ pub fn new(config: &AttentionKernelConfig) -> Self {
+ Self {
+ non_trie: Mutex::new(None),
+ trie: Mutex::new(None),
+ head_dim: config.head_dim,
+ num_groups: config.num_groups,
+ num_q_heads: config.num_q_heads,
+ scale: config.scale,
+ }
+ }
+
+ pub fn should_encode(
+ &self,
+ mask: MaskKind,
+ suffix_length: u32,
+ kv_length: u32,
+ ) -> bool {
+ policy::should_encode(self.head_dim, mask, suffix_length, kv_length)
+ }
+
+ pub fn encode<'a, KT: BufferArg<'a, Metal>, VT: BufferArg<'a, Metal>>(
+ &self,
+ mask: MaskKind,
+ arguments: AttentionArguments<'a, Metal, KT, VT>,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<Allocation<Metal>, MetalError> {
+ let suffix_length = arguments.suffix_length;
+ assert!(arguments.cache.ring_params().is_none(), "ring KV cache is unsupported");
+ assert!(arguments.sinks.is_none(), "attention sinks are unsupported");
+
+ let kv_length = arguments.cache.prefix_len() + suffix_length;
+ let mut output =
+ encoder.allocate_constant_for_shape(&[suffix_length, self.num_q_heads, self.head_dim], DataType::BF16)?;
+ let core = self.get_or_create(encoder.context(), mask)?;
+ let num_splits = choose_splits(
+ core.head_dim,
+ suffix_length,
+ kv_length,
+ (core.num_q_heads / core.num_groups * suffix_length).div_ceil(core.block_rows) * core.num_groups,
+ BLOCK_K,
+ core.gpu_core_count,
+ );
+ core.encode(
+ arguments.queries,
+ arguments.keys,
+ arguments.values,
+ &mut output,
+ arguments.trie,
+ suffix_length,
+ kv_length,
+ 1,
+ num_splits,
+ encoder,
+ )?;
+ Ok(output)
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
new file mode 100644
index 000000000..b759f5964
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
@@ -0,0 +1,112 @@
+use super::MaskKind;
+use crate::{
+ backends::{common::kernel::AttentionKernelConfig, metal::context::MetalContext},
+ data_type::DataType,
+};
+
+const GEMM_GROUPED_HEAD_DIMS: [u32; 2] = [128, 256];
+const GEMM_GROUPED_DECODE_SUFFIX_MIN: u32 = 16;
+const GEMM_GROUPED_DECODE_SUFFIX_MAX: u32 = 64;
+const GEMM_GROUPED_PREFILL_SUFFIX_MAX: u32 = 1024;
+const GEMM_GROUPED_MIN_KV_LENGTH: u32 = 1024;
+pub const MAX_TRIE_SUFFIX: u32 = 64;
+
+pub fn is_supported(
+ arguments: &AttentionKernelConfig,
+ context: &MetalContext,
+) -> bool {
+ GEMM_GROUPED_HEAD_DIMS.contains(&arguments.head_dim)
+ && arguments.data_type == DataType::BF16
+ && context.supports_mxu()
+ && !arguments.has_sinks
+ && !arguments.is_kv_cache_ring
+ && arguments.sliding_window_size.is_none()
+ && arguments.scale.is_none_or(|scale| scale > 0.0)
+ && arguments.num_groups > 0
+ && arguments.num_q_heads.is_multiple_of(arguments.num_groups)
+}
+
+pub fn should_encode(
+ head_dim: u32,
+ mask: MaskKind,
+ suffix_length: u32,
+ kv_length: u32,
+) -> bool {
+ kv_length >= GEMM_GROUPED_MIN_KV_LENGTH
+ && ((GEMM_GROUPED_DECODE_SUFFIX_MIN..=GEMM_GROUPED_DECODE_SUFFIX_MAX).contains(&suffix_length)
+ || (head_dim == 256
+ && mask == MaskKind::Causal
+ && (GEMM_GROUPED_DECODE_SUFFIX_MAX + 1..=GEMM_GROUPED_PREFILL_SUFFIX_MAX).contains(&suffix_length)))
+}
+
+type MeasuredSplits = (u32, u32, &'static [(u32, u32)]);
+
+const MEASURED_SUFFIX_MIN: u32 = 16;
+const MEASURED_SUFFIX_MAX: u32 = 64;
+
+// TODO: validate this table per chip (M1-M5) before changing the policy.
+// TODO: add a simdgroup/non-MXU implementation before widening availability.
+// TODO: measure the D128/D256 crossover before widening suffix ranges.
+const MEASURED_TG_PER_CORE_TENTHS: &[MeasuredSplits] = &[
+ (256, 16, &[(0, 24), (5120, 18), (131072, 39)]),
+ (256, 32, &[(0, 24), (5120, 18), (32768, 60)]),
+ (256, 64, &[(0, 24), (5120, 60)]),
+ (128, 16, &[(0, 16), (51200, 40)]),
+ (128, 32, &[(0, 16), (5120, 40), (51200, 80)]),
+ (128, 64, &[(0, 32), (5120, 40), (32768, 80)]),
+];
+const TG_PER_CORE_FALLBACK: u32 = 6;
+const TENTHS_PER_RATIO: u32 = 10;
+
+fn tg_per_core_tenths_for(
+ steps: &[(u32, u32)],
+ kv_length: u32,
+) -> u32 {
+ steps.iter().rev().find(|(minimum_kv, _)| kv_length >= *minimum_kv).map_or(steps[0].1, |(_, tenths)| *tenths)
+}
+
+fn splits_for_ratio(
+ tg_per_core_tenths: u32,
+ gpu_core_count: u32,
+ unsplit_threadgroups: u32,
+) -> u32 {
+ (tg_per_core_tenths * gpu_core_count.max(1)).div_ceil(TENTHS_PER_RATIO * unsplit_threadgroups.max(1))
+}
+
+fn tabled_splits(
+ head_dim: u32,
+ suffix_length: u32,
+ kv_length: u32,
+ unsplit_threadgroups: u32,
+ gpu_core_count: u32,
+) -> Option<u32> {
+ if !(MEASURED_SUFFIX_MIN..=MEASURED_SUFFIX_MAX).contains(&suffix_length) {
+ return None;
+ }
+ MEASURED_TG_PER_CORE_TENTHS
+ .iter()
+ .filter(|(row_head_dim, _, _)| *row_head_dim == head_dim)
+ .min_by_key(|(_, row_suffix, _)| (row_suffix.abs_diff(suffix_length), u32::MAX - row_suffix))
+ .map(|(_, _, steps)| {
+ splits_for_ratio(tg_per_core_tenths_for(steps, kv_length), gpu_core_count, unsplit_threadgroups)
+ })
+}
+
+pub fn choose_splits(
+ head_dim: u32,
+ suffix_length: u32,
+ kv_length: u32,
+ unsplit_threadgroups: u32,
+ block_k: u32,
+ gpu_core_count: u32,
+) -> u32 {
+ let splits = tabled_splits(head_dim, suffix_length, kv_length, unsplit_threadgroups, gpu_core_count)
+ .unwrap_or_else(|| {
+ splits_for_ratio(TENTHS_PER_RATIO * TG_PER_CORE_FALLBACK, gpu_core_count, unsplit_threadgroups)
+ });
+ splits.clamp(1, kv_length.div_ceil(block_k).max(1))
+}
+
+#[cfg(test)]
+#[path = "../../../../../../unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs"]
+mod tests;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal
index 851e8661b..b75c7709b 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal
@@ -4,7 +4,7 @@
#include "kv_cache_update.h"
template <typename T>
-VARIANTS(T, float, bfloat, half)
+VARIANTS(T, float, bfloat)
PUBLIC KERNEL(KVCacheUpdate) (
device T* in_place_keys,
device T* in_place_values,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
new file mode 100644
index 000000000..452afdf94
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
@@ -0,0 +1,155 @@
+mod fallback;
+mod gemm;
+mod gemm_grouped;
+mod single_pass;
+mod two_pass;
+
+use crate::backends::{
+ common::{
+ Allocation, BufferArg, Encoder,
+ kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig},
+ },
+ metal::{Metal, context::MetalContext, error::MetalError},
+};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum MaskKind {
+ None,
+ Causal,
+ Trie,
+}
+
+impl MaskKind {
+ fn for_attention(
+ is_causal: bool,
+ is_trie: bool,
+ ) -> Option<Self> {
+ match (is_causal, is_trie) {
+ (true, true) => Some(Self::Trie),
+ (true, false) => Some(Self::Causal),
+ (false, false) => Some(Self::None),
+ (false, true) => None,
+ }
+ }
+
+ fn is_causal(self) -> bool {
+ !matches!(self, Self::None)
+ }
+
+ fn is_trie(self) -> bool {
+ matches!(self, Self::Trie)
+ }
+}
+
+const SINGLE_PASS_KV_THRESHOLD: u32 = 1_024;
+const D256_SINGLE_PASS_KV_THRESHOLD: u32 = 150_000;
+const D128_SHORT_KV_GEMM_THRESHOLD: u32 = 2_048;
+
+pub struct AttentionMetalKernel {
+ head_dim: u32,
+ is_causal: bool,
+ grouped: Option<gemm_grouped::AttentionGemmGrouped>,
+ gemm: Option<gemm::AttentionGemm>,
+ fallback: Option<fallback::AttentionFallback>,
+ two_pass: two_pass::AttentionTwoPass,
+ single_pass: single_pass::AttentionSinglePass,
+}
+
+impl AttentionKernel for AttentionMetalKernel {
+ type Backend = Metal;
+
+ fn new(
+ context: &MetalContext,
+ config: AttentionKernelConfig,
+ ) -> Result<Self, MetalError> {
+ let grouped = gemm_grouped::AttentionGemmGrouped::is_supported(&config, context)
+ .then(|| gemm_grouped::AttentionGemmGrouped::new(&config));
+ let gemm = gemm::AttentionGemm::is_supported(&config).then(|| gemm::AttentionGemm::new(&config));
+ let fallback = fallback::AttentionFallback::is_supported(&config)
+ .then(|| fallback::AttentionFallback::new(&config, context))
+ .transpose()?;
+ Ok(Self {
+ head_dim: config.head_dim,
+ is_causal: config.is_causal,
+ grouped,
+ gemm,
+ fallback,
+ two_pass: two_pass::AttentionTwoPass::new(&config),
+ single_pass: single_pass::AttentionSinglePass::new(&config),
+ })
+ }
+
+ fn encode<'a, KT, VT>(
+ &self,
+ arguments: AttentionArguments<'a, Metal, KT, VT>,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<Allocation<Metal>, MetalError>
+ where
+ KT: BufferArg<'a, Metal>,
+ VT: BufferArg<'a, Metal>,
+ {
+ encoder.push_debug_group("attention core");
+ let result = self.encode_impl(arguments, encoder);
+ encoder.pop_debug_group();
+ result
+ }
+}
+
+impl AttentionMetalKernel {
+ fn encode_impl<'a, KT, VT>(
+ &self,
+ arguments: AttentionArguments<'a, Metal, KT, VT>,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<Allocation<Metal>, MetalError>
+ where
+ KT: BufferArg<'a, Metal>,
+ VT: BufferArg<'a, Metal>,
+ {
+ let suffix_length = arguments.suffix_length;
+ let kv_length = arguments.cache.prefix_len() + suffix_length;
+ let is_trie = arguments.trie.is_some();
+ let is_ring = arguments.cache.ring_params().is_some();
+ let mask = MaskKind::for_attention(self.is_causal, is_trie);
+
+ if !is_ring
+ && let Some(mask) = mask
+ && let Some(grouped) = &self.grouped
+ {
+ if grouped.should_encode(mask, suffix_length, kv_length) {
+ return grouped.encode(mask, arguments, encoder);
+ }
+
+ // These measured sibling exceptions apply only when the
+ // grouped core was built for this layer.
+ if self.head_dim == 256 && (1..=16).contains(&suffix_length) && kv_length > D256_SINGLE_PASS_KV_THRESHOLD {
+ return self.single_pass.encode(arguments, encoder);
+ }
+ if (self.head_dim == 256 && (9..=16).contains(&suffix_length))
+ || (self.head_dim == 128
+ && (9..=16).contains(&suffix_length)
+ && kv_length > D128_SHORT_KV_GEMM_THRESHOLD)
+ {
+ return self.two_pass.encode(arguments, encoder);
+ }
+ }
+
+ if suffix_length > 8 {
+ if let Some(gemm) = &self.gemm {
+ return gemm.encode(arguments, encoder);
+ }
+ if !is_trie && let Some(fallback) = &self.fallback {
+ return fallback.encode(arguments, encoder);
+ }
+ }
+
+ if kv_length > SINGLE_PASS_KV_THRESHOLD {
+ self.two_pass.encode(arguments, encoder)
+ } else {
+ self.single_pass.encode(arguments, encoder)
+ }
+ }
+}
+
+#[cfg(test)]
+#[path = "../../../../../unit/backends/metal/kernel/attention/mod.rs"]
+mod tests;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/qkv_norm.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/qkv_norm.metal
index df042a68d..373e729d7 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/qkv_norm.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/qkv_norm.metal
@@ -11,10 +11,10 @@ using namespace metal;
// - One SIMD-group (32 threads) processes one head.
// - One threadgroup (one SIMD-group) is dispatched per head.
template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>
-VARIANTS(InputT, float, half, bfloat)
-VARIANTS(ScaleT, float, half, bfloat)
-VARIANTS(OutputT, float, half, bfloat)
-VARIANTS(AccumT, float, half)
+VARIANTS(InputT, float, bfloat)
+VARIANTS(ScaleT, float, bfloat)
+VARIANTS(OutputT, float, bfloat)
+VARIANTS(AccumT, float)
PUBLIC KERNEL(QKVNorm)(
const device InputT* qkv_input OPTIONAL(!in_place),
const device ScaleT* scales OPTIONAL(has_scales),
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal
index 876609196..d49cb6b98 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal
@@ -2,7 +2,7 @@
#include "../common/dsl.h"
template <typename T>
-VARIANTS(T, float, half, bfloat)
+VARIANTS(T, float, bfloat)
PUBLIC KERNEL(SigmoidGate)(
const device T* gate,
device T* output,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rs
new file mode 100644
index 000000000..ae72c18c5
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rs
@@ -0,0 +1,81 @@
+use std::collections::{HashMap, hash_map::Entry};
+
+use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};
+
+use crate::backends::{
+ common::{
+ Allocation, BufferArg, Encoder,
+ kernel::{AttentionArguments, AttentionKernelConfig, AttentionSinglePassKernel},
+ },
+ metal::{Metal, context::MetalContext, error::MetalError, kernel::AttentionSinglePassMetalKernel},
+};
+
+pub struct AttentionSinglePass {
+ kernels: Mutex<HashMap<bool, AttentionSinglePassMetalKernel>>,
+ config: AttentionKernelConfig,
+}
+
+impl AttentionSinglePass {
+ pub fn new(config: &AttentionKernelConfig) -> Self {
+ Self {
+ kernels: Mutex::new(HashMap::new()),
+ config: *config,
+ }
+ }
+
+ fn get_or_create(
+ &self,
+ context: &MetalContext,
+ is_trie: bool,
+ ) -> Result<MappedMutexGuard<'_, AttentionSinglePassMetalKernel>, MetalError> {
+ let mut kernels = self.kernels.lock();
+ if let Entry::Vacant(entry) = kernels.entry(is_trie) {
+ let kernel = AttentionSinglePassMetalKernel::new(
+ context,
+ self.config.data_type,
+ self.config.head_dim,
+ self.config.has_sinks,
+ self.config.is_kv_cache_ring,
+ self.config.is_causal,
+ is_trie,
+ self.config.sliding_window_size.is_some(),
+ )?;
+ entry.insert(kernel);
+ }
+ Ok(MutexGuard::map(kernels, |kernels| kernels.get_mut(&is_trie).expect("kernel was just initialized")))
+ }
+
+ pub fn encode<'a, KT: BufferArg<'a, Metal>, VT: BufferArg<'a, Metal>>(
+ &self,
+ arguments: AttentionArguments<'a, Metal, KT, VT>,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<Allocation<Metal>, MetalError> {
+ let config = self.config;
+ let mut output = encoder.allocate_constant_for_shape(
+ &[arguments.suffix_length, config.num_q_heads, config.head_dim],
+ config.data_type,
+ )?;
+ let kernel = self.get_or_create(encoder.context(), arguments.trie.is_some())?;
+ kernel.encode(
+ arguments.queries,
+ arguments.keys,
+ arguments.values,
+ &mut output,
+ config.num_q_heads / config.num_groups,
+ arguments.cache.prefix_len() + arguments.suffix_length,
+ config.head_dim,
+ config.num_groups * config.head_dim,
+ config.head_dim,
+ config.num_groups * config.head_dim,
+ arguments.cache.ring_params(),
+ config.scale.unwrap_or(1.0 / (config.head_dim as f32).sqrt()),
+ arguments.trie,
+ config.sliding_window_size,
+ arguments.sinks,
+ config.num_q_heads,
+ arguments.suffix_length,
+ encoder,
+ );
+ Ok(output)
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rs
new file mode 100644
index 000000000..111a5bda2
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rs
@@ -0,0 +1,120 @@
+use std::collections::{HashMap, hash_map::Entry};
+
+use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};
+
+use crate::{
+ backends::{
+ common::{
+ Allocation, BufferArg, Encoder,
+ kernel::{AttentionArguments, AttentionKernelConfig},
+ },
+ metal::{
+ Metal,
+ context::MetalContext,
+ error::MetalError,
+ kernel::{AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel},
+ },
+ },
+ data_type::DataType,
+};
+
+const PARTIAL_DATA_TYPE: DataType = DataType::F32;
+const PARTIAL_BLOCKS: u32 = 32;
+
+pub struct AttentionTwoPass {
+ passes: Mutex<HashMap<bool, AttentionTwoPass1MetalKernel>>,
+ second: Mutex<Option<AttentionTwoPass2MetalKernel>>,
+ config: AttentionKernelConfig,
+}
+
+impl AttentionTwoPass {
+ pub fn new(config: &AttentionKernelConfig) -> Self {
+ Self {
+ passes: Mutex::new(HashMap::new()),
+ second: Mutex::new(None),
+ config: *config,
+ }
+ }
+
+ fn get_or_create(
+ &self,
+ context: &MetalContext,
+ is_trie: bool,
+ ) -> Result<MappedMutexGuard<'_, AttentionTwoPass1MetalKernel>, MetalError> {
+ let mut passes = self.passes.lock();
+ if let Entry::Vacant(entry) = passes.entry(is_trie) {
+ entry.insert(AttentionTwoPass1MetalKernel::new(
+ context,
+ self.config.data_type,
+ self.config.head_dim,
+ self.config.has_sinks,
+ self.config.is_kv_cache_ring,
+ self.config.is_causal,
+ is_trie,
+ self.config.sliding_window_size.is_some(),
+ )?);
+ }
+ Ok(MutexGuard::map(passes, |passes| passes.get_mut(&is_trie).expect("passes were just initialized")))
+ }
+
+ fn get_or_create_second(
+ &self,
+ context: &MetalContext,
+ ) -> Result<MappedMutexGuard<'_, AttentionTwoPass2MetalKernel>, MetalError> {
+ let mut second = self.second.lock();
+ if second.is_none() {
+ *second = Some(AttentionTwoPass2MetalKernel::new(context, self.config.data_type, self.config.head_dim)?);
+ }
+ Ok(MutexGuard::map(second, |second| second.as_mut().expect("pass was just initialized")))
+ }
+
+ pub fn encode<'a, KT: BufferArg<'a, Metal>, VT: BufferArg<'a, Metal>>(
+ &self,
+ arguments: AttentionArguments<'a, Metal, KT, VT>,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<Allocation<Metal>, MetalError> {
+ let config = self.config;
+ let mut partials = encoder.allocate_scratch_for_shape(
+ &[arguments.suffix_length, config.num_q_heads, PARTIAL_BLOCKS, config.head_dim],
+ PARTIAL_DATA_TYPE,
+ )?;
+ let mut sums = encoder.allocate_scratch_for_shape(
+ &[arguments.suffix_length, config.num_q_heads, PARTIAL_BLOCKS],
+ PARTIAL_DATA_TYPE,
+ )?;
+ let mut maxs = encoder.allocate_scratch_for_shape(
+ &[arguments.suffix_length, config.num_q_heads, PARTIAL_BLOCKS],
+ PARTIAL_DATA_TYPE,
+ )?;
+ let first = self.get_or_create(encoder.context(), arguments.trie.is_some())?;
+ first.encode(
+ arguments.queries,
+ arguments.keys,
+ arguments.values,
+ &mut partials,
+ &mut sums,
+ &mut maxs,
+ config.num_q_heads / config.num_groups,
+ arguments.cache.prefix_len() + arguments.suffix_length,
+ config.head_dim,
+ config.num_groups * config.head_dim,
+ config.head_dim,
+ config.num_groups * config.head_dim,
+ arguments.cache.ring_params(),
+ config.scale.unwrap_or(1.0 / (config.head_dim as f32).sqrt()),
+ config.num_q_heads,
+ arguments.suffix_length,
+ arguments.trie,
+ config.sliding_window_size,
+ arguments.sinks,
+ encoder,
+ );
+ let mut output = encoder.allocate_constant_for_shape(
+ &[arguments.suffix_length, config.num_q_heads, config.head_dim],
+ config.data_type,
+ )?;
+ let second = self.get_or_create_second(encoder.context())?;
+ second.encode(&partials, &sums, &maxs, &mut output, config.num_q_heads, arguments.suffix_length, encoder);
+ Ok(output)
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/mod.rs
index f061464d8..7c2cf7a05 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/mod.rs
@@ -9,7 +9,7 @@ use crate::backends::{
metal::Metal,
};
-pub mod attention;
+mod attention;
pub mod gdn;
pub mod matmul;
mod radix_top_k_small;
@@ -22,7 +22,7 @@ impl Kernels for MetalKernels {
type Backend = Metal;
autogen_kernels!();
- type AttentionGemmCore = attention::AttentionGemmMetalCore;
+ type AttentionKernel = attention::AttentionMetalKernel;
type DeltaNetChunkedPrefill = gdn::chunked::MetalDeltaNetChunkedPrefill;
type DeltaNetTreeVerify = gdn::tree_verify::MetalDeltaNetTreeVerify;
type MatmulKernel = matmul::MatmulMetalKernel;
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rs
deleted file mode 100644
index 1bb3f9278..000000000
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rs
+++ /dev/null
@@ -1,107 +0,0 @@
-use crate::{
- backends::common::{
- Allocation, Backend, BufferArg, Encoder, Kernels,
- kernel::attention_gemm::AttentionGemmCore as AttentionGemmCoreTrait,
- },
- data_type::DataType,
- encodable_block::mixer::attention::{
- KVCacheView,
- core::{fallback::AttentionFallbackCore, single_pass::AttentionSinglePassCore, two_pass::AttentionTwoPassCore},
- },
-};
-
-mod fallback;
-mod single_pass;
-mod two_pass;
-
-pub struct AttentionCoreNewArguments {
- pub head_dim: u32,
- pub num_groups: u32,
- pub num_q_heads: u32,
- pub has_sinks: bool,
- pub is_kv_cache_ring: bool,
- pub is_causal: bool,
- pub is_trie: bool,
- pub sliding_window_size: Option<u32>,
- pub scale: Option<f32>,
- pub data_type: DataType,
-}
-
-pub struct AttentionCoreEncodeArguments<'a, B: Backend, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>> {
- pub queries: &'a Allocation<B>,
- pub keys: KT,
- pub values: VT,
- pub suffix_length: u32,
- pub trie: Option<&'a Allocation<B>>,
- pub sinks: Option<&'a Allocation<B>>,
- pub cache: KVCacheView,
-}
-
-pub struct AttentionCores<B: Backend> {
- gemm: Option<<B::Kernels as Kernels>::AttentionGemmCore>,
- fallback: Option<AttentionFallbackCore<B>>,
- two_pass: AttentionTwoPassCore<B>,
- single_pass: AttentionSinglePassCore<B>,
-}
-
-impl<B: Backend> AttentionCores<B> {
- pub fn new(
- arguments: AttentionCoreNewArguments,
- context: &B::Context,
- ) -> Result<Self, B::Error> {
- let gemm = if <<B::Kernels as Kernels>::AttentionGemmCore as AttentionGemmCoreTrait>::is_supported(
- &arguments, context,
- )? {
- Some(<<B::Kernels as Kernels>::AttentionGemmCore as AttentionGemmCoreTrait>::new(context, &arguments)?)
- } else {
- None
- };
- let fallback = if arguments.head_dim == 512 && !arguments.is_trie {
- Some(AttentionFallbackCore::new(&arguments, context)?)
- } else {
- None
- };
- let two_pass = AttentionTwoPassCore::new(&arguments, context)?;
- let single_pass = AttentionSinglePassCore::new(&arguments, context)?;
-
- Ok(Self {
- gemm,
- fallback,
- two_pass,
- single_pass,
- })
- }
- pub fn encode<'a, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>>(
- &self,
- arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,
- encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, <B as Backend>::Error> {
- encoder.push_debug_group("attention core");
-
- let output = if arguments.suffix_length > 8
- && let Some(gemm) = &self.gemm
- {
- gemm.encode(arguments, encoder)
- } else if arguments.suffix_length > 8
- && let Some(fallback) = &self.fallback
- {
- fallback.encode(arguments, encoder)
- } else if arguments.cache.prefix_len() + arguments.suffix_length > 1024 {
- self.two_pass.encode(arguments, encoder)
- } else {
- self.single_pass.encode(arguments, encoder)
- }?;
-
- encoder.pop_debug_group();
-
- Ok(output)
- }
-}
-
-#[cfg(test)]
-#[path = "../../../../../unit/encodable_block/attention_test.rs"]
-mod tests;
-
-#[cfg(test)]
-#[path = "../../../../../unit/encodable_block/attention_gemm_test.rs"]
-mod gemm_tests;
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rs
deleted file mode 100644
index bffd0153b..000000000
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rs
+++ /dev/null
@@ -1,74 +0,0 @@
-use crate::{
- backends::common::{Allocation, Backend, BufferArg, Encoder, Kernels, kernel::AttentionSinglePassKernel},
- data_type::DataType,
- encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},
-};
-
-pub struct AttentionSinglePassCore<B: Backend> {
- head_dim: u32,
- num_groups: u32,
- num_q_heads: u32,
- sliding_window_size: Option<u32>,
- scale: Option<f32>,
- data_type: DataType,
- kernel: <B::Kernels as Kernels>::AttentionSinglePassKernel,
-}
-
-impl<B: Backend> AttentionSinglePassCore<B> {
- pub fn new(
- arguments: &AttentionCoreNewArguments,
- context: &B::Context,
- ) -> Result<Self, B::Error> {
- let kernel = <B::Kernels as Kernels>::AttentionSinglePassKernel::new(
- context,
- arguments.data_type,
- arguments.head_dim,
- arguments.has_sinks,
- arguments.is_kv_cache_ring,
- arguments.is_causal,
- arguments.is_trie,
- arguments.sliding_window_size.is_some(),
- )?;
-
- Ok(Self {
- head_dim: arguments.head_dim,
- num_groups: arguments.num_groups,
- num_q_heads: arguments.num_q_heads,
- sliding_window_size: arguments.sliding_window_size,
- scale: arguments.scale,
- data_type: arguments.data_type,
- kernel,
- })
- }
-
- pub fn encode<'a, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>>(
- &self,
- arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,
- encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, B::Error> {
- let mut output = encoder
- .allocate_constant_for_shape(&[arguments.suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;
- self.kernel.encode(
- arguments.queries,
- arguments.keys,
- arguments.values,
- &mut output,
- self.num_q_heads / self.num_groups,
- arguments.cache.prefix_len() + arguments.suffix_length,
- self.head_dim,
- self.num_groups * self.head_dim,
- self.head_dim,
- self.num_groups * self.head_dim,
- arguments.cache.ring_params(),
- self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),
- arguments.trie,
- self.sliding_window_size,
- arguments.sinks,
- self.num_q_heads,
- arguments.suffix_length,
- encoder,
- );
-
- Ok(output)
- }
-}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rs
deleted file mode 100644
index ca072829b..000000000
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rs
+++ /dev/null
@@ -1,102 +0,0 @@
-use crate::{
- backends::common::{
- Allocation, Backend, BufferArg, Encoder, Kernels,
- kernel::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},
- },
- data_type::DataType,
- encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},
-};
-
-const INNER_DATA_TYPE: DataType = DataType::F32;
-const TWO_PASS_BLOCKS: u32 = 32;
-
-pub struct AttentionTwoPassCore<B: Backend> {
- head_dim: u32,
- num_groups: u32,
- num_q_heads: u32,
- sliding_window_size: Option<u32>,
- scale: Option<f32>,
- data_type: DataType,
- pass_1: <B::Kernels as Kernels>::AttentionTwoPass1Kernel,
- pass_2: <B::Kernels as Kernels>::AttentionTwoPass2Kernel,
-}
-
-impl<B: Backend> AttentionTwoPassCore<B> {
- pub fn new(
- arguments: &AttentionCoreNewArguments,
- context: &B::Context,
- ) -> Result<Self, B::Error> {
- let pass_1 = <B::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
- context,
- arguments.data_type,
- arguments.head_dim,
- arguments.has_sinks,
- arguments.is_kv_cache_ring,
- arguments.is_causal,
- arguments.is_trie,
- arguments.sliding_window_size.is_some(),
- )?;
-
- let pass_2 =
- <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, arguments.data_type, arguments.head_dim)?;
-
- Ok(Self {
- head_dim: arguments.head_dim,
- num_groups: arguments.num_groups,
- num_q_heads: arguments.num_q_heads,
- sliding_window_size: arguments.sliding_window_size,
- scale: arguments.scale,
- data_type: arguments.data_type,
- pass_1,
- pass_2,
- })
- }
-
- pub fn encode<'a, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>>(
- &self,
- arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,
- encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, B::Error> {
- let mut partials = encoder.allocate_scratch_for_shape(
- &[arguments.suffix_length, self.num_q_heads, TWO_PASS_BLOCKS, self.head_dim],
- INNER_DATA_TYPE,
- )?;
- let mut sums = encoder.allocate_scratch_for_shape(
- &[arguments.suffix_length, self.num_q_heads, TWO_PASS_BLOCKS],
- INNER_DATA_TYPE,
- )?;
- let mut maxs = encoder.allocate_scratch_for_shape(
- &[arguments.suffix_length, self.num_q_heads, TWO_PASS_BLOCKS],
- INNER_DATA_TYPE,
- )?;
-
- self.pass_1.encode(
- arguments.queries,
- arguments.keys,
- arguments.values,
- &mut partials,
- &mut sums,
- &mut maxs,
- self.num_q_heads / self.num_groups,
- arguments.cache.prefix_len() + arguments.suffix_length,
- self.head_dim,
- self.num_groups * self.head_dim,
- self.head_dim,
- self.num_groups * self.head_dim,
- arguments.cache.ring_params(),
- self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),
- self.num_q_heads,
- arguments.suffix_length,
- arguments.trie,
- self.sliding_window_size,
- arguments.sinks,
- encoder,
- );
-
- let mut output = encoder
- .allocate_constant_for_shape(&[arguments.suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;
- self.pass_2.encode(&partials, &sums, &maxs, &mut output, self.num_q_heads, arguments.suffix_length, encoder);
-
- Ok(output)
- }
-}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
index 1aef18ff2..3e7927833 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs
@@ -3,7 +3,7 @@ use thiserror::Error;
use crate::{
backends::common::{
Allocation, Backend, Encoder, Kernels,
- kernel::{AttentionPrepareKernel, SigmoidGateKernel},
+ kernel::{AttentionKernel, AttentionKernelConfig, AttentionPrepareKernel, SigmoidGateKernel},
},
config::{rope::AnyRoPEConfig, token_mixer::attention::AttentionConfig},
data_type::DataType,
@@ -13,7 +13,6 @@ use crate::{
mixer::{
Mixer, MixerState,
attention::{
- core::{AttentionCoreNewArguments, AttentionCores},
mode::LinearProjection,
qkv_norm::{QKVNorm, QKVNormError},
rope::PrecalculatedRoPE,
@@ -24,7 +23,6 @@ use crate::{
utils::maybe_mut::MaybeMut,
};
-pub mod core;
mod mode;
mod qkv_norm;
mod state;
@@ -44,8 +42,7 @@ pub struct Attention<B: Backend> {
prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
gate_projection: Option<Box<dyn Linear<B>>>,
sinks: Option<Allocation<B>>,
- flat_core: AttentionCores<B>,
- trie_core: AttentionCores<B>,
+ kernel: <B::Kernels as Kernels>::AttentionKernel,
gate_kernel: Option<<B::Kernels as Kernels>::SigmoidGateKernel>,
out_projection: Box<dyn Linear<B>>,
}
@@ -167,37 +164,19 @@ impl<B: Backend> Attention<B> {
let ring_capacity = sliding_window_size;
let is_kv_cache_ring = ring_capacity.is_some();
- let flat_core = AttentionCores::new(
- AttentionCoreNewArguments {
- head_dim,
- num_groups,
- num_q_heads,
- has_sinks: sinks.is_some(),
- is_kv_cache_ring,
- is_causal,
- is_trie: false,
- sliding_window_size,
- scale: config.scale,
- data_type,
- },
+ let kernel = <B::Kernels as Kernels>::AttentionKernel::new(
context,
- )
- .map_err(AttentionNewError::Backend)?;
-
- let trie_core = AttentionCores::new(
- AttentionCoreNewArguments {
+ AttentionKernelConfig {
head_dim,
num_groups,
num_q_heads,
has_sinks: sinks.is_some(),
is_kv_cache_ring,
is_causal,
- is_trie: true,
sliding_window_size,
scale: config.scale,
data_type,
},
- context,
)
.map_err(AttentionNewError::Backend)?;
@@ -230,8 +209,7 @@ impl<B: Backend> Attention<B> {
prepare,
gate_projection,
sinks,
- flat_core,
- trie_core,
+ kernel,
gate_kernel,
out_projection,
},
diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
index 218cbb124..863e5e21d 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs
@@ -2,17 +2,14 @@ use crate::{
backends::common::{
Allocation, Backend, BufferArgMut, Encoder,
gpu_types::trie::TrieNode,
- kernel::{AttentionPrepareKernel, SigmoidGateKernel},
+ kernel::{AttentionArguments, AttentionKernel, AttentionPrepareKernel, SigmoidGateKernel},
},
encodable_block::{
batch_topology::BatchTopology,
linear::Linear,
mixer::{
MixerState,
- attention::{
- Attention, KVCacheView, core::AttentionCoreEncodeArguments, qkv_norm::QKVNorm, rope::PrecalculatedRoPE,
- state::AttentionState,
- },
+ attention::{Attention, KVCacheView, qkv_norm::QKVNorm, rope::PrecalculatedRoPE, state::AttentionState},
},
},
utils::maybe_mut::MaybeMut,
@@ -104,8 +101,8 @@ impl<B: Backend> Attention<B> {
let cache = self.ring_capacity.map_or_else(|| KVCacheView::full(0), |_| KVCacheView::ring(0, 0));
- self.flat_core.encode(
- AttentionCoreEncodeArguments {
+ self.kernel.encode(
+ AttentionArguments {
queries: &queries,
keys: &keys,
values: &values,
@@ -163,16 +160,16 @@ impl<B: Backend> Attention<B> {
state: &AttentionState<B>,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let (core, trie) = if batch_dim.is_flat() {
- (&self.flat_core, None)
+ let trie = if batch_dim.is_flat() {
+ None
} else {
let mut trie = encoder.allocate_constant(batch_dim.size() as usize * size_of::<TrieNode>())?;
trie.copyin(batch_dim.nodes());
- (&self.trie_core, Some(trie))
+ Some(trie)
};
- core.encode(
- AttentionCoreEncodeArguments {
+ self.kernel.encode(
+ AttentionArguments {
queries,
keys: state.keys.as_ref(),
values: state.values.as_ref(),
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index 309210ccd..07bcfa28e 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -9,8 +9,8 @@ use crate::{
TreeIdx,
},
kernel::{
- AncestorAttentionKernel, WeaverFrontierInsertChildrenKernel, WeaverFrontierSelectKernel,
- WeaverTopChildrenKernel, radix_top_k_small::RadixTopKSmall,
+ AncestorAttentionKernel, AttentionArguments, AttentionKernel, WeaverFrontierInsertChildrenKernel,
+ WeaverFrontierSelectKernel, WeaverTopChildrenKernel, radix_top_k_small::RadixTopKSmall,
},
},
config::{rope::AnyRoPEConfig, weaver::WeaverConfig},
@@ -18,7 +18,7 @@ use crate::{
encodable_block::{
embedding::{Embedding, EmbeddingError},
linear::{Linear, LinearBlockError},
- mixer::attention::{KVCacheView, core::AttentionCoreEncodeArguments, rope::PrecalculatedRoPE},
+ mixer::attention::{KVCacheView, rope::PrecalculatedRoPE},
mlp::MlpBlockError,
normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
weaver_layer::{PreparedPrefixAttention, WeaverLayer},
@@ -330,7 +330,7 @@ impl<B: Backend> Weaver<B> {
let attention_output = layer
.prefix_attention
.encode(
- AttentionCoreEncodeArguments {
+ AttentionArguments {
queries: &queries,
keys: &kv_cache,
values: (&kv_cache, kv_plane_bytes),
diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
index dacfd8d11..8c6a52468 100644
--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
@@ -2,7 +2,7 @@ use crate::{
array::size_for_shape,
backends::common::{
Allocation, AsBufferRangeMut, Backend, Encoder, Kernels,
- kernel::{AncestorAttentionKernel, AttentionPrepareKernel},
+ kernel::{AncestorAttentionKernel, AttentionKernel, AttentionKernelConfig, AttentionPrepareKernel},
},
config::{
activation::{AnyActivation, silu::SiLU},
@@ -11,10 +11,7 @@ use crate::{
},
encodable_block::{
linear::Linear,
- mixer::attention::{
- core::{AttentionCoreNewArguments, AttentionCores},
- rope::PrecalculatedRoPE,
- },
+ mixer::attention::rope::PrecalculatedRoPE,
mlp::Mlp,
normalization::{Normalization, PostLayerScalar, ShortcutMode},
weaver::{DATA_TYPE, ROPE_DATA_TYPE, WeaverNewError},
@@ -31,7 +28,7 @@ pub struct WeaverLayer<B: Backend> {
pub pre_attention_norm: Normalization<B>,
pub qkv_projection: Box<dyn Linear<B>>,
attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
- pub prefix_attention: AttentionCores<B>,
+ pub prefix_attention: <B::Kernels as Kernels>::AttentionKernel,
pub ancestor_attention: <B::Kernels as Kernels>::AncestorAttentionKernel,
out_projection: Box<dyn Linear<B>>,
pre_mlp_norm: Normalization<B>,
@@ -77,20 +74,19 @@ impl<B: Backend> WeaverLayer<B> {
DATA_TYPE,
&parameter_tree.subtree("out_projection"),
)?;
- let prefix_attention = AttentionCores::new(
- AttentionCoreNewArguments {
+ let prefix_attention = <B::Kernels as Kernels>::AttentionKernel::new(
+ context,
+ AttentionKernelConfig {
head_dim,
num_groups: num_heads,
num_q_heads: num_heads,
has_sinks: false,
is_kv_cache_ring: false,
is_causal: true,
- is_trie: false,
sliding_window_size: None,
scale: Some(attention_scale),
data_type: DATA_TYPE,
},
- context,
)
.map_err(WeaverNewError::Backend)?;
let attention_prepare =
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_fallback_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_fallback_test.rs
deleted file mode 100644
index faf7d9789..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_fallback_test.rs
+++ /dev/null
@@ -1,209 +0,0 @@
-use half::bf16;
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::{
- common::{
- Allocation, Backend, Context, Encoder, Kernels,
- kernel::{
- AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, AttentionSinglePassKernel,
- SoftmaxKernel,
- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
- },
- },
- cpu::Cpu,
- },
- data_type::DataType,
- tests::{
- assert::assert_eq_float,
- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},
- },
-};
-
-const NUM_HEADS: u32 = 4;
-const NUM_GROUPS: u32 = 2;
-const SEQ: u32 = 16;
-const SUFFIX: u32 = 3;
-const HEAD_DIM: u32 = 512;
-const GQA: u32 = NUM_HEADS / NUM_GROUPS;
-
-fn make_inputs() -> (Box<[bf16]>, Box<[bf16]>, Box<[bf16]>) {
- let q = (0..(NUM_HEADS * SUFFIX * HEAD_DIM) as usize)
- .map(|i| bf16::from_f32((i as f32 * 0.013 + 0.5).sin() * 0.5))
- .collect();
- let k = (0..(NUM_GROUPS * SEQ * HEAD_DIM) as usize)
- .map(|i| bf16::from_f32((i as f32 * 0.007 + 1.0).cos() * 0.5))
- .collect();
- let v = (0..(NUM_GROUPS * SEQ * HEAD_DIM) as usize)
- .map(|i| bf16::from_f32((i as f32 * 0.011 + 2.0).sin() * 0.5))
- .collect();
- (q, k, v)
-}
-
-// Ground truth: fused single-pass attention on the CPU backend.
-fn reference(
- q: &[bf16],
- k: &[bf16],
- v: &[bf16],
- scale: f32,
-) -> Vec<bf16> {
- let ctx = <Cpu as Backend>::Context::new().unwrap();
- let kernel = <<Cpu as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
- &ctx,
- DataType::BF16,
- HEAD_DIM,
- false,
- false,
- true,
- false,
- false,
- )
- .unwrap();
- let qa = alloc_allocation_with_data::<Cpu, bf16>(&ctx, q);
- let ka = alloc_allocation_with_data::<Cpu, bf16>(&ctx, k);
- let va = alloc_allocation_with_data::<Cpu, bf16>(&ctx, v);
- let mut out = alloc_allocation::<Cpu, bf16>(&ctx, (NUM_HEADS * SUFFIX * HEAD_DIM) as usize);
- let mut enc = Encoder::new(ctx.as_ref()).unwrap();
- kernel.encode(
- &qa,
- &ka,
- &va,
- &mut out,
- GQA,
- SEQ,
- SEQ * HEAD_DIM,
- HEAD_DIM,
- SEQ * HEAD_DIM,
- HEAD_DIM,
- None,
- scale,
- None::<&Allocation<Cpu>>,
- None,
- None::<&Allocation<Cpu>>,
- NUM_HEADS,
- SUFFIX,
- &mut enc,
- );
- enc.end_encoding().submit().wait_until_completed().unwrap();
- allocation_to_vec::<Cpu, bf16>(&out)
-}
-
-// Production fallback pipeline (matmul → scatter → softmax → matmul → scatter)
-// driven directly via the kernel-cache traits, for backends other than CPU.
-fn pipeline_output<B: Backend>(
- q: &[bf16],
- k: &[bf16],
- v: &[bf16],
- scale: f32,
-) -> Vec<bf16> {
- let ctx = B::Context::new().unwrap();
- let softmax = <<B as Backend>::Kernels as Kernels>::SoftmaxKernel::new(&ctx, DataType::BF16, false).unwrap();
- let scatter_scores = <<B as Backend>::Kernels as Kernels>::AttentionFallbackScatterScoresKernel::new(
- &ctx,
- DataType::BF16,
- false,
- true,
- false,
- false,
- )
- .unwrap();
- let scatter_values =
- <<B as Backend>::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(&ctx, DataType::BF16).unwrap();
- let mut matmul =
- <<B as Backend>::Kernels as Kernels>::MatmulKernel::new(&ctx, DataType::BF16, DataType::BF16, DataType::BF16)
- .unwrap();
-
- let qa = alloc_allocation_with_data::<B, bf16>(&ctx, q);
- let ka = alloc_allocation_with_data::<B, bf16>(&ctx, k);
- let va = alloc_allocation_with_data::<B, bf16>(&ctx, v);
- let mut scores = alloc_allocation::<B, bf16>(&ctx, (NUM_HEADS * SUFFIX * SEQ) as usize);
- let mut grp_s = alloc_allocation::<B, bf16>(&ctx, (GQA * SUFFIX * SEQ) as usize);
- let mut grp_o = alloc_allocation::<B, bf16>(&ctx, (GQA * SUFFIX * HEAD_DIM) as usize);
- let mut out = alloc_allocation::<B, bf16>(&ctx, (NUM_HEADS * SUFFIX * HEAD_DIM) as usize);
-
- let dt = DataType::BF16.size_in_bytes();
- let q_row = HEAD_DIM as usize * dt;
- let s_row = SEQ as usize * dt;
- let head_stride = (SEQ * HEAD_DIM) as usize;
- let mut enc = Encoder::new(ctx.as_ref()).unwrap();
-
- for g in 0..NUM_GROUPS {
- matmul
- .encode(
- MatmulArguments {
- a: MatmulA::FullPrecision {
- values: &qa,
- offset: g as usize * GQA as usize * SUFFIX as usize * q_row,
- },
- b: MatmulB::FullPrecision {
- b: (&ka, g as usize * head_stride * dt),
- },
- b_leading_dimension: Some(HEAD_DIM),
- b_transpose: true,
- d: &mut grp_s,
- d_transform: MatmulDOps {
- ab_scale: scale,
- ..MatmulDOps::none()
- },
- gather_indices: None,
- m: GQA * SUFFIX,
- n: SEQ,
- k: HEAD_DIM,
- },
- &mut enc,
- )
- .expect("encode failed");
- scatter_scores.encode(
- &grp_s,
- &mut scores,
- None,
- None::<&Allocation<B>>,
- None,
- g,
- GQA,
- SEQ,
- SUFFIX,
- GQA * SUFFIX * SEQ,
- &mut enc,
- );
- }
- softmax.encode(&mut scores, None::<&Allocation<B>>, SEQ, NUM_HEADS, SUFFIX, &mut enc);
- for g in 0..NUM_GROUPS {
- matmul
- .encode(
- MatmulArguments {
- a: MatmulA::FullPrecision {
- values: &scores,
- offset: g as usize * GQA as usize * SUFFIX as usize * s_row,
- },
- b: MatmulB::FullPrecision {
- b: (&va, g as usize * head_stride * dt),
- },
- b_leading_dimension: Some(HEAD_DIM),
- b_transpose: false,
- d: &mut grp_o,
- d_transform: MatmulDOps::none(),
- gather_indices: None,
- m: GQA * SUFFIX,
- n: HEAD_DIM,
- k: SEQ,
- },
- &mut enc,
- )
- .expect("encode failed");
- scatter_values.encode(&grp_o, &mut out, g, GQA, SUFFIX, NUM_HEADS, HEAD_DIM, GQA * SUFFIX * HEAD_DIM, &mut enc);
- }
- enc.end_encoding().submit().wait_until_completed().unwrap();
- allocation_to_vec::<B, bf16>(&out)
-}
-
-#[uzu_test]
-fn test_head_dim_512_bf16() {
- let (q, k, v) = make_inputs();
- let scale = 1.0 / (HEAD_DIM as f32).sqrt();
- let expected = reference(&q, &k, &v, scale);
- for_each_non_cpu_backend!(|B| {
- let output = pipeline_output::<B>(&q, &k, &v, scale);
- assert_eq_float::<bf16>(&expected, &output, 1e-2, "AttentionFallback bf16");
- });
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rs
deleted file mode 100644
index bda3b9bf2..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rs
+++ /dev/null
@@ -1,295 +0,0 @@
-use std::fmt::{Debug, Display};
-
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- array::ArrayElement,
- backends::{
- common::{
- Allocation, Backend, Context, Encoder, Kernels, gpu_types::ring::RingParams,
- kernel::AttentionSinglePassKernel,
- },
- cpu::Cpu,
- },
- data_type::DataType,
- tests::{
- assert::assert_eq_float,
- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},
- },
-};
-
-struct Input<T: ArrayElement + Float> {
- queries: Box<[T]>,
- keys: Box<[T]>,
- values: Box<[T]>,
- num_heads: u32,
- gqa_factor: u32,
- sequence_length: u32,
- suffix_length: u32,
- head_dim: u32,
- scale: f32,
- do_causal: bool,
- ring_params: Option<RingParams>,
- sliding_window_size: Option<u32>,
-}
-
-fn get_input<T: ArrayElement + Float>(
- num_heads: u32,
- num_kv_heads: u32,
- sequence_length: u32,
- suffix_length: u32,
- head_dim: u32,
- do_causal: bool,
-) -> Input<T> {
- let gqa_factor = num_heads / num_kv_heads;
-
- // queries: [num_heads * suffix_length, head_dim]
- let q_size = (num_heads * suffix_length * head_dim) as usize;
- let mut queries = vec![T::zero(); q_size];
- for i in 0..q_size {
- queries[i] = T::from((i as f32 * 0.13 + 0.5).sin() * 0.5).unwrap();
- }
-
- // keys: [num_kv_heads, sequence_length, head_dim]
- let k_size = (num_kv_heads * sequence_length * head_dim) as usize;
- let mut keys = vec![T::zero(); k_size];
- for i in 0..k_size {
- keys[i] = T::from((i as f32 * 0.07 + 1.0).cos() * 0.5).unwrap();
- }
-
- // values: [num_kv_heads, sequence_length, head_dim]
- let v_size = (num_kv_heads * sequence_length * head_dim) as usize;
- let mut values = vec![T::zero(); v_size];
- for i in 0..v_size {
- values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();
- }
-
- let scale = 1.0 / (head_dim as f32).sqrt();
-
- Input {
- queries: queries.into_boxed_slice(),
- keys: keys.into_boxed_slice(),
- values: values.into_boxed_slice(),
- num_heads,
- gqa_factor,
- sequence_length,
- suffix_length,
- head_dim,
- scale,
- do_causal,
- ring_params: None,
- sliding_window_size: None,
- }
-}
-
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
- let context = B::Context::new().expect("Failed to create Context");
-
- let kernel = <<B as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
- &context,
- T::data_type(),
- input.head_dim,
- false,
- input.ring_params.is_some(),
- input.do_causal,
- false,
- input.sliding_window_size.is_some(),
- )
- .expect("Failed to create AttentionSinglePassKernel");
-
- let queries_allocation = alloc_allocation_with_data::<B, T>(&context, &input.queries);
- let keys_allocation = alloc_allocation_with_data::<B, T>(&context, &input.keys);
- let values_allocation = alloc_allocation_with_data::<B, T>(&context, &input.values);
-
- let output_size = (input.suffix_length * input.num_heads * input.head_dim) as usize;
- let mut output_allocation = alloc_allocation::<B, T>(&context, output_size);
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- &queries_allocation,
- &keys_allocation,
- &values_allocation,
- &mut output_allocation,
- input.gqa_factor,
- input.sequence_length,
- input.sequence_length * input.head_dim,
- input.head_dim,
- input.sequence_length * input.head_dim,
- input.head_dim,
- input.ring_params,
- input.scale,
- None::<&Allocation<B>>,
- input.sliding_window_size,
- None::<&Allocation<B>>,
- input.num_heads,
- input.suffix_length,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- allocation_to_vec::<B, T>(&output_allocation)
-}
-
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
- input: &Input<T>,
- expected: &[T],
-) {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
- 1e-2
- } else {
- 1e-5
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<T, B>(input);
- let msg = format!(
- "AttentionSinglePass failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={})",
- std::any::type_name::<B>(),
- input.num_heads,
- input.sequence_length,
- input.suffix_length,
- input.head_dim,
- input.do_causal,
- );
- assert_eq_float::<T>(expected, &output, eps, &msg);
- });
-}
-
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
- // Non-causal, single token
- let input = get_input::<T>(4, 4, 8, 1, 64, false);
- let expected = get_output::<T, Cpu>(&input);
- test_internal(&input, &expected);
-
- // Non-causal, multiple tokens
- let input = get_input::<T>(4, 4, 8, 4, 64, false);
- let expected = get_output::<T, Cpu>(&input);
- test_internal(&input, &expected);
-}
-
-fn test_causal<T: ArrayElement + Float + Debug + Display>() {
- // Causal, single token decode
- let input = get_input::<T>(4, 4, 16, 1, 64, true);
- let expected = get_output::<T, Cpu>(&input);
- test_internal(&input, &expected);
-
- // Causal, multi-token prefill
- let input = get_input::<T>(4, 4, 8, 4, 64, true);
- let expected = get_output::<T, Cpu>(&input);
- test_internal(&input, &expected);
-}
-
-fn test_gqa<T: ArrayElement + Float + Debug + Display>() {
- // GQA: 8 query heads, 2 kv heads
- let input = get_input::<T>(8, 2, 8, 1, 64, false);
- let expected = get_output::<T, Cpu>(&input);
- test_internal(&input, &expected);
-
- // GQA causal
- let input = get_input::<T>(8, 2, 8, 4, 64, true);
- let expected = get_output::<T, Cpu>(&input);
- test_internal(&input, &expected);
-}
-
-fn test_noncausal_ring<T: ArrayElement + Float + Debug + Display>() {
- let mut input = get_input::<T>(8, 2, 8, 4, 64, false);
- input.ring_params = Some(RingParams {
- ring_offset: 2,
- ring_length: 4,
- });
- input.sliding_window_size = Some(4);
- let expected = get_output::<T, Cpu>(&input);
- test_internal(&input, &expected);
-}
-
-fn test_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {
- let input = get_input::<T>(4, 4, 8, 2, head_dim, true);
- let expected = get_output::<T, Cpu>(&input);
- test_internal(&input, &expected);
-}
-
-// Basic tests
-#[uzu_test]
-fn test_basic_f32() {
- test_basic::<f32>();
-}
-
-#[uzu_test]
-fn test_basic_f16() {
- test_basic::<f16>();
-}
-
-#[uzu_test]
-fn test_basic_bf16() {
- test_basic::<bf16>();
-}
-
-// Causal tests
-#[uzu_test]
-fn test_causal_f32() {
- test_causal::<f32>();
-}
-
-#[uzu_test]
-fn test_causal_f16() {
- test_causal::<f16>();
-}
-
-#[uzu_test]
-fn test_causal_bf16() {
- test_causal::<bf16>();
-}
-
-// GQA tests
-#[uzu_test]
-fn test_gqa_f32() {
- test_gqa::<f32>();
-}
-
-#[uzu_test]
-fn test_gqa_f16() {
- test_gqa::<f16>();
-}
-
-#[uzu_test]
-fn test_gqa_bf16() {
- test_gqa::<bf16>();
-}
-
-#[uzu_test]
-fn test_noncausal_ring_bf16() {
- test_noncausal_ring::<bf16>();
-}
-
-// Head dim 128
-#[uzu_test]
-fn test_head_dim_128_f32() {
- test_head_dim::<f32>(128);
-}
-
-#[uzu_test]
-fn test_head_dim_128_f16() {
- test_head_dim::<f16>(128);
-}
-
-#[uzu_test]
-fn test_head_dim_128_bf16() {
- test_head_dim::<bf16>(128);
-}
-
-#[uzu_test]
-fn test_head_dim_512_f32() {
- test_head_dim::<f32>(512);
-}
-
-#[uzu_test]
-fn test_head_dim_512_f16() {
- test_head_dim::<f16>(512);
-}
-
-#[uzu_test]
-fn test_head_dim_512_bf16() {
- test_head_dim::<bf16>(512);
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_two_pass_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_two_pass_test.rs
deleted file mode 100644
index 193239007..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_two_pass_test.rs
+++ /dev/null
@@ -1,446 +0,0 @@
-use std::fmt::{Debug, Display};
-
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- array::ArrayElement,
- backends::{
- common::{
- Allocation, Backend, Context, Encoder, Kernels,
- kernel::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},
- },
- cpu::Cpu,
- },
- data_type::DataType,
- tests::{
- assert::assert_eq_float,
- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},
- },
-};
-
-const TOTAL_BLOCKS_COUNT: u32 = 32;
-
-// --- First pass types and helpers ---
-
-struct FirstPassInput<T: ArrayElement + Float> {
- queries: Box<[T]>,
- keys: Box<[T]>,
- values: Box<[T]>,
- num_heads: u32,
- gqa_factor: u32,
- sequence_length: u32,
- suffix_length: u32,
- head_dim: u32,
- scale: f32,
- do_causal: bool,
-}
-
-struct FirstPassOutput {
- partials: Vec<f32>,
- sums: Vec<f32>,
- maxs: Vec<f32>,
-}
-
-fn get_first_pass_input<T: ArrayElement + Float>(
- num_heads: u32,
- num_kv_heads: u32,
- sequence_length: u32,
- suffix_length: u32,
- head_dim: u32,
- do_causal: bool,
-) -> FirstPassInput<T> {
- let gqa_factor = num_heads / num_kv_heads;
-
- let q_size = (num_heads * suffix_length * head_dim) as usize;
- let mut queries = vec![T::zero(); q_size];
- for i in 0..q_size {
- queries[i] = T::from((i as f32 * 0.13 + 0.5).sin() * 0.5).unwrap();
- }
-
- let k_size = (num_kv_heads * sequence_length * head_dim) as usize;
- let mut keys = vec![T::zero(); k_size];
- for i in 0..k_size {
- keys[i] = T::from((i as f32 * 0.07 + 1.0).cos() * 0.5).unwrap();
- }
-
- let v_size = (num_kv_heads * sequence_length * head_dim) as usize;
- let mut values = vec![T::zero(); v_size];
- for i in 0..v_size {
- values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();
- }
-
- let scale = 1.0 / (head_dim as f32).sqrt();
-
- FirstPassInput {
- queries: queries.into_boxed_slice(),
- keys: keys.into_boxed_slice(),
- values: values.into_boxed_slice(),
- num_heads,
- gqa_factor,
- sequence_length,
- suffix_length,
- head_dim,
- scale,
- do_causal,
- }
-}
-
-fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassInput<T>) -> FirstPassOutput {
- let context = B::Context::new().expect("Failed to create Context");
-
- let kernel = <<B as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
- &context,
- T::data_type(),
- input.head_dim,
- false,
- false,
- input.do_causal,
- false,
- false,
- )
- .expect("Failed to create AttentionTwoPass1Kernel");
-
- let queries = alloc_allocation_with_data::<B, T>(&context, &input.queries);
- let keys = alloc_allocation_with_data::<B, T>(&context, &input.keys);
- let values = alloc_allocation_with_data::<B, T>(&context, &input.values);
-
- let total_offsets = (input.suffix_length * input.num_heads) as usize;
- let partials_size = total_offsets * TOTAL_BLOCKS_COUNT as usize * input.head_dim as usize;
- let sums_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;
- let maxs_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;
-
- let mut partials = alloc_allocation::<B, f32>(&context, partials_size);
- let mut sums = alloc_allocation::<B, f32>(&context, sums_size);
- let mut maxs = alloc_allocation::<B, f32>(&context, maxs_size);
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- &queries,
- &keys,
- &values,
- &mut partials,
- &mut sums,
- &mut maxs,
- input.gqa_factor,
- input.sequence_length,
- input.sequence_length * input.head_dim,
- input.head_dim,
- input.sequence_length * input.head_dim,
- input.head_dim,
- None,
- input.scale,
- input.num_heads,
- input.suffix_length,
- None::<&Allocation<B>>,
- None,
- None::<&Allocation<B>>,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- FirstPassOutput {
- partials: allocation_to_vec(&partials),
- sums: allocation_to_vec(&sums),
- maxs: allocation_to_vec(&maxs),
- }
-}
-
-// --- Second pass types and helpers ---
-
-struct SecondPassInput {
- partials: Box<[f32]>,
- sums: Box<[f32]>,
- maxs: Box<[f32]>,
- num_heads: u32,
- suffix_length: u32,
- head_dim: u32,
-}
-
-fn get_second_pass_input(
- num_heads: u32,
- suffix_length: u32,
- head_dim: u32,
-) -> SecondPassInput {
- let total_offsets = (suffix_length * num_heads) as usize;
- let partials_size = total_offsets * TOTAL_BLOCKS_COUNT as usize * head_dim as usize;
- let sums_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;
- let maxs_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;
-
- let mut partials = vec![0.0f32; partials_size];
- for i in 0..partials_size {
- partials[i] = (i as f32 * 0.03 + 0.1).sin() * 0.5;
- }
-
- let mut sums = vec![0.0f32; sums_size];
- for i in 0..sums_size {
- // Ensure sums are positive (they represent sum of exp scores)
- sums[i] = (i as f32 * 0.07 + 1.0).cos().abs() + 0.1;
- }
-
- let mut maxs = vec![0.0f32; maxs_size];
- for i in 0..maxs_size {
- maxs[i] = (i as f32 * 0.05 + 0.5).sin() * 2.0;
- }
-
- SecondPassInput {
- partials: partials.into_boxed_slice(),
- sums: sums.into_boxed_slice(),
- maxs: maxs.into_boxed_slice(),
- num_heads,
- suffix_length,
- head_dim,
- }
-}
-
-fn get_second_pass_output<T: ArrayElement + Float, B: Backend>(input: &SecondPassInput) -> Vec<T> {
- let context = B::Context::new().expect("Failed to create Context");
-
- let kernel =
- <<B as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(&context, T::data_type(), input.head_dim)
- .expect("Failed to create AttentionTwoPass2Kernel");
-
- let partials = alloc_allocation_with_data::<B, f32>(&context, &input.partials);
- let sums = alloc_allocation_with_data::<B, f32>(&context, &input.sums);
- let maxs = alloc_allocation_with_data::<B, f32>(&context, &input.maxs);
-
- let output_size = (input.suffix_length * input.num_heads * input.head_dim) as usize;
- let mut output = alloc_allocation::<B, T>(&context, output_size);
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(&partials, &sums, &maxs, &mut output, input.num_heads, input.suffix_length, &mut encoder);
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- allocation_to_vec(&output)
-}
-
-// --- First pass tests ---
-
-fn test_first_pass_internal<T: ArrayElement + Float + Debug + Display>(
- input: &FirstPassInput<T>,
- expected: &FirstPassOutput,
-) {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
- 1e-2
- } else {
- 1e-5
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_first_pass_output::<T, B>(input);
- let msg = format!(
- "AttentionTwoPass1 failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={})",
- std::any::type_name::<B>(),
- input.num_heads,
- input.sequence_length,
- input.suffix_length,
- input.head_dim,
- input.do_causal,
- );
- assert_eq_float::<f32>(&expected.partials, &output.partials, eps, &format!("{msg} [partials]"));
- assert_eq_float::<f32>(&expected.sums, &output.sums, eps, &format!("{msg} [sums]"));
- assert_eq_float::<f32>(&expected.maxs, &output.maxs, eps, &format!("{msg} [maxs]"));
- });
-}
-
-fn test_first_pass_basic<T: ArrayElement + Float + Debug + Display>() {
- let input = get_first_pass_input::<T>(4, 4, 8, 1, 64, false);
- let expected = get_first_pass_output::<T, Cpu>(&input);
- test_first_pass_internal(&input, &expected);
-
- let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, false);
- let expected = get_first_pass_output::<T, Cpu>(&input);
- test_first_pass_internal(&input, &expected);
-}
-
-fn test_first_pass_causal<T: ArrayElement + Float + Debug + Display>() {
- let input = get_first_pass_input::<T>(4, 4, 16, 1, 64, true);
- let expected = get_first_pass_output::<T, Cpu>(&input);
- test_first_pass_internal(&input, &expected);
-
- let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, true);
- let expected = get_first_pass_output::<T, Cpu>(&input);
- test_first_pass_internal(&input, &expected);
-}
-
-fn test_first_pass_gqa<T: ArrayElement + Float + Debug + Display>() {
- let input = get_first_pass_input::<T>(8, 2, 8, 1, 64, false);
- let expected = get_first_pass_output::<T, Cpu>(&input);
- test_first_pass_internal(&input, &expected);
-
- let input = get_first_pass_input::<T>(8, 2, 8, 4, 64, true);
- let expected = get_first_pass_output::<T, Cpu>(&input);
- test_first_pass_internal(&input, &expected);
-}
-
-fn test_first_pass_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {
- let input = get_first_pass_input::<T>(4, 4, 8, 2, head_dim, true);
- let expected = get_first_pass_output::<T, Cpu>(&input);
- test_first_pass_internal(&input, &expected);
-}
-
-// --- Second pass tests ---
-
-fn test_second_pass_internal<T: ArrayElement + Float + Debug + Display>(
- input: &SecondPassInput,
- expected: &[T],
-) {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
- 1e-2
- } else {
- 1e-5
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_second_pass_output::<T, B>(input);
- let msg = format!(
- "AttentionTwoPass2 failed (backend={}, heads={}, suffix={}, head_dim={})",
- std::any::type_name::<B>(),
- input.num_heads,
- input.suffix_length,
- input.head_dim,
- );
- assert_eq_float::<T>(expected, &output, eps, &msg);
- });
-}
-
-fn test_second_pass_basic<T: ArrayElement + Float + Debug + Display>() {
- let input = get_second_pass_input(4, 1, 64);
- let expected = get_second_pass_output::<T, Cpu>(&input);
- test_second_pass_internal::<T>(&input, &expected);
-
- let input = get_second_pass_input(4, 4, 64);
- let expected = get_second_pass_output::<T, Cpu>(&input);
- test_second_pass_internal::<T>(&input, &expected);
-}
-
-fn test_second_pass_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {
- let input = get_second_pass_input(4, 2, head_dim);
- let expected = get_second_pass_output::<T, Cpu>(&input);
- test_second_pass_internal::<T>(&input, &expected);
-}
-
-// --- First pass test entries ---
-
-#[uzu_test]
-fn test_first_pass_basic_f32() {
- test_first_pass_basic::<f32>();
-}
-
-#[uzu_test]
-fn test_first_pass_basic_f16() {
- test_first_pass_basic::<f16>();
-}
-
-#[uzu_test]
-fn test_first_pass_basic_bf16() {
- test_first_pass_basic::<bf16>();
-}
-
-#[uzu_test]
-fn test_first_pass_causal_f32() {
- test_first_pass_causal::<f32>();
-}
-
-#[uzu_test]
-fn test_first_pass_causal_f16() {
- test_first_pass_causal::<f16>();
-}
-
-#[uzu_test]
-fn test_first_pass_causal_bf16() {
- test_first_pass_causal::<bf16>();
-}
-
-#[uzu_test]
-fn test_first_pass_gqa_f32() {
- test_first_pass_gqa::<f32>();
-}
-
-#[uzu_test]
-fn test_first_pass_gqa_f16() {
- test_first_pass_gqa::<f16>();
-}
-
-#[uzu_test]
-fn test_first_pass_gqa_bf16() {
- test_first_pass_gqa::<bf16>();
-}
-
-#[uzu_test]
-fn test_first_pass_head_dim_128_f32() {
- test_first_pass_head_dim::<f32>(128);
-}
-
-#[uzu_test]
-fn test_first_pass_head_dim_128_f16() {
- test_first_pass_head_dim::<f16>(128);
-}
-
-#[uzu_test]
-fn test_first_pass_head_dim_128_bf16() {
- test_first_pass_head_dim::<bf16>(128);
-}
-
-#[uzu_test]
-fn test_first_pass_head_dim_512_f32() {
- test_first_pass_head_dim::<f32>(512);
-}
-
-#[uzu_test]
-fn test_first_pass_head_dim_512_f16() {
- test_first_pass_head_dim::<f16>(512);
-}
-
-#[uzu_test]
-fn test_first_pass_head_dim_512_bf16() {
- test_first_pass_head_dim::<bf16>(512);
-}
-
-// --- Second pass test entries ---
-
-#[uzu_test]
-fn test_second_pass_basic_f32() {
- test_second_pass_basic::<f32>();
-}
-
-#[uzu_test]
-fn test_second_pass_basic_f16() {
- test_second_pass_basic::<f16>();
-}
-
-#[uzu_test]
-fn test_second_pass_basic_bf16() {
- test_second_pass_basic::<bf16>();
-}
-
-#[uzu_test]
-fn test_second_pass_head_dim_128_f32() {
- test_second_pass_head_dim::<f32>(128);
-}
-
-#[uzu_test]
-fn test_second_pass_head_dim_128_f16() {
- test_second_pass_head_dim::<f16>(128);
-}
-
-#[uzu_test]
-fn test_second_pass_head_dim_128_bf16() {
- test_second_pass_head_dim::<bf16>(128);
-}
-
-#[uzu_test]
-fn test_second_pass_head_dim_512_f32() {
- test_second_pass_head_dim::<f32>(512);
-}
-
-#[uzu_test]
-fn test_second_pass_head_dim_512_f16() {
- test_second_pass_head_dim::<f16>(512);
-}
-
-#[uzu_test]
-fn test_second_pass_head_dim_512_bf16() {
- test_second_pass_head_dim::<bf16>(512);
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/kv_cache_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
similarity index 97%
rename from crates/uzu-engine/unit/backends/common/kernel/kv_cache_update_test.rs
rename to crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
index 72d8d047a..1e5ee58f2 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/kv_cache_update_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
@@ -1,6 +1,6 @@
use std::fmt::{Debug, Display};
-use half::{bf16, f16};
+use half::bf16;
#[cfg(backend = "metal")]
use ndarray::{Array, Array3};
use num_traits::Float;
@@ -215,11 +215,6 @@ fn test_single_copy_f32() {
test_single_copy_internal::<f32>();
}
-#[uzu_test]
-fn test_single_copy_f16() {
- test_single_copy_internal::<f16>();
-}
-
#[uzu_test]
fn test_single_copy_bf16() {
test_single_copy_internal::<bf16>();
@@ -231,11 +226,6 @@ fn test_multi_copy_f32() {
test_multi_copy_internal::<f32>();
}
-#[uzu_test]
-fn test_multi_copy_f16() {
- test_multi_copy_internal::<f16>();
-}
-
#[uzu_test]
fn test_multi_copy_bf16() {
test_multi_copy_internal::<bf16>();
@@ -247,11 +237,6 @@ fn test_no_copy_f32() {
test_no_copy_internal::<f32>();
}
-#[uzu_test]
-fn test_no_copy_f16() {
- test_no_copy_internal::<f16>();
-}
-
#[uzu_test]
fn test_no_copy_bf16() {
test_no_copy_internal::<bf16>();
@@ -263,11 +248,6 @@ fn test_large_f32() {
test_large_internal::<f32>();
}
-#[uzu_test]
-fn test_large_f16() {
- test_large_internal::<f16>();
-}
-
#[uzu_test]
fn test_large_bf16() {
test_large_internal::<bf16>();
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/mod.rs
index 2d613a765..44cda3596 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/mod.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/mod.rs
@@ -1,5 +1,4 @@
mod ancestor_attention_test;
-mod attention_fallback_test;
-mod attention_single_pass_test;
-mod attention_two_pass_test;
+mod kv_cache_update_test;
+mod qkv_norm_test;
mod sigmoid_gate_test;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/qkv_norm_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rs
similarity index 91%
rename from crates/uzu-engine/unit/backends/common/kernel/qkv_norm_test.rs
rename to crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rs
index b4335f276..6f7ce5b4a 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/qkv_norm_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rs
@@ -1,6 +1,6 @@
use std::fmt::{Debug, Display};
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use uzu_engine_macros::uzu_test;
@@ -163,10 +163,9 @@ fn test_internal<
input: &Input<InputT, ScaleT, OutputT>,
expected: &[OutputT],
) {
- let eps = if matches!(InputT::data_type(), DataType::F16 | DataType::BF16)
- || matches!(ScaleT::data_type(), DataType::F16 | DataType::BF16)
- || matches!(OutputT::data_type(), DataType::F16 | DataType::BF16)
- || matches!(AccumT::data_type(), DataType::F16 | DataType::BF16)
+ let eps = if [InputT::data_type(), ScaleT::data_type(), OutputT::data_type(), AccumT::data_type()]
+ .into_iter()
+ .any(|data_type| data_type == DataType::BF16)
{
1e-2
} else {
@@ -318,47 +317,17 @@ fn test_q_norm_f32_f32_f32_f32() {
test_q_norm::<f32, f32, f32, f32>();
}
-#[uzu_test]
-fn test_q_norm_f16_f16_f16_f32() {
- test_q_norm::<f16, f16, f16, f32>();
-}
-
-#[uzu_test]
-fn test_q_norm_f16_f16_f16_f16() {
- test_q_norm::<f16, f16, f16, f16>();
-}
-
#[uzu_test]
fn test_q_norm_bf16_bf16_bf16_f32() {
test_q_norm::<bf16, bf16, bf16, f32>();
}
-#[uzu_test]
-fn test_q_norm_f32_f16_f32_f32() {
- test_q_norm::<f32, f16, f32, f32>();
-}
-
-#[uzu_test]
-fn test_q_norm_f16_f32_f16_f32() {
- test_q_norm::<f16, f32, f16, f32>();
-}
-
// K norm tests
#[uzu_test]
fn test_k_norm_f32_f32_f32_f32() {
test_k_norm::<f32, f32, f32, f32>();
}
-#[uzu_test]
-fn test_k_norm_f16_f16_f16_f32() {
- test_k_norm::<f16, f16, f16, f32>();
-}
-
-#[uzu_test]
-fn test_k_norm_f16_f16_f16_f16() {
- test_k_norm::<f16, f16, f16, f16>();
-}
-
#[uzu_test]
fn test_k_norm_bf16_bf16_bf16_f32() {
test_k_norm::<bf16, bf16, bf16, f32>();
@@ -369,11 +338,6 @@ fn test_v_norm_no_scales_f32_f32_f32_f32() {
test_v_norm_no_scales::<f32, f32, f32, f32>();
}
-#[uzu_test]
-fn test_v_norm_no_scales_f16_f16_f16_f32() {
- test_v_norm_no_scales::<f16, f16, f16, f32>();
-}
-
#[uzu_test]
fn test_v_norm_no_scales_bf16_bf16_bf16_f32() {
test_v_norm_no_scales::<bf16, bf16, bf16, f32>();
@@ -385,11 +349,6 @@ fn test_addressing_f32_f32_f32_f32() {
test_addressing::<f32, f32, f32, f32>();
}
-#[uzu_test]
-fn test_addressing_f16_f16_f16_f32() {
- test_addressing::<f16, f16, f16, f32>();
-}
-
#[uzu_test]
fn test_v_addressing_f32_f32_f32_f32() {
test_v_addressing::<f32, f32, f32, f32>();
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs
index 2f3e7c179..5f668e5f7 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs
@@ -1,6 +1,6 @@
use std::fmt::Debug;
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use uzu_engine_macros::uzu_test;
@@ -97,20 +97,6 @@ fn test_sigmoid_gate_f32() {
});
}
-#[uzu_test]
-fn test_sigmoid_gate_f16() {
- run_test::<f16>(&Config {
- num_heads: 8,
- head_dim: 64,
- suffix_length: 4,
- });
- run_test::<f16>(&Config {
- num_heads: 16,
- head_dim: 256,
- suffix_length: 1,
- });
-}
-
#[uzu_test]
fn test_sigmoid_gate_bf16() {
run_test::<bf16>(&Config {
diff --git a/crates/uzu-engine/unit/backends/common/kernel/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/mod.rs
index 6ab0887dd..e56e9b0d9 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/mod.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/mod.rs
@@ -4,13 +4,11 @@ mod attention;
mod embedding;
mod gated_act_mul_test;
mod gdn;
-mod kv_cache_update_test;
mod logit_transform_test;
mod matmul;
mod moe;
mod normalization_test;
mod pooling;
-mod qkv_norm_test;
mod radix_top_k_small_test;
mod short_conv;
mod ssm;
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
new file mode 100644
index 000000000..33b42c2c5
--- /dev/null
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
@@ -0,0 +1,49 @@
+use uzu_engine_macros::uzu_test;
+
+use super::{MaskKind, choose_splits, should_encode};
+
+fn splits(
+ cores: u32,
+ head_dim: u32,
+ suffix_length: u32,
+ kv_length: u32,
+) -> u32 {
+ let (gqa, rows, groups) = if head_dim == 128 {
+ (4, 64, 8)
+ } else {
+ (6, 32, 4)
+ };
+ let threadgroups = (gqa * suffix_length).div_ceil(rows) * groups;
+ choose_splits(head_dim, suffix_length, kv_length, threadgroups, 32, cores)
+}
+
+#[uzu_test]
+fn measured_and_fallback_boundaries() {
+ for (cores, head_dim, suffix, kv, expected) in [
+ (40, 256, 16, 5_120, 6),
+ (40, 256, 32, 32_768, 10),
+ (40, 128, 64, 32_768, 10),
+ (40, 256, 17, 262_144, 10),
+ (40, 256, 128, 262_144, 3),
+ (40, 256, 16, 128, 4),
+ (10, 256, 32, 262_144, 3),
+ (10, 256, 128, 262_144, 1),
+ ] {
+ assert_eq!(splits(cores, head_dim, suffix, kv), expected);
+ }
+}
+
+#[uzu_test]
+fn should_encode_boundaries() {
+ for (head_dim, mask, suffix_length, kv_length, expected) in [
+ (128, MaskKind::Causal, 16, 1_024, true),
+ (128, MaskKind::Causal, 32, 1_023, false),
+ (256, MaskKind::Causal, 16, 1_023, false),
+ (256, MaskKind::Causal, 8, 150_001, false),
+ (256, MaskKind::Causal, 1_024, 1_024, true),
+ (256, MaskKind::None, 1_024, 1_024, false),
+ (256, MaskKind::Trie, 65, 1_024, false),
+ ] {
+ assert_eq!(should_encode(head_dim, mask, suffix_length, kv_length), expected);
+ }
+}
diff --git a/crates/uzu-engine/unit/encodable_block/attention_gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs
similarity index 66%
rename from crates/uzu-engine/unit/encodable_block/attention_gemm_test.rs
rename to crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs
index d37dcb66e..ba21fe864 100644
--- a/crates/uzu-engine/unit/encodable_block/attention_gemm_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs
@@ -1,23 +1,24 @@
use std::fmt::{Debug, Display};
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use uzu_engine_macros::uzu_test;
use crate::{
array::ArrayElement,
backends::{
- common::{Backend, Context, Encoder, Kernels, kernel::attention_gemm::AttentionGemmCore},
+ common::{
+ Backend, Context, Encoder, Kernels,
+ kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig},
+ },
cpu::Cpu,
+ metal::Metal,
},
data_type::DataType,
- encodable_block::mixer::attention::{
- KVCacheView,
- core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments, AttentionCores},
- },
+ encodable_block::mixer::attention::KVCacheView,
tests::{
assert::assert_eq_float,
- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
},
};
@@ -30,10 +31,13 @@ struct Input<T: ArrayElement + Float> {
sequence_length: usize,
suffix_length: usize,
head_dim: usize,
- scale: f32,
do_causal: bool,
}
+fn attention_config<T: ArrayElement + Float>(input: &Input<T>) -> AttentionKernelConfig {
+ super::default_attention_config::<T>(input.head_dim, input.num_heads, input.num_kv_heads, input.do_causal)
+}
+
fn get_test_data<T: ArrayElement + Float>(
num_heads: usize,
num_kv_heads: usize,
@@ -60,8 +64,6 @@ fn get_test_data<T: ArrayElement + Float>(
values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();
}
- let scale = 1.0 / (head_dim as f32).sqrt();
-
let input = Input {
queries: queries.into_boxed_slice(),
keys: keys.into_boxed_slice(),
@@ -71,7 +73,6 @@ fn get_test_data<T: ArrayElement + Float>(
sequence_length,
suffix_length,
head_dim,
- scale,
do_causal,
};
@@ -82,25 +83,14 @@ fn get_test_data<T: ArrayElement + Float>(
fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
- let new_arguments = AttentionCoreNewArguments {
- head_dim: input.head_dim as u32,
- num_groups: input.num_kv_heads as u32,
- num_q_heads: input.num_heads as u32,
- has_sinks: false,
- is_kv_cache_ring: false,
- is_causal: input.do_causal,
- is_trie: false,
- sliding_window_size: None,
- scale: Some(input.scale),
- data_type: T::data_type(),
- };
+ let config = attention_config(input);
let queries_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.queries);
let keys_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.keys);
let values_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.values);
let segment_prefix_length = input.sequence_length - input.suffix_length;
- let args = AttentionCoreEncodeArguments {
+ let args = AttentionArguments {
queries: &queries_allocation,
keys: &keys_allocation,
values: &values_allocation,
@@ -111,52 +101,71 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
};
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- let pooled_output =
- if <<B as Backend>::Kernels as Kernels>::AttentionGemmCore::is_supported(&new_arguments, context.as_ref())
- .expect("Failed to query AttentionGemm support")
- {
- let core = <<B as Backend>::Kernels as Kernels>::AttentionGemmCore::new(context.as_ref(), &new_arguments)
- .expect("Failed to create AttentionGemmCore");
- core.encode(args, &mut encoder).expect("Failed to encode AttentionGemm")
- } else {
- let core =
- AttentionCores::<B>::new(new_arguments, context.as_ref()).expect("Failed to create AttentionCore");
- core.encode(args, &mut encoder).expect("Failed to encode AttentionCore")
- };
+ let kernel = <B::Kernels as Kernels>::AttentionKernel::new(context.as_ref(), config)
+ .expect("Failed to create attention kernel");
+ let pooled_output = kernel.encode(args, &mut encoder).expect("Failed to encode attention");
let mut output_allocation =
alloc_allocation::<B, T>(context.as_ref(), input.suffix_length * input.num_heads * input.head_dim);
encoder.encode_copy(&pooled_output, .., &mut output_allocation, ..);
- let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(pooled_output);
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(completed);
allocation_to_vec::<B, T>(&output_allocation)
}
+fn get_gemm_output<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
+ let context = <Metal as Backend>::Context::new().expect("Failed to create Metal context");
+ let config = attention_config(input);
+ let queries = alloc_allocation_with_data::<Metal, T>(context.as_ref(), &input.queries);
+ let keys = alloc_allocation_with_data::<Metal, T>(context.as_ref(), &input.keys);
+ let values = alloc_allocation_with_data::<Metal, T>(context.as_ref(), &input.values);
+ let cache = KVCacheView::full((input.sequence_length - input.suffix_length) as u32);
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ let pooled = super::gemm::AttentionGemm::new(&config)
+ .encode(
+ AttentionArguments {
+ queries: &queries,
+ keys: &keys,
+ values: &values,
+ suffix_length: input.suffix_length as u32,
+ trie: None,
+ sinks: None,
+ cache,
+ },
+ &mut encoder,
+ )
+ .expect("Failed to encode AttentionGemm");
+ let mut output =
+ alloc_allocation::<Metal, T>(context.as_ref(), input.suffix_length * input.num_heads * input.head_dim);
+ encoder.encode_copy(&pooled, .., &mut output, ..);
+ drop(pooled);
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(completed);
+ allocation_to_vec::<Metal, T>(&output)
+}
+
fn test_internal<T: ArrayElement + Float + Debug + Display>(
input: &Input<T>,
expected: &[T],
) {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
+ let eps = if T::data_type() == DataType::BF16 {
1e-2
} else {
1e-5
};
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<T, B>(input);
- let msg = format!(
- "AttentionGemm failed (backend={}, heads={}, kv_heads={}, seq={}, suffix={}, head_dim={}, causal={})",
- std::any::type_name::<B>(),
- input.num_heads,
- input.num_kv_heads,
- input.sequence_length,
- input.suffix_length,
- input.head_dim,
- input.do_causal,
- );
- assert_eq_float::<T>(expected, &output, eps, &msg);
- });
+ let output = get_gemm_output(input);
+ let msg = format!(
+ "AttentionGemm failed (heads={}, kv_heads={}, seq={}, suffix={}, head_dim={}, causal={})",
+ input.num_heads,
+ input.num_kv_heads,
+ input.sequence_length,
+ input.suffix_length,
+ input.head_dim,
+ input.do_causal,
+ );
+ assert_eq_float::<T>(expected, &output, eps, &msg);
}
fn test_basic<T: ArrayElement + Float + Debug + Display>() {
@@ -201,11 +210,6 @@ fn test_basic_f32() {
test_basic::<f32>();
}
-#[uzu_test]
-fn test_basic_f16() {
- test_basic::<f16>();
-}
-
#[uzu_test]
fn test_basic_bf16() {
test_basic::<bf16>();
@@ -216,11 +220,6 @@ fn test_causal_f32() {
test_causal::<f32>();
}
-#[uzu_test]
-fn test_causal_f16() {
- test_causal::<f16>();
-}
-
#[uzu_test]
fn test_causal_bf16() {
test_causal::<bf16>();
@@ -231,11 +230,6 @@ fn test_gqa_f32() {
test_gqa::<f32>();
}
-#[uzu_test]
-fn test_gqa_f16() {
- test_gqa::<f16>();
-}
-
#[uzu_test]
fn test_gqa_bf16() {
test_gqa::<bf16>();
@@ -246,11 +240,6 @@ fn test_head_dim_128_f32() {
test_head_dim::<f32>(128);
}
-#[uzu_test]
-fn test_head_dim_128_f16() {
- test_head_dim::<f16>(128);
-}
-
#[uzu_test]
fn test_head_dim_128_bf16() {
test_head_dim::<bf16>(128);
@@ -261,11 +250,6 @@ fn test_unaligned_f32() {
test_unaligned::<f32>();
}
-#[uzu_test]
-fn test_unaligned_f16() {
- test_unaligned::<f16>();
-}
-
#[uzu_test]
fn test_unaligned_bf16() {
test_unaligned::<bf16>();
@@ -273,7 +257,7 @@ fn test_unaligned_bf16() {
#[uzu_test]
fn test_prefill_mxu() {
- let (input, expected) = get_test_data::<f16>(4, 4, 128, 128, 64, true);
+ let (input, expected) = get_test_data::<bf16>(4, 4, 128, 128, 64, true);
test_internal(&input, &expected);
let (input, expected) = get_test_data::<bf16>(8, 2, 100, 96, 128, true);
test_internal(&input, &expected);
diff --git a/crates/uzu-engine/unit/encodable_block/attention_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
similarity index 60%
rename from crates/uzu-engine/unit/encodable_block/attention_test.rs
rename to crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
index 52981e5ba..273fc8db6 100644
--- a/crates/uzu-engine/unit/encodable_block/attention_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
@@ -1,26 +1,23 @@
-#![cfg(backend = "metal")]
-
+use half::bf16;
use ndarray::{Array4, s};
-use test_tag::tag;
use uzu_engine_macros::uzu_test;
use crate::{
backends::{
common::{
Allocation, Backend, Context, Encoder, Kernels,
- kernel::{
- AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel,
- attention_gemm::AttentionGemmCore,
- },
+ gpu_types::trie::TrieNode as GpuTrieNode,
+ kernel::{AttentionArguments, AttentionKernel},
},
+ cpu::Cpu,
metal::Metal,
},
- data_type::DataType,
- encodable_block::mixer::attention::{
- KVCacheView,
- core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},
+ encodable_block::{mixer::attention::KVCacheView, sampling::PRng},
+ tests::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, submit_encoder},
},
- tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, submit_encoder},
+ trie::TrieNode,
};
fn reference_attention(
@@ -215,45 +212,37 @@ fn convert_kernel_output(
}
fn run_single_pass_attention(
- kernel: &<<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel,
+ kernel: &super::single_pass::AttentionSinglePass,
context: &<Metal as Backend>::Context,
queries: &Array4<f32>,
keys: &Array4<f32>,
values: &Array4<f32>,
sinks: Option<&[f32]>,
- scale: f32,
+ _scale: f32,
) -> Result<Array4<f32>, Box<dyn std::error::Error>> {
let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
- let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();
+ let (_batch_size, _num_kv_heads, _seq_len, _head_dim) = keys.dim();
let query_buffer = create_query_allocation(queries, context);
let key_cache_buffer = create_attention_cache_allocation(keys, seq_len, context);
let value_cache_buffer = create_attention_cache_allocation(values, seq_len, context);
let sinks_buffer = sinks.map(|sinks| create_sinks_allocation(sinks, context));
- let mut output_buffer = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);
-
let mut encoder = Encoder::new(context).expect("Failed to create encoder");
-
- kernel.encode(
- &query_buffer,
- &key_cache_buffer,
- &value_cache_buffer,
- &mut output_buffer,
- (num_heads / num_kv_heads) as u32,
- seq_len as u32,
- head_dim as u32,
- (num_kv_heads * head_dim) as u32,
- head_dim as u32,
- (num_kv_heads * head_dim) as u32,
- None,
- scale,
- None::<&Allocation<Metal>>,
- None,
- sinks_buffer.as_ref(),
- num_heads as u32,
- seq_len as u32,
+ let pooled_output = kernel.encode(
+ AttentionArguments {
+ queries: &query_buffer,
+ keys: &key_cache_buffer,
+ values: &value_cache_buffer,
+ suffix_length: seq_len as u32,
+ trie: None,
+ sinks: sinks_buffer.as_ref(),
+ cache: KVCacheView::full(0),
+ },
&mut encoder,
- );
+ )?;
+ let mut output_buffer = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);
+ encoder.encode_copy(&pooled_output, .., &mut output_buffer, ..);
+ drop(pooled_output);
submit_encoder(encoder);
let output_slice: Vec<f32> = allocation_to_vec(&output_buffer);
@@ -263,47 +252,26 @@ fn run_single_pass_attention(
}
fn create_single_pass_kernel(
- context: &<Metal as Backend>::Context,
head_dim: usize,
+ num_q_heads: usize,
+ num_groups: usize,
has_sinks: bool,
is_causal: bool,
-) -> <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel {
- <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
- context,
- DataType::F32,
- head_dim as u32,
- has_sinks,
- false,
- is_causal,
- false,
- false,
- )
- .expect("Failed to create attention single pass kernel")
+) -> super::single_pass::AttentionSinglePass {
+ let mut config = super::default_attention_config::<f32>(head_dim, num_q_heads, num_groups, is_causal);
+ config.has_sinks = has_sinks;
+ config.scale = None;
+ super::single_pass::AttentionSinglePass::new(&config)
}
-fn create_two_pass_kernels(
- context: &<Metal as Backend>::Context,
+fn create_two_pass_kernel(
head_dim: usize,
+ num_q_heads: usize,
+ num_groups: usize,
is_causal: bool,
-) -> (
- <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel,
- <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel,
-) {
- (
- <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
- context,
- DataType::F32,
- head_dim as u32,
- false,
- false,
- is_causal,
- false,
- false,
- )
- .expect("Failed to create AttentionTwoPass1Kernel"),
- <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, DataType::F32, head_dim as u32)
- .expect("Failed to create AttentionTwoPass2Kernel"),
- )
+) -> super::two_pass::AttentionTwoPass {
+ let config = super::default_attention_config::<f32>(head_dim, num_q_heads, num_groups, is_causal);
+ super::two_pass::AttentionTwoPass::new(&config)
}
fn run_gemm_attention(
@@ -318,21 +286,10 @@ fn run_gemm_attention(
let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();
- let core = <<Metal as Backend>::Kernels as Kernels>::AttentionGemmCore::new(
- context,
- &AttentionCoreNewArguments {
- head_dim: head_dim as u32,
- num_groups: num_kv_heads as u32,
- num_q_heads: num_heads as u32,
- has_sinks: sinks.is_some(),
- is_kv_cache_ring: false,
- is_causal,
- is_trie: false,
- sliding_window_size: None,
- scale: Some(scale),
- data_type: DataType::F32,
- },
- )?;
+ let mut config = super::default_attention_config::<f32>(head_dim, num_heads, num_kv_heads, is_causal);
+ config.has_sinks = sinks.is_some();
+ config.scale = Some(scale);
+ let kernel = super::gemm::AttentionGemm::new(&config);
let query_allocation = create_query_allocation(queries, context);
let key_allocation = create_attention_cache_allocation(keys, seq_len, context);
@@ -341,7 +298,7 @@ fn run_gemm_attention(
let sinks_allocation = sinks.map(|sinks| create_sinks_allocation(sinks, context));
let mut encoder = Encoder::new(context).expect("Failed to create encoder");
- let args = AttentionCoreEncodeArguments {
+ let args = AttentionArguments {
queries: &query_allocation,
keys: &key_allocation,
values: &value_allocation,
@@ -351,7 +308,7 @@ fn run_gemm_attention(
cache: KVCacheView::full(0),
};
- let pooled_output = core.encode(args, &mut encoder)?;
+ let pooled_output = kernel.encode(args, &mut encoder)?;
let mut output_allocation = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);
encoder.encode_copy(&pooled_output, .., &mut output_allocation, ..);
let completed = encoder.end_encoding().submit().wait_until_completed()?;
@@ -397,53 +354,13 @@ fn test_single_pass_attention_basic() {
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
- let kernel = create_single_pass_kernel(&context, head_dim, false, false);
+ let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, false, false);
let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)
.expect("single-pass attention");
compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention").unwrap();
}
-#[uzu_test]
-fn test_gemm_attention_basic() {
- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
-
- let batch_size = 1;
- let num_heads = 4;
- let num_kv_heads = 4;
- let seq_len = 32;
- let head_dim = 64;
- let scale = 1.0 / (head_dim as f32).sqrt();
-
- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
-
- let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
- let kernel_output =
- run_gemm_attention(&context, &queries, &keys, &values, None, scale, false).expect("run gemm attention");
-
- compare_results(&kernel_output, &reference_output, 1e-2, "Gemm attention").unwrap();
-}
-
-#[uzu_test]
-fn test_gemm_attention_f32_head_dim_128() {
- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
-
- let batch_size = 1;
- let num_heads = 4;
- let num_kv_heads = 4;
- let seq_len = 32;
- let head_dim = 128;
- let scale = 1.0 / (head_dim as f32).sqrt();
-
- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);
-
- let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
- let kernel_output = run_gemm_attention(&context, &queries, &keys, &values, None, scale, false)
- .expect("run gemm attention f32 head_dim=128");
-
- compare_results(&kernel_output, &reference_output, 1e-2, "Gemm attention f32 head_dim=128").unwrap();
-}
-
#[uzu_test]
fn test_matrix_attention_matches_vector_and_cpu_seq256() {
let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
@@ -459,7 +376,7 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);
let reference_output = reference_attention(&queries, &keys, &values, None, scale, is_causal);
- let kernel = create_single_pass_kernel(&context, head_dim, false, is_causal);
+ let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, false, is_causal);
let vector_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)
.expect("run vector attention");
let matrix_output =
@@ -493,7 +410,7 @@ fn test_single_pass_attention_with_sinks() {
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 444);
let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();
- let kernel = create_single_pass_kernel(&context, head_dim, true, false);
+ let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, true, false);
let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);
let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale)
@@ -516,7 +433,7 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 777);
let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();
- let kernel = create_single_pass_kernel(&context, head_dim, true, false);
+ let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, true, false);
let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);
let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale)
@@ -539,7 +456,7 @@ fn test_single_pass_attention_gqa() {
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- let kernel = create_single_pass_kernel(&context, head_dim, false, false);
+ let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, false, false);
let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)
.expect("single-pass attention GQA");
@@ -548,65 +465,36 @@ fn test_single_pass_attention_gqa() {
}
fn run_two_pass_attention(
- kernel_pass1: &<<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel,
- kernel_pass2: &<<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel,
+ kernel: &super::two_pass::AttentionTwoPass,
context: &<Metal as Backend>::Context,
queries: &Array4<f32>,
keys: &Array4<f32>,
values: &Array4<f32>,
sinks: Option<&[f32]>,
- scale: f32,
+ _scale: f32,
) -> Result<Array4<f32>, Box<dyn std::error::Error>> {
let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
- let (_, num_kv_heads, _, _) = keys.dim();
-
let queries_buffer = create_query_allocation(queries, context);
let keys_buffer = create_attention_cache_allocation(keys, seq_len, context);
let values_buffer = create_attention_cache_allocation(values, seq_len, context);
let sinks_buffer = sinks.map(|sinks| create_sinks_allocation(sinks, context));
-
- let total_blocks_count = 32;
- let partials_size = num_heads * seq_len * total_blocks_count * head_dim;
- let sums_maxs_size = num_heads * seq_len * total_blocks_count;
-
- let mut partials_buffer = alloc_allocation::<Metal, f32>(context, partials_size);
- let mut sums_buffer = alloc_allocation::<Metal, f32>(context, sums_maxs_size);
- let mut maxs_buffer = alloc_allocation::<Metal, f32>(context, sums_maxs_size);
- let mut output_buffer = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);
-
let mut encoder = Encoder::new(context).expect("Failed to create encoder");
- kernel_pass1.encode(
- &queries_buffer,
- &keys_buffer,
- &values_buffer,
- &mut partials_buffer,
- &mut sums_buffer,
- &mut maxs_buffer,
- (num_heads / num_kv_heads) as u32,
- seq_len as u32,
- head_dim as u32,
- (num_kv_heads * head_dim) as u32,
- head_dim as u32,
- (num_kv_heads * head_dim) as u32,
- None,
- scale,
- num_heads as u32,
- seq_len as u32,
- None::<&Allocation<Metal>>,
- None,
- sinks_buffer.as_ref(),
- &mut encoder,
- );
- kernel_pass2.encode(
- &partials_buffer,
- &sums_buffer,
- &maxs_buffer,
- &mut output_buffer,
- num_heads as u32,
- seq_len as u32,
+ let pooled_output = kernel.encode(
+ AttentionArguments {
+ queries: &queries_buffer,
+ keys: &keys_buffer,
+ values: &values_buffer,
+ suffix_length: seq_len as u32,
+ trie: None,
+ sinks: sinks_buffer.as_ref(),
+ cache: KVCacheView::full(0),
+ },
&mut encoder,
- );
+ )?;
+ let mut output_buffer = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);
+ encoder.encode_copy(&pooled_output, .., &mut output_buffer, ..);
+ drop(pooled_output);
submit_encoder(encoder);
let output_slice: Vec<f32> = allocation_to_vec(&output_buffer);
@@ -631,10 +519,9 @@ fn test_two_pass_attention() {
let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
- let (kernel_pass1, kernel_pass2) = create_two_pass_kernels(&context, head_dim, is_causal);
+ let kernel = create_two_pass_kernel(head_dim, num_heads, num_kv_heads, is_causal);
let kernel_output =
- run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale)
- .expect("two-pass attention");
+ run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale).expect("two-pass attention");
compare_results(&kernel_output, &reference_output, 1e-2, "Two-pass attention").unwrap();
}
@@ -655,100 +542,202 @@ fn test_two_pass_attention_gqa() {
let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
- let (kernel_pass1, kernel_pass2) = create_two_pass_kernels(&context, head_dim, is_causal);
- let kernel_output =
- run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale)
- .expect("two-pass attention GQA");
+ let kernel = create_two_pass_kernel(head_dim, num_heads, num_kv_heads, is_causal);
+ let kernel_output = run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)
+ .expect("two-pass attention GQA");
compare_results(&kernel_output, &reference_output, 1e-2, "Two-pass attention GQA").unwrap();
}
-#[tag(heavy)]
+type AttentionShape = (usize, usize, usize, usize, usize, bool);
+
+fn attention_data(shape: AttentionShape) -> (Vec<bf16>, Vec<bf16>) {
+ let (head_dim, _, num_groups, suffix_length, prefix_length, _) = shape;
+ let row = num_groups * head_dim;
+ let count = (prefix_length + suffix_length) * row;
+ (fill_attention(count, 1.0), fill_attention(count, 2.0))
+}
+
+fn fill_attention(
+ count: usize,
+ phase: f32,
+) -> Vec<bf16> {
+ (0..count).map(|i| bf16::from_f32(((i as f32) * 0.017 + phase).sin() * 0.5)).collect()
+}
+
+fn run_attention<B: Backend>(
+ context: &B::Context,
+ shape: AttentionShape,
+ trie_nodes: Option<&[GpuTrieNode]>,
+) -> Vec<bf16> {
+ let (keys, values) = attention_data(shape);
+ let (_, _, _, _, prefix_length, _) = shape;
+ let (head_dim, num_q_heads, num_groups, _, _, is_causal) = shape;
+ let config = super::default_attention_config::<bf16>(head_dim, num_q_heads, num_groups, is_causal);
+ let kernel = <B::Kernels as Kernels>::AttentionKernel::new(context, config).expect("attention kernel");
+ run_attention_with_kernel::<B>(
+ context,
+ &kernel,
+ shape,
+ &keys,
+ &values,
+ KVCacheView::full(prefix_length as u32),
+ trie_nodes,
+ )
+}
+
+fn run_attention_with_kernel<B: Backend>(
+ context: &B::Context,
+ kernel: &<B::Kernels as Kernels>::AttentionKernel,
+ shape: AttentionShape,
+ keys_data: &[bf16],
+ values_data: &[bf16],
+ cache: KVCacheView,
+ trie_nodes: Option<&[GpuTrieNode]>,
+) -> Vec<bf16> {
+ let (head_dim, num_q_heads, _, suffix_length, _, _) = shape;
+ let queries =
+ alloc_allocation_with_data::<B, bf16>(context, &fill_attention(num_q_heads * suffix_length * head_dim, 0.5));
+ let keys = alloc_allocation_with_data::<B, bf16>(context, keys_data);
+ let values = alloc_allocation_with_data::<B, bf16>(context, values_data);
+ let trie = trie_nodes.map(|nodes| {
+ let words: Vec<u32> = nodes.iter().flat_map(|node| [node.trie_start, node.trie_end, node.height]).collect();
+ alloc_allocation_with_data::<B, u32>(context, &words)
+ });
+ let arguments = AttentionArguments {
+ queries: &queries,
+ keys: &keys,
+ values: &values,
+ suffix_length: suffix_length as u32,
+ trie: trie.as_ref(),
+ sinks: None,
+ cache,
+ };
+ let mut encoder = Encoder::<B>::new(context).expect("encoder");
+ let pooled = kernel.encode(arguments, &mut encoder).expect("encode");
+ let mut output = alloc_allocation::<B, bf16>(context, suffix_length * num_q_heads * head_dim);
+ encoder.encode_copy(&pooled, .., &mut output, ..);
+ let completed = encoder.end_encoding().submit().wait_until_completed().expect("submit");
+ drop(pooled);
+ drop(completed);
+ allocation_to_vec::<B, bf16>(&output)
+}
+
#[uzu_test]
-fn perf_two_pass_attention() {
- use std::time::Instant;
+fn attention_kernel_matches_cpu() {
+ let trie: Vec<GpuTrieNode> =
+ TrieNode::flat(0, &[0, 1, 2, 3, 4], &PRng::new(0)).linearize().token_subtrie_ranges().collect();
+ let cpu_context = <Cpu as Backend>::Context::new().expect("CPU attention context");
+ let metal_context = <Metal as Backend>::Context::new().expect("Metal attention context");
+ for &(head_dim, num_q_heads, num_groups, suffix_length, prefix_length, causal, use_trie) in &[
+ (512, 8, 8, 9, 0, false, false),
+ (128, 8, 2, 16, 1024, false, false),
+ (256, 6, 1, 32, 1024, true, false),
+ (256, 6, 1, 31, 1024, true, true),
+ (512, 8, 8, 1, 0, false, false),
+ (512, 8, 8, 1, 1024, false, false),
+ (64, 8, 8, 9, 0, false, false),
+ ] {
+ let nodes = use_trie.then_some(trie.as_slice());
+ let shape = (head_dim, num_q_heads, num_groups, suffix_length, prefix_length, causal);
+ let expected = run_attention::<Cpu>(cpu_context.as_ref(), shape, nodes);
+ let actual = run_attention::<Metal>(metal_context.as_ref(), shape, nodes);
+ let label = format!("attention kernel D{head_dim} S{suffix_length} L{prefix_length} causal={causal}");
+ assert_eq_float::<bf16>(&expected, &actual, 1e-2, &label);
+ }
+}
- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
+#[uzu_test]
+fn attention_kernel_reuses_instance_for_flat_and_trie() {
+ let shape = (256, 6, 1, 5, 1024, true);
+ let trie: Vec<GpuTrieNode> =
+ TrieNode::flat(0, &[0, 1, 2, 3, 4], &PRng::new(0)).linearize().token_subtrie_ranges().collect();
+ let context = <Metal as Backend>::Context::new().expect("Metal attention context");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionKernel::new(
+ context.as_ref(),
+ super::default_attention_config::<bf16>(shape.0, shape.1, shape.2, shape.5),
+ )
+ .expect("attention kernel");
+ let (keys, values) = attention_data(shape);
+ let cache = KVCacheView::full(1024);
+ let flat = run_attention_with_kernel::<Metal>(context.as_ref(), &kernel, shape, &keys, &values, cache, None);
+ let trie_output =
+ run_attention_with_kernel::<Metal>(context.as_ref(), &kernel, shape, &keys, &values, cache, Some(&trie));
+ let cpu_context = <Cpu as Backend>::Context::new().expect("CPU attention context");
+ let cpu_kernel = <<Cpu as Backend>::Kernels as Kernels>::AttentionKernel::new(
+ cpu_context.as_ref(),
+ super::default_attention_config::<bf16>(shape.0, shape.1, shape.2, shape.5),
+ )
+ .expect("CPU attention kernel");
+ let expected_flat =
+ run_attention_with_kernel::<Cpu>(cpu_context.as_ref(), &cpu_kernel, shape, &keys, &values, cache, None);
+ let expected_trie =
+ run_attention_with_kernel::<Cpu>(cpu_context.as_ref(), &cpu_kernel, shape, &keys, &values, cache, Some(&trie));
+ assert_eq_float::<bf16>(&expected_flat, &flat, 1e-2, "flat mask specialization");
+ assert_eq_float::<bf16>(&expected_trie, &trie_output, 1e-2, "trie mask specialization");
+}
- // ---- Problem sizes requiring two-pass ----
- let batch_size = 1;
- let num_heads = 32;
- let num_kv_heads = 32;
- let seq_len = 8192; // Large sequence length (prefix + suffix)
- let suffix_length = 1; // Only processing 1 new token (realistic inference)
- let head_dim = 128;
- let scale = 1.0 / (head_dim as f32).sqrt();
- let is_causal = false;
-
- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
- let (kernel_pass1, kernel_pass2) = create_two_pass_kernels(&context, head_dim, is_causal);
-
- let suffix_start = seq_len - suffix_length;
- let queries_suffix = queries.slice(s![.., .., suffix_start.., ..]).to_owned();
- let queries_buffer = create_query_allocation(&queries_suffix, &context);
- let keys_buffer = create_attention_cache_allocation(&keys, seq_len, &context);
- let values_buffer = create_attention_cache_allocation(&values, seq_len, &context);
-
- let total_blocks_count = 32;
- let partials_size = num_heads * suffix_length * total_blocks_count * head_dim;
- let sums_maxs_size = num_heads * suffix_length * total_blocks_count;
-
- let mut partials_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), partials_size);
- let mut sums_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), sums_maxs_size);
- let mut maxs_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), sums_maxs_size);
- let mut output_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), num_heads * suffix_length * head_dim);
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
-
- let sinks_allocation: Option<Allocation<Metal>> = None;
- kernel_pass1.encode(
- &queries_buffer,
- &keys_buffer,
- &values_buffer,
- &mut partials_buffer,
- &mut sums_buffer,
- &mut maxs_buffer,
- (num_heads / num_kv_heads) as u32,
- seq_len as u32,
- head_dim as u32,
- (num_kv_heads * head_dim) as u32,
- head_dim as u32,
- (num_kv_heads * head_dim) as u32,
- None,
- scale,
- num_heads as u32,
- suffix_length as u32,
- None::<&Allocation<Metal>>,
+#[uzu_test]
+fn attention_kernel_ring_matches_full_on_wrap() {
+ const HEAD_DIM: usize = 64;
+ const NUM_HEADS: usize = 8;
+ const CAPACITY: usize = 8;
+ const CONTEXT: usize = 12;
+ const SUFFIX: usize = 2;
+ let shape = (HEAD_DIM, NUM_HEADS, NUM_HEADS, SUFFIX, CONTEXT, true);
+ let row = NUM_HEADS * HEAD_DIM;
+ let (full_keys, full_values) = attention_data(shape);
+ let mut ring_keys = vec![bf16::ZERO; (CAPACITY + SUFFIX) * row];
+ let mut ring_values = vec![bf16::ZERO; (CAPACITY + SUFFIX) * row];
+ let ring_offset = (CONTEXT - CAPACITY) % CAPACITY;
+ for index in 0..CAPACITY {
+ let source = CONTEXT - CAPACITY + index;
+ let destination = (ring_offset + index) % CAPACITY;
+ ring_keys[destination * row..(destination + 1) * row]
+ .copy_from_slice(&full_keys[source * row..(source + 1) * row]);
+ ring_values[destination * row..(destination + 1) * row]
+ .copy_from_slice(&full_values[source * row..(source + 1) * row]);
+ }
+ for index in 0..SUFFIX {
+ let source = CONTEXT + index;
+ let destination = CAPACITY + index;
+ ring_keys[destination * row..(destination + 1) * row]
+ .copy_from_slice(&full_keys[source * row..(source + 1) * row]);
+ ring_values[destination * row..(destination + 1) * row]
+ .copy_from_slice(&full_values[source * row..(source + 1) * row]);
+ }
+
+ let context = <Metal as Backend>::Context::new().expect("Metal attention context");
+ let full_kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionKernel::new(context.as_ref(), {
+ let mut config = super::default_attention_config::<bf16>(shape.0, shape.1, shape.2, shape.5);
+ config.sliding_window_size = Some(CAPACITY as u32);
+ config
+ })
+ .expect("full attention kernel");
+ let ring_kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionKernel::new(context.as_ref(), {
+ let mut config = super::default_attention_config::<bf16>(shape.0, shape.1, shape.2, shape.5);
+ config.is_kv_cache_ring = true;
+ config.sliding_window_size = Some(CAPACITY as u32);
+ config
+ })
+ .expect("ring attention kernel");
+ let full = run_attention_with_kernel::<Metal>(
+ context.as_ref(),
+ &full_kernel,
+ shape,
+ &full_keys,
+ &full_values,
+ KVCacheView::full(CONTEXT as u32),
None,
- sinks_allocation.as_ref(),
- &mut encoder,
);
- kernel_pass2.encode(
- &partials_buffer,
- &sums_buffer,
- &maxs_buffer,
- &mut output_buffer,
- num_heads as u32,
- suffix_length as u32,
- &mut encoder,
- );
- let host_timer = Instant::now();
- let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
- let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
-
- let gpu_time_ms = completed.gpu_execution_time().as_secs_f64() * 1e3;
- println!(
- "Two-pass attention perf (heads={}, prefix={}, suffix={}, head_dim={}): GPU={:.2} ms, Host-side={:.2} ms",
- num_heads,
- seq_len - suffix_length,
- suffix_length,
- head_dim,
- gpu_time_ms,
- host_elapsed_ms
+ let ring = run_attention_with_kernel::<Metal>(
+ context.as_ref(),
+ &ring_kernel,
+ shape,
+ &ring_keys,
+ &ring_values,
+ KVCacheView::ring(CAPACITY as u32, ring_offset as u32),
+ None,
);
-
- let output_slice: Vec<f32> = allocation_to_vec(&output_buffer);
- for &val in output_slice.iter().take(100) {
- assert!(val.is_finite(), "Output contains non-finite values");
- }
+ assert_eq_float::<bf16>(&full, &ring, 1e-2, "wrapped ring attention");
}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs
new file mode 100644
index 000000000..888095db5
--- /dev/null
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs
@@ -0,0 +1,26 @@
+#![cfg(backend = "metal")]
+
+use super::{gemm, single_pass, two_pass};
+use crate::{array::ArrayElement, backends::common::kernel::AttentionKernelConfig};
+
+mod gemm_test;
+mod kernel_test;
+
+fn default_attention_config<T: ArrayElement>(
+ head_dim: usize,
+ num_q_heads: usize,
+ num_groups: usize,
+ is_causal: bool,
+) -> AttentionKernelConfig {
+ AttentionKernelConfig {
+ head_dim: head_dim as u32,
+ num_groups: num_groups as u32,
+ num_q_heads: num_q_heads as u32,
+ has_sinks: false,
+ is_kv_cache_ring: false,
+ is_causal,
+ sliding_window_size: None,
+ scale: Some(1.0 / (head_dim as f32).sqrt()),
+ data_type: T::data_type(),
+ }
+}