A high-performance inference engine for AI models
Add AttentionGemmGrouped (#765)
CCYeh8af3f886d60e
diff --git a/crates/uzu-engine/src/backends/common/kernel/attention.rs b/crates/uzu-engine/src/backends/common/kernel/attention.rsnew file mode 100644index 000000000..6311c8a29--- /dev/null+++ b/crates/uzu-engine/src/backends/common/kernel/attention.rs@@ -0,0 +1,46 @@+use crate::{+ backends::common::{Allocation, Backend, BufferArg, Encoder, Kernels},+ data_type::DataType,+ encodable_block::mixer::attention::KVCacheView,+};++pub trait AttentionKernel: Sized + Send + Sync {+ type Backend: Backend<Kernels: Kernels<AttentionKernel = Self>>;++ fn new(+ context: &<Self::Backend as Backend>::Context,+ config: AttentionKernelConfig,+ ) -> Result<Self, <Self::Backend as Backend>::Error>;++ fn encode<'a, KT, VT>(+ &self,+ arguments: AttentionArguments<'a, Self::Backend, KT, VT>,+ encoder: &mut Encoder<Self::Backend>,+ ) -> Result<Allocation<Self::Backend>, <Self::Backend as Backend>::Error>+ where+ KT: BufferArg<'a, Self::Backend>,+ VT: BufferArg<'a, Self::Backend>;+}++#[derive(Clone, Copy)]+pub struct AttentionKernelConfig {+ pub head_dim: u32,+ pub num_groups: u32,+ pub num_q_heads: u32,+ pub has_sinks: bool,+ pub is_kv_cache_ring: bool,+ pub is_causal: bool,+ pub sliding_window_size: Option<u32>,+ pub scale: Option<f32>,+ pub data_type: DataType,+}++pub struct AttentionArguments<'a, B: Backend, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>> {+ pub queries: &'a Allocation<B>,+ pub keys: KT,+ pub values: VT,+ pub suffix_length: u32,+ pub trie: Option<&'a Allocation<B>>,+ pub sinks: Option<&'a Allocation<B>>,+ pub cache: KVCacheView,+}diff --git a/crates/uzu-engine/src/backends/common/kernel/attention_gemm/kernel.rs b/crates/uzu-engine/src/backends/common/kernel/attention_gemm/kernel.rsdeleted file mode 100644index ce758b8f1..000000000--- a/crates/uzu-engine/src/backends/common/kernel/attention_gemm/kernel.rs+++ /dev/null@@ -1,53 +0,0 @@-use crate::{- backends::common::{- Allocation, Backend, BufferArg, Encoder,- kernel::{Kernels, Unsupported},- },- encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},-};--pub trait AttentionGemmCore: Sized + Send + Sync {- type Backend: Backend<Kernels: Kernels<AttentionGemmCore = Self>>;-- fn is_supported(- arguments: &AttentionCoreNewArguments,- context: &<Self::Backend as Backend>::Context,- ) -> Result<bool, <Self::Backend as Backend>::Error>;-- fn new(- context: &<Self::Backend as Backend>::Context,- arguments: &AttentionCoreNewArguments,- ) -> Result<Self, <Self::Backend as Backend>::Error>;-- fn encode<'a, KT: BufferArg<'a, Self::Backend>, VT: BufferArg<'a, Self::Backend>>(- &self,- arguments: AttentionCoreEncodeArguments<'a, Self::Backend, KT, VT>,- encoder: &mut Encoder<Self::Backend>,- ) -> Result<Allocation<Self::Backend>, <Self::Backend as Backend>::Error>;-}--impl<B: Backend<Kernels: Kernels<AttentionGemmCore = Unsupported<B>>>> AttentionGemmCore for Unsupported<B> {- type Backend = B;-- fn is_supported(- _arguments: &AttentionCoreNewArguments,- _context: &B::Context,- ) -> Result<bool, B::Error> {- Ok(false)- }-- fn new(- _context: &B::Context,- _arguments: &AttentionCoreNewArguments,- ) -> Result<Self, B::Error> {- unreachable!("unsupported attention core should not be constructed")- }-- fn encode<'a, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>>(- &self,- _arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,- _encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, B::Error> {- match self.never {}- }-}diff --git a/crates/uzu-engine/src/backends/common/kernel/attention_gemm/mod.rs b/crates/uzu-engine/src/backends/common/kernel/attention_gemm/mod.rsdeleted file mode 100644index 608e559cc..000000000--- a/crates/uzu-engine/src/backends/common/kernel/attention_gemm/mod.rs+++ /dev/null@@ -1,3 +0,0 @@-mod kernel;--pub use kernel::AttentionGemmCore;diff --git a/crates/uzu-engine/src/backends/common/kernel/mod.rs b/crates/uzu-engine/src/backends/common/kernel/mod.rsindex ea0b5ea2e..365a80af5 100644--- a/crates/uzu-engine/src/backends/common/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/common/kernel/mod.rs@@ -3,7 +3,7 @@ use std::{convert::Infallible, marker::PhantomData};use crate::backends::common::Backend;pub mod activation_transform;-pub mod attention_gemm;+pub mod attention;pub mod delta_net_chunked_prefill;pub mod delta_net_tree_verify;pub mod gated_act_mul;@@ -11,6 +11,7 @@ pub mod matmul;pub mod radix_top_k_small;pub use activation_transform::ActivationTransform;+pub use attention::{AttentionArguments, AttentionKernel, AttentionKernelConfig};pub use gated_act_mul::{GatedActMul, GatedActMulSettings};include!(concat!(env!("OUT_DIR"), "/traits.rs"));@@ -19,7 +20,7 @@ pub trait Kernels: Sized {type Backend: Backend<Kernels = Self>;autogen_kernels!();- type AttentionGemmCore: attention_gemm::AttentionGemmCore<Backend = Self::Backend>;+ type AttentionKernel: attention::AttentionKernel<Backend = Self::Backend>;type DeltaNetChunkedPrefill: delta_net_chunked_prefill::DeltaNetChunkedPrefill<Backend = Self::Backend>;type DeltaNetTreeVerify: delta_net_tree_verify::DeltaNetTreeVerify<Backend = Self::Backend>;type MatmulKernel: matmul::MatmulKernel<Backend = Self::Backend>;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_fallback.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_fallback.rsdeleted file mode 100644index 842e3b020..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_fallback.rs+++ /dev/null@@ -1,108 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::{- array::ArrayElement,- backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},-};--#[kernel(AttentionFallbackScatterScores)]-#[variants(T, f32, f16, bf16)]-pub fn attention_fallback_scatter_scores<T: ArrayElement + Float>(- group_scores: *const T,- scores: *mut T,- #[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,- #[optional(is_trie)] trie: Option<*const TrieNode>,- #[optional(is_sliding_window)] sliding_window_size: Option<u32>,- group_index: u32,- gqa_factor: u32,- sequence_length: u32,- suffix_length: u32,- total_elements: u32,- #[specialize] is_kv_cache_ring: bool,- #[specialize] is_causal: bool,- #[specialize] is_trie: bool,- #[specialize] is_sliding_window: bool,-) {- assert_eq!(ring_params.is_some(), is_kv_cache_ring);- assert_eq!(sliding_window_size.is_some(), is_sliding_window);- let expected_total_elements = gqa_factor- .checked_mul(suffix_length)- .and_then(|elements| elements.checked_mul(sequence_length))- .expect("scatter scores element count overflow");- assert_eq!(total_elements, expected_total_elements);-- let prefix_length = sequence_length - suffix_length;- let suffix_position = ring_params.map(|params| params.ring_length).unwrap_or(prefix_length);-- for head_in_group in 0..gqa_factor {- let head_index = group_index * gqa_factor + head_in_group;- for query_index in 0..suffix_length {- let query_position = if is_trie {- let trie_node = unsafe { &*trie.unwrap().add(query_index as usize) };- suffix_position + trie_node.height- } else {- suffix_position + query_index- };-- for sequence_index in 0..sequence_length {- let source_index =- ((head_in_group * suffix_length + query_index) * sequence_length + sequence_index) as usize;- let destination_index =- ((head_index * suffix_length + query_index) * sequence_length + sequence_index) as usize;- let use_key = should_use_key(- ring_params,- trie,- sliding_window_size,- query_index,- prefix_length,- suffix_position,- query_position,- sequence_index,- is_causal,- );- unsafe {- *scores.add(destination_index) = if use_key {- *group_scores.add(source_index)- } else {- T::from(f32::NEG_INFINITY).unwrap()- };- }- }- }- }-}--#[kernel(AttentionFallbackScatterValues)]-#[variants(T, f32, f16, bf16)]-pub fn attention_fallback_scatter_values<T: ArrayElement + Float>(- group_output: *const T,- out: *mut T,- group_index: u32,- gqa_factor: u32,- suffix_length: u32,- num_heads: u32,- head_dim: u32,- total_elements: u32,-) {- let expected_total_elements = gqa_factor- .checked_mul(suffix_length)- .and_then(|elements| elements.checked_mul(head_dim))- .expect("scatter values element count overflow");- assert_eq!(total_elements, expected_total_elements);-- for head_in_group in 0..gqa_factor as usize {- for query_index in 0..suffix_length as usize {- for dim_index in 0..head_dim as usize {- let source_index =- (head_in_group * suffix_length as usize + query_index) * head_dim as usize + dim_index;- let head_index = group_index as usize * gqa_factor as usize + head_in_group;- let destination_index = (query_index * num_heads as usize + head_index) * head_dim as usize + dim_index;- unsafe {- *out.add(destination_index) = *group_output.add(source_index);- }- }- }- }-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rsindex 75eb1a955..dea58b22b 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_single_pass.rs@@ -1,4 +1,4 @@-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;@@ -8,7 +8,7 @@ use crate::{};#[kernel(AttentionSinglePass)]-#[variants(T, f32, f16, bf16)]+#[variants(T, f32, bf16)]#[variants(HEAD_DIM, 64, 128, 256, 512)]pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(queries: *const T,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_two_pass.rsdeleted file mode 100644index e696d7cdc..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/attention_two_pass.rs+++ /dev/null@@ -1,190 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::{- array::ArrayElement,- backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},-};--const TOTAL_BLOCKS_COUNT: u32 = 32;--#[kernel(AttentionTwoPass1)]-#[variants(T, f32, f16, bf16)]-#[variants(HEAD_DIM, 64, 128, 256, 512)]-pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(- queries: *const T,- keys: *const T,- values: *const T,- out: *mut f32,- sums: *mut f32,- maxs: *mut f32,- gqa_factor: u32,- sequence_length: u32,- k_head_stride: u32,- k_seq_stride: u32,- v_head_stride: u32,- v_seq_stride: u32,- #[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,- scale: f32,- num_heads: u32,- suffix_length: u32,- #[optional(is_trie)] trie: Option<*const TrieNode>,- #[optional(is_sliding_window)] sliding_window_size: Option<u32>,- #[optional(has_sinks)] sinks: Option<*const T>,- #[specialize] has_sinks: bool,- #[specialize] is_kv_cache_ring: bool,- #[specialize] is_causal: bool,- #[specialize] is_trie: bool,- #[specialize] is_sliding_window: bool,-) {- assert_eq!(ring_params.is_some(), is_kv_cache_ring);- assert_eq!(sliding_window_size.is_some(), is_sliding_window);-- let value_dim = HEAD_DIM;- let mut q = vec![0.0f32; HEAD_DIM as usize];- let mut o = vec![0.0f32; HEAD_DIM as usize];-- let prefix_length = sequence_length - suffix_length;- let suffix_position = if let Some(ring_params) = ring_params {- ring_params.ring_length- } else {- prefix_length- };-- for head_idx in 0..num_heads {- for q_seq_idx in 0..suffix_length {- let query_position = if is_trie {- let trie_node = unsafe { &*trie.unwrap().add(q_seq_idx as usize) };- suffix_position + trie_node.height- } else {- suffix_position + q_seq_idx- };-- for block_idx in 0..TOTAL_BLOCKS_COUNT {- let o_offset = q_seq_idx * num_heads + head_idx;- let q_offset = head_idx * suffix_length + q_seq_idx;- let kv_head_idx = head_idx / gqa_factor;-- let queries_base: *const T = unsafe { queries.add((q_offset * HEAD_DIM) as usize) };- let keys_base: *const T = unsafe { keys.add((kv_head_idx * k_head_stride) as usize) };- let values_base: *const T = unsafe { values.add((kv_head_idx * v_head_stride) as usize) };- let out_base: *mut f32 =- unsafe { out.add((o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim) as usize) };-- // Read the query and scale- for j in 0..HEAD_DIM as usize {- q[j] = scale * unsafe { *queries_base.add(j) }.to_f32().unwrap();- }- o.fill(0f32);-- let mut max_score = -1e9f32;- let mut sum_exp_score = 0.0f32;-- if has_sinks && block_idx == 0 {- max_score = unsafe { *sinks.unwrap().add(head_idx as usize) }.to_f32().unwrap();- sum_exp_score = 1.0;- }-- // Iterate over sequence positions assigned to this block- let mut i = block_idx;- while i < sequence_length {- if should_use_key(- ring_params,- trie,- sliding_window_size,- q_seq_idx,- prefix_length,- suffix_position,- query_position,- i,- is_causal,- ) {- let keys_ptr = unsafe { keys_base.add((i * k_seq_stride) as usize) };-- // Compute the i-th score- let mut score = 0.0f32;- for j in 0..HEAD_DIM as usize {- score += q[j] * unsafe { *keys_ptr.add(j) }.to_f32().unwrap();- }-- // Update the accumulators- let new_max = f32::max(max_score, score);- let factor = (max_score - new_max).exp();- let exp_score = (score - new_max).exp();-- max_score = new_max;- sum_exp_score = sum_exp_score * factor + exp_score;-- // Update the output accumulator- let values_ptr = unsafe { values_base.add((i * v_seq_stride) as usize) };- for j in 0..HEAD_DIM as usize {- o[j] = o[j] * factor + exp_score * unsafe { *values_ptr.add(j) }.to_f32().unwrap();- }- }- i += TOTAL_BLOCKS_COUNT;- }-- // Write partial output, sum, and max for this block- for j in 0..HEAD_DIM as usize {- unsafe {- *out_base.add(j) = o[j];- }- }- unsafe {- *sums.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) = sum_exp_score;- *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) = max_score;- }- }- }- }-}--#[kernel(AttentionTwoPass2)]-#[variants(T, f32, f16, bf16)]-#[variants(HEAD_DIM, 64, 128, 256, 512)]-pub fn attention_two_pass2<T: ArrayElement + Float, const HEAD_DIM: u32>(- partials: *const f32,- sums: *const f32,- maxs: *const f32,- out: *mut T,- num_heads: u32,- suffix_length: u32,-) {- for head_idx in 0..num_heads {- for q_seq_idx in 0..suffix_length {- let o_offset = q_seq_idx * num_heads + head_idx;-- // Find global max across all blocks- let mut global_max = f32::NEG_INFINITY;- for block_idx in 0..TOTAL_BLOCKS_COUNT {- let max_val = unsafe { *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };- global_max = f32::max(global_max, max_val);- }-- // Compute global sum of exponentials with correction- let mut global_sum = 0.0f32;- for block_idx in 0..TOTAL_BLOCKS_COUNT {- let sum_val = unsafe { *sums.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };- let max_val = unsafe { *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };- global_sum += sum_val * (max_val - global_max).exp();- }-- // Combine partial outputs- let out_ptr = unsafe { out.add((o_offset * HEAD_DIM) as usize) };- for j in 0..HEAD_DIM as usize {- let mut val = 0.0f32;- for block_idx in 0..TOTAL_BLOCKS_COUNT {- let partial_val = unsafe {- *partials.add((o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM + block_idx * HEAD_DIM) as usize + j)- };- let max_val = unsafe { *maxs.add((o_offset * TOTAL_BLOCKS_COUNT + block_idx) as usize) };- val += partial_val * (max_val - global_max).exp();- }- unsafe {- *out_ptr.add(j) = T::from(val / global_sum).unwrap();- }- }- }- }-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rsindex 026a4486b..736c515c3 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/kv_cache_update.rs@@ -1,11 +1,11 @@-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;use crate::array::ArrayElement;#[kernel(KVCacheUpdate)]-#[variants(T, f32, bf16, f16)]+#[variants(T, f32, bf16)]pub fn kv_cache_update<T: ArrayElement + Float>(in_place_keys: *mut T,in_place_values: *mut T,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rsindex 052bec54a..dc9d18668 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/mod.rs@@ -1,10 +1,83 @@pub mod ancestor_attention;-pub mod attention_fallback;pub mod attention_prepare;pub mod attention_single_pass;-pub mod attention_two_pass;pub mod kv_cache_update;pub mod qkv_norm;pub mod sigmoid_gate;mod mask;++use crate::backends::{+ common::{+ Allocation, BufferArg, Encoder,+ kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig, AttentionSinglePassKernel},+ },+ cpu::{Cpu, context::CpuContext, error::CpuError},+};++pub struct AttentionCpuKernel {+ config: AttentionKernelConfig,+}++impl AttentionKernel for AttentionCpuKernel {+ type Backend = Cpu;++ fn new(+ _context: &CpuContext,+ config: AttentionKernelConfig,+ ) -> Result<Self, CpuError> {+ Ok(Self {+ config,+ })+ }++ fn encode<'a, KT, VT>(+ &self,+ arguments: AttentionArguments<'a, Cpu, KT, VT>,+ encoder: &mut Encoder<Cpu>,+ ) -> Result<Allocation<Cpu>, CpuError>+ where+ KT: BufferArg<'a, Cpu>,+ VT: BufferArg<'a, Cpu>,+ {+ let config = self.config;+ let mut output = encoder.allocate_constant_for_shape(+ &[arguments.suffix_length, config.num_q_heads, config.head_dim],+ config.data_type,+ )?;++ let single_pass = <attention_single_pass::AttentionSinglePassCpuKernel as AttentionSinglePassKernel>::new(+ encoder.context(),+ config.data_type,+ config.head_dim,+ config.has_sinks,+ config.is_kv_cache_ring,+ config.is_causal,+ arguments.trie.is_some(),+ config.sliding_window_size.is_some(),+ )?;+ encoder.push_debug_group("attention core");+ single_pass.encode(+ arguments.queries,+ arguments.keys,+ arguments.values,+ &mut output,+ config.num_q_heads / config.num_groups,+ arguments.cache.prefix_len() + arguments.suffix_length,+ config.head_dim,+ config.num_groups * config.head_dim,+ config.head_dim,+ config.num_groups * config.head_dim,+ arguments.cache.ring_params(),+ config.scale.unwrap_or(1.0 / (config.head_dim as f32).sqrt()),+ arguments.trie,+ config.sliding_window_size,+ arguments.sinks,+ config.num_q_heads,+ arguments.suffix_length,+ encoder,+ );+ encoder.pop_debug_group();+ Ok(output)+ }+}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rsindex 4a37c2061..007fd160a 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/qkv_norm.rs@@ -1,14 +1,14 @@-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;use crate::array::ArrayElement;#[kernel(QKVNorm)]-#[variants(InputT, f32, f16, bf16)]-#[variants(ScaleT, f32, f16, bf16)]-#[variants(OutputT, f32, f16, bf16)]-#[variants(AccumT, f32, f16)]+#[variants(InputT, f32, bf16)]+#[variants(ScaleT, f32, bf16)]+#[variants(OutputT, f32, bf16)]+#[variants(AccumT, f32)]pub fn qkv_norm<InputT: ArrayElement + Float,ScaleT: ArrayElement + Float,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rsindex 6bf4883fc..dd3a07e25 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/sigmoid_gate.rs@@ -1,11 +1,11 @@-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use uzu_engine_macros::kernel;use crate::array::ArrayElement;#[kernel(SigmoidGate)]-#[variants(T, f32, f16, bf16)]+#[variants(T, f32, bf16)]pub fn sigmoid_gate<T: ArrayElement + Float>(gate: *const T,output: *mut T,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/mod.rsindex fe72506a6..5cbd2a504 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs@@ -33,7 +33,7 @@ impl Kernels for CpuKernels {type Backend = Cpu;autogen_kernels!();- type AttentionGemmCore = Unsupported<Cpu>;+ type AttentionKernel = attention::AttentionCpuKernel;type DeltaNetChunkedPrefill = Unsupported<Cpu>;type DeltaNetTreeVerify = Unsupported<Cpu>;type MatmulKernel = matmul::MatmulCpuKernel;diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metalindex 14e2e12b2..dbb45dfe0 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal@@ -11,8 +11,8 @@ using namespace uzu::ring;using namespace uzu::trie;template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AttentionFallbackScatterScores)(+VARIANTS(T, float, bfloat)+KERNEL(AttentionFallbackScatterScores)(const device T* group_scores,device T* scores,const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),@@ -59,8 +59,8 @@ PUBLIC KERNEL(AttentionFallbackScatterScores)(}template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AttentionFallbackScatterValues)(+VARIANTS(T, float, bfloat)+KERNEL(AttentionFallbackScatterValues)(const device T* group_output,device T* out,const constant uint& group_index,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metalindex 3010a678a..b31d7b054 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal@@ -37,7 +37,7 @@ struct AttentionGemmLayout {};template <typename T, uint BK, uint BD, bool USE_MXU>-VARIANTS(T, float, half, bfloat)+VARIANTS(T, float, bfloat)VARIANTS(BK, 16, 32)VARIANTS(BD, 64, 128, 256)VARIANTS(USE_MXU, false, true)diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metalindex 923a9a135..42869cc1d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -13,7 +13,7 @@ using namespace uzu::ring;using namespace uzu::trie;template <typename T, uint HEAD_DIM>-VARIANTS(T, float, half, bfloat)+VARIANTS(T, float, bfloat)VARIANTS(HEAD_DIM, 64, 128, 256, 512)PUBLIC KERNEL(AttentionSinglePass)(const device T* queries,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metalindex a88f9433e..3594bcbce 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -18,9 +18,9 @@ using namespace uzu::trie;// Each simdgroup independently handles one query head and derives its KV head// via integer division. Simdgroups sharing a KV head benefit from L1 cache.template <typename T, uint HEAD_DIM>-VARIANTS(T, float, half, bfloat)+VARIANTS(T, float, bfloat)VARIANTS(HEAD_DIM, 64, 128, 256, 512)-PUBLIC KERNEL(AttentionTwoPass1)(+KERNEL(AttentionTwoPass1)(const device T* queries,const device T* keys,const device T* values,@@ -139,9 +139,9 @@ PUBLIC KERNEL(AttentionTwoPass1)(}template <typename T, uint HEAD_DIM>-VARIANTS(T, float, half, bfloat)+VARIANTS(T, float, bfloat)VARIANTS(HEAD_DIM, 64, 128, 256, 512)-PUBLIC KERNEL(AttentionTwoPass2)(+KERNEL(AttentionTwoPass2)(const device float* partials,const device float* sums,const device float* maxs,diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rssimilarity index 62%rename from crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rsrename to crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rsindex bb97d7057..589d67806 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/fallback.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs@@ -1,62 +1,77 @@use parking_lot::Mutex;use crate::{- backends::common::{- Allocation, Backend, BufferArg, Encoder, Kernels,- kernel::{- AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, SoftmaxKernel,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},+ backends::{+ common::{+ Allocation, BufferArg, Encoder, Kernels,+ kernel::{+ AttentionArguments, AttentionKernelConfig, SoftmaxKernel,+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},+ },+ },+ metal::{+ Metal,+ context::MetalContext,+ error::MetalError,+ kernel::{+ AttentionFallbackScatterScoresMetalKernel, AttentionFallbackScatterValuesMetalKernel, MetalKernels,+ },},},data_type::DataType,- encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},};-pub struct AttentionFallbackCore<B: Backend> {+const HEAD_DIM: u32 = 512;++pub struct AttentionFallback {head_dim: u32,num_groups: u32,num_q_heads: u32,sliding_window_size: Option<u32>,scale: Option<f32>,data_type: DataType,- scatter_scores: <B::Kernels as Kernels>::AttentionFallbackScatterScoresKernel,- scatter_values: <B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel,- softmax: <B::Kernels as Kernels>::SoftmaxKernel,- matmul: Mutex<<B::Kernels as Kernels>::MatmulKernel>,+ scatter_scores: AttentionFallbackScatterScoresMetalKernel,+ scatter_values: AttentionFallbackScatterValuesMetalKernel,+ softmax: <MetalKernels as Kernels>::SoftmaxKernel,+ matmul: Mutex<<MetalKernels as Kernels>::MatmulKernel>,}-impl<B: Backend> AttentionFallbackCore<B> {- pub fn new(- arguments: &AttentionCoreNewArguments,- context: &B::Context,- ) -> Result<Self, B::Error> {- assert!(!arguments.is_trie, "trie not supported by attention fallback"); // Is it?+impl AttentionFallback {+ pub fn is_supported(config: &AttentionKernelConfig) -> bool {+ config.head_dim == HEAD_DIM && matches!(config.data_type, DataType::BF16 | DataType::F32)+ }- let scatter_scores = <B::Kernels as Kernels>::AttentionFallbackScatterScoresKernel::new(+ pub fn new(+ config: &AttentionKernelConfig,+ context: &MetalContext,+ ) -> Result<Self, MetalError> {+ let scatter_scores = AttentionFallbackScatterScoresMetalKernel::new(+ context,+ config.data_type,+ config.is_kv_cache_ring,+ config.is_causal,+ false,+ config.sliding_window_size.is_some(),+ )?;+ let scatter_values = AttentionFallbackScatterValuesMetalKernel::new(context, config.data_type)?;+ let softmax = <<MetalKernels as Kernels>::SoftmaxKernel as SoftmaxKernel>::new(context,- arguments.data_type,- arguments.is_kv_cache_ring,- arguments.is_causal,- arguments.is_trie,- arguments.sliding_window_size.is_some(),+ config.data_type,+ config.has_sinks,)?;- let scatter_values =- <B::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(context, arguments.data_type)?;- let softmax = <B::Kernels as Kernels>::SoftmaxKernel::new(context, arguments.data_type, arguments.has_sinks)?;- let matmul = Mutex::new(<B::Kernels as Kernels>::MatmulKernel::new(+ let matmul = Mutex::new(<<MetalKernels as Kernels>::MatmulKernel as MatmulKernel>::new(context,- arguments.data_type,- arguments.data_type,- arguments.data_type,+ config.data_type,+ config.data_type,+ config.data_type,)?);-Ok(Self {- head_dim: arguments.head_dim,- num_groups: arguments.num_groups,- num_q_heads: arguments.num_q_heads,- sliding_window_size: arguments.sliding_window_size,- scale: arguments.scale,- data_type: arguments.data_type,+ head_dim: config.head_dim,+ num_groups: config.num_groups,+ num_q_heads: config.num_q_heads,+ sliding_window_size: config.sliding_window_size,+ scale: config.scale,+ data_type: config.data_type,scatter_scores,scatter_values,softmax,@@ -64,20 +79,19 @@ impl<B: Backend> AttentionFallbackCore<B> {})}- pub fn encode<'a, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>>(+ pub fn encode<'a, KT: BufferArg<'a, Metal>, VT: BufferArg<'a, Metal>>(&self,- arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,- encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, B::Error> {+ arguments: AttentionArguments<'a, Metal, KT, VT>,+ encoder: &mut Encoder<Metal>,+ ) -> Result<Allocation<Metal>, MetalError> {+ assert!(arguments.trie.is_none(), "fallback does not support trie");let suffix_length = arguments.suffix_length;let sequence_length = arguments.cache.prefix_len() + suffix_length;let gqa_factor = self.num_q_heads / self.num_groups;- let scale = self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt());-+ let scale = self.scale.unwrap_or(1.0 / (self.head_dim as f32).sqrt());let dt_bytes = self.data_type.size_in_bytes();let head_dim_bytes = self.head_dim as usize * dt_bytes;let group_rows = (gqa_factor * suffix_length) as usize;-let mut output =encoder.allocate_constant_for_shape(&[suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;let mut scores =@@ -113,7 +127,7 @@ impl<B: Backend> AttentionFallbackCore<B> {&group_scores,&mut scores,arguments.cache.ring_params(),- None::<&Allocation<B>>,+ None::<&Allocation<Metal>>,self.sliding_window_size,group_index,gqa_factor,@@ -125,10 +139,8 @@ impl<B: Backend> AttentionFallbackCore<B> {}self.softmax.encode(&mut scores, arguments.sinks, sequence_length, self.num_q_heads, suffix_length, encoder);-let mut group_output =encoder.allocate_scratch_for_shape(&[gqa_factor * suffix_length, self.head_dim], self.data_type)?;-for group_index in 0..self.num_groups {self.matmul.lock().encode(MatmulArguments {@@ -162,7 +174,6 @@ impl<B: Backend> AttentionFallbackCore<B> {encoder,);}-Ok(output)}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rssimilarity index 71%rename from crates/uzu-engine/src/backends/metal/kernel/attention.rsrename to crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rsindex dfb95d2b6..3501e1a6d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs@@ -4,14 +4,17 @@ use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};use crate::{backends::{- common::{Allocation, BufferArg, Encoder, gpu_types::AttnParams, kernel::attention_gemm::AttentionGemmCore},+ common::{+ Allocation, BufferArg, Encoder,+ gpu_types::AttnParams,+ kernel::{AttentionArguments, AttentionKernelConfig},+ },metal::{Metal, context::MetalContext, error::MetalError, kernel::AttentionGemmMetalKernel},},data_type::DataType,- encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},};-pub struct AttentionGemmMetalCore {+pub struct AttentionGemm {kernels: Mutex<HashMap<AttentionGemmKey, AttentionGemmMetalKernel>>,head_dim: u32,num_groups: u32,@@ -22,7 +25,6 @@ pub struct AttentionGemmMetalCore {simd_bk: u32,is_kv_cache_ring: bool,is_causal: bool,- is_trie: bool,is_sliding_window: bool,has_sinks: bool,}@@ -32,6 +34,7 @@ struct AttentionGemmKey {use_mxu: bool,align_q: bool,align_k: bool,+ is_trie: bool,}fn retile_params(@@ -47,7 +50,33 @@ fn retile_params(params}-impl AttentionGemmMetalCore {+impl AttentionGemm {+ pub fn is_supported(config: &AttentionKernelConfig) -> bool {+ matches!(config.head_dim, 64 | 128 | 256) && matches!(config.data_type, DataType::BF16 | DataType::F32)+ }++ pub fn new(config: &AttentionKernelConfig) -> Self {+ let simd_bk = if config.head_dim < 128 {+ 32+ } else {+ 16+ };+ Self {+ kernels: Mutex::new(HashMap::new()),+ head_dim: config.head_dim,+ num_groups: config.num_groups,+ num_q_heads: config.num_q_heads,+ sliding_window_size: config.sliding_window_size,+ scale: config.scale,+ data_type: config.data_type,+ simd_bk,+ is_kv_cache_ring: config.is_kv_cache_ring,+ is_causal: config.is_causal,+ is_sliding_window: config.sliding_window_size.is_some(),+ has_sinks: config.has_sinks,+ }+ }+fn get_or_create(&self,context: &MetalContext,@@ -70,7 +99,7 @@ impl AttentionGemmMetalCore {key.align_k,self.is_kv_cache_ring,self.is_causal,- self.is_trie,+ key.is_trie,self.is_sliding_window,self.has_sinks,)?;@@ -78,48 +107,10 @@ impl AttentionGemmMetalCore {}Ok(MutexGuard::map(kernels, |kernels| kernels.get_mut(&key).expect("kernel was just initialized")))}-}-impl AttentionGemmCore for AttentionGemmMetalCore {- type Backend = Metal;-- fn is_supported(- arguments: &AttentionCoreNewArguments,- _context: &MetalContext,- ) -> Result<bool, MetalError> {- Ok(matches!(arguments.head_dim, 64 | 128 | 256))- }-- fn new(- _context: &MetalContext,- arguments: &AttentionCoreNewArguments,- ) -> Result<Self, MetalError> {- let simd_bk = if arguments.head_dim < 128 {- 32- } else {- 16- };-- Ok(Self {- kernels: Mutex::new(HashMap::new()),- head_dim: arguments.head_dim,- num_groups: arguments.num_groups,- num_q_heads: arguments.num_q_heads,- sliding_window_size: arguments.sliding_window_size,- scale: arguments.scale,- data_type: arguments.data_type,- simd_bk,- is_kv_cache_ring: arguments.is_kv_cache_ring,- is_causal: arguments.is_causal,- is_trie: arguments.is_trie,- is_sliding_window: arguments.sliding_window_size.is_some(),- has_sinks: arguments.has_sinks,- })- }-- fn encode<'a, KT: BufferArg<'a, Metal>, VT: BufferArg<'a, Metal>>(+ pub fn encode<'a, KT: BufferArg<'a, Metal>, VT: BufferArg<'a, Metal>>(&self,- arguments: AttentionCoreEncodeArguments<'a, Metal, KT, VT>,+ arguments: AttentionArguments<'a, Metal, KT, VT>,encoder: &mut Encoder<Metal>,) -> Result<Allocation<Metal>, MetalError> {let mut output = encoder@@ -127,7 +118,7 @@ impl AttentionGemmCore for AttentionGemmMetalCore {let use_mxu = arguments.suffix_length >= 64&& encoder.context().supports_mxu()- && matches!(self.data_type, DataType::BF16 | DataType::F16)+ && matches!(self.data_type, DataType::BF16)&& matches!(self.head_dim, 64 | 128);let (bq, bk) = if use_mxu {(64, 32)@@ -141,7 +132,7 @@ impl AttentionGemmCore for AttentionGemmMetalCore {v_strides: [0, self.head_dim, self.num_groups * self.head_dim],o_strides: [0, self.head_dim, self.num_q_heads * self.head_dim],gqa_factor: self.num_q_heads / self.num_groups,- scale: self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),+ scale: self.scale.unwrap_or(1.0 / (self.head_dim as f32).sqrt()),q_len: arguments.suffix_length,k_len: arguments.cache.prefix_len() + arguments.suffix_length,q_off: arguments.cache.prefix_len(),@@ -158,9 +149,9 @@ impl AttentionGemmCore for AttentionGemmMetalCore {use_mxu,align_q: params.q_rem == 0,align_k: params.k_rem == 0,+ is_trie: arguments.trie.is_some(),};let kernel = self.get_or_create(encoder.context(), key)?;-kernel.encode(arguments.queries,arguments.keys,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metalnew file mode 100644index 000000000..8478518d5--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_gemm_grouped.metal@@ -0,0 +1,452 @@+#include <metal_stdlib>+#include "../../common/dsl.h"+#include "../../common/integral_constant.h"+#include "../../common/thread_context.h"+#include "../../matmul/common/fragment.h"+#include "../../matmul/common/loader.h"+#include "../../matmul/common/mxu_fragment/ops.h"+#include "../../generated/attention.h"+#include "../../generated/trie.h"++using namespace metal;+using namespace uzu::matmul;+using namespace uzu::attention;+using namespace uzu::trie;++#define ATTENTION_GQA_SIMDGROUPS 8++template <typename T, uint BK, uint BD, uint D_SLICES>+struct AttentionGemmGroupedLayout {+ using Ops = MxuFragmentOps<>;++ UZU_CONST int SIMDGROUPS = ATTENTION_GQA_SIMDGROUPS;+ UZU_CONST int TILE_ROWS = Ops::FRAGMENT_ROWS;+ UZU_CONST int SLICE_COLS = int(BD) / int(D_SLICES);+ UZU_CONST int BANDS = SIMDGROUPS / int(D_SLICES);+ UZU_CONST int BLOCK_ROWS = BANDS * TILE_ROWS;+ UZU_CONST int SCORE_TILE_ELEMENTS = TILE_ROWS * int(BK);+ UZU_CONST int SCORE_EXCHANGE_SIZE = SIMDGROUPS * SCORE_TILE_ELEMENTS;+};++template <class FragmentType, class Fn>+METAL_FUNC void map_lane_rows(thread FragmentType& tile, const short2 position, Fn fn) {+ using Ops = typename FragmentType::FragmentOpsType;+ using Element = typename FragmentType::ElementType;+ static_assert(FragmentType::ROW_FRAGMENTS == 1, "requires one row fragment");+ static_assert(+ Ops::THREAD_ELEMENT_ROWS * Ops::THREAD_ELEMENT_COLS == Ops::ELEMENTS_PER_THREAD,+ "requires row-major thread elements"+ );++ thread Element* data = tile.elements();+ METAL_PRAGMA_UNROLL+ for (ushort fragment_col = 0; fragment_col < FragmentType::COL_FRAGMENTS; ++fragment_col) {+ const ushort fragment_base = fragment_col * Ops::ELEMENTS_PER_THREAD;+ const short col_base = position.x + short(fragment_col) * short(Ops::FRAGMENT_COLS);+ METAL_PRAGMA_UNROLL+ for (ushort row = 0; row < Ops::THREAD_ELEMENT_ROWS; ++row) {+ METAL_PRAGMA_UNROLL+ for (ushort col = 0; col < Ops::THREAD_ELEMENT_COLS; ++col) {+ const ushort element = fragment_base + row * Ops::THREAD_ELEMENT_COLS + col;+ data[element] = Element(fn(row, col_base + short(col), data[element]));+ }+ }+ }+}++// TODO: support more BD and should tune more to replace AttentionGEMM+template <typename T, uint BK, uint BD, uint D_SLICES>+VARIANTS(T, bfloat)+VARIANTS(BK, 32)+VARIANTS(BD, 128, 256)+VARIANTS(D_SLICES, 2, 4)+CONSTRAINT(D_SLICES == BD / 64)+KERNEL(AttentionGemmGrouped)(+ const device T* q,+ const device T* k,+ const device T* v,+ device T* o OPTIONAL(!split_kv),+ device float* partials OPTIONAL(split_kv),+ device float* partial_maxs OPTIONAL(split_kv),+ device float* partial_sums OPTIONAL(split_kv),+ const device TrieNode* trie OPTIONAL(is_trie),+ const constant AttnParams& params,+ const constant uint& m_tiles,+ const constant uint& kv_heads,+ const constant uint& batch_count,+ const constant uint& num_splits,+ const bool split_kv SPECIALIZE,+ const bool is_causal SPECIALIZE,+ const bool is_trie SPECIALIZE,+ threadgroup float score_exchange[AttentionGemmGroupedLayout<T, BK, BD, D_SLICES>::SCORE_EXCHANGE_SIZE],+ const ThreadContext thread_context,+ const uint tile_split_flat GROUPS(m_tiles * num_splits),+ const uint kv_head_idx GROUPS(kv_heads),+ const uint batch_idx GROUPS(batch_count),+ const uint lid THREADS(ATTENTION_GQA_SIMDGROUPS * METAL_SIMD_SIZE)+) {+ using Layout = AttentionGemmGroupedLayout<T, BK, BD, D_SLICES>;+ using Ops = typename Layout::Ops;+ using AccumType = float;++ constexpr int TILE_ROWS = Layout::TILE_ROWS;+ constexpr int SLICE_COLS = Layout::SLICE_COLS;+ constexpr int BLOCK_ROWS = Layout::BLOCK_ROWS;+ constexpr int SCORE_TILE_ELEMENTS = Layout::SCORE_TILE_ELEMENTS;+ constexpr int KEY_COL_FRAGMENTS = int(BK) / Ops::FRAGMENT_COLS;+ constexpr int HEAD_DIM_FRAGMENTS_PER_SLICE = SLICE_COLS / Ops::FRAGMENT_ROWS;+ constexpr int VALUE_COL_FRAGMENTS = SLICE_COLS / Ops::FRAGMENT_COLS;+ constexpr int ROWS_PER_LANE = Ops::THREAD_ELEMENT_ROWS;++ static_assert(Layout::SIMDGROUPS == 8, "requires eight simdgroups");+ static_assert(KEY_COL_FRAGMENTS % 2 == 0, "QK fragments must be even");+ static_assert(VALUE_COL_FRAGMENTS % 2 == 0, "PV fragments must be even");++ using QueryFragment = OperandFragment<T, 1, 1, Ops>;+ using KeyFragment = OperandFragment<T, 1, KEY_COL_FRAGMENTS, Ops, ReadTranspose>;+ using ValueFragment = OperandFragment<T, KEY_COL_FRAGMENTS, VALUE_COL_FRAGMENTS, Ops>;+ using ScoreFragment = Fragment<AccumType, 1, KEY_COL_FRAGMENTS, Ops>;+ using OutputFragment = Fragment<AccumType, 1, VALUE_COL_FRAGMENTS, Ops>;++ const ushort lane = ushort(thread_context.simd_lane_id);+ const ushort simdgroup_index = ushort(thread_context.simdgroup_index);+ const ushort band = simdgroup_index / ushort(D_SLICES);+ const ushort slice = simdgroup_index % ushort(D_SLICES);++ const uint m_tile_idx = split_kv ? (tile_split_flat % m_tiles) : tile_split_flat;+ const uint split_idx = split_kv ? (tile_split_flat / m_tiles) : 0u;++ const uint grouped_rows = params.gqa_factor * params.q_len;+ const uint row_base = m_tile_idx * uint(BLOCK_ROWS) + uint(band) * uint(TILE_ROWS);+ const uint head_in_group = row_base / params.q_len;+ const uint query_base = row_base - head_in_group * params.q_len;+ const uint q_head_idx = kv_head_idx * params.gqa_factor + head_in_group;++ const uint query_row_stride = params.q_strides[2];+ const uint key_row_stride = params.k_strides[2];+ const uint value_row_stride = params.v_strides[2];+ const uint output_row_stride = params.o_strides[2];+ const uint slice_offset = uint(slice) * uint(SLICE_COLS);++ q += size_t(batch_idx) * params.q_strides[0] + size_t(q_head_idx) * params.q_strides[1] ++ size_t(query_base) * query_row_stride + slice_offset;+ k += size_t(batch_idx) * params.k_strides[0] + size_t(kv_head_idx) * params.k_strides[1] + slice_offset;+ v += size_t(batch_idx) * params.v_strides[0] + size_t(kv_head_idx) * params.v_strides[1] + slice_offset;++ QueryFragment query_frags[HEAD_DIM_FRAGMENTS_PER_SLICE];+ {+ auto query_source = fragment_source(q, int(query_row_stride));+ const uint valid_rows = row_base < grouped_rows ? grouped_rows - row_base : 0u;+ if (valid_rows < uint(TILE_ROWS)) {+ query_source = query_source.bounded(short(valid_rows), short(Ops::FRAGMENT_COLS));+ }+ METAL_PRAGMA_UNROLL+ for (short head_dim_fragment_idx = 0; head_dim_fragment_idx < HEAD_DIM_FRAGMENTS_PER_SLICE;+ ++head_dim_fragment_idx) {+ query_frags[head_dim_fragment_idx].load_from(+ lane,+ query_source.advanced(head_dim_fragment_idx * Ops::FRAGMENT_ROWS)+ );+ }+ }++ OutputFragment output;+ output.clear();++ AccumType max_score[ROWS_PER_LANE];+ AccumType sum_score[ROWS_PER_LANE];+ METAL_PRAGMA_UNROLL+ for (int r = 0; r < ROWS_PER_LANE; ++r) {+ max_score[r] = -INFINITY;+ sum_score[r] = AccumType(0);+ }++ const uint prefix_length = params.q_off;+ const uint key_length = params.k_len;+ const short2 score_position = ScoreFragment::get_position(lane);+ uint last_visible_key[ROWS_PER_LANE];+ ulong row_trie_mask[ROWS_PER_LANE];+ METAL_PRAGMA_UNROLL+ for (ushort r = 0; r < ROWS_PER_LANE; ++r) {+ last_visible_key[r] = 0;+ row_trie_mask[r] = 0ul;+ }+ if (is_causal || is_trie) {+ METAL_PRAGMA_UNROLL+ for (ushort r = 0; r < ROWS_PER_LANE; ++r) {+ const uint row = row_base + uint(score_position.y) + uint(r) * uint(Ops::THREAD_ELEMENT_ROW_STRIDE);+ last_visible_key[r] = prefix_length + row % params.q_len;+ }+ if (is_trie) {+ const ulong all_suffix_keys = (params.q_len >= 64u) ? ~0ul : ((1ul << params.q_len) - 1ul);+ METAL_PRAGMA_UNROLL+ for (ushort r = 0; r < ROWS_PER_LANE; ++r) {+ row_trie_mask[r] = is_causal ? 0ul : all_suffix_keys;+ }+ if (is_causal) {+ for (uint node_index = 0; node_index < params.q_len; ++node_index) {+ const TrieNode node = trie[node_index];+ METAL_PRAGMA_UNROLL+ for (ushort r = 0; r < ROWS_PER_LANE; ++r) {+ const uint query_index = last_visible_key[r] - prefix_length;+ if (query_index >= node.trie_start && query_index <= node.trie_end) {+ row_trie_mask[r] |= (1ul << node_index);+ }+ }+ }+ }+ }+ }++ threadgroup float* exchange_slot = score_exchange + simdgroup_index * SCORE_TILE_ELEMENTS;+ const ushort band_exchange_base = band * ushort(D_SLICES);+ const uint key_block_stride = uint(BK) * key_row_stride;+ const uint value_block_stride = uint(BK) * value_row_stride;++ const uint block_begin = split_kv ? (split_idx * params.nk) / num_splits : 0u;+ uint block_end = split_kv ? ((split_idx + 1u) * params.nk) / num_splits : params.nk;++ if (is_causal) {+ const uint tile_first_row = m_tile_idx * uint(BLOCK_ROWS);+ const uint tile_last_row = min(tile_first_row + uint(BLOCK_ROWS) - 1u, grouped_rows - 1u);+ const uint max_query_index = (tile_first_row / params.q_len == tile_last_row / params.q_len)+ ? (tile_last_row % params.q_len)+ : (params.q_len - 1u);+ const uint causal_blocks = (uint(prefix_length) + max_query_index + uint(BK)) / uint(BK);+ block_end = min(block_end, causal_blocks);+ }++ const uint tail_keys = params.k_rem;+ const uint block_count = block_end > block_begin ? block_end - block_begin : 0u;+ const bool owns_tail_block = (tail_keys != 0u) && (block_count > 0u) && (block_end == params.nk);+ const uint full_blocks = owns_tail_block ? block_count - 1u : block_count;++ k += size_t(block_begin) * key_block_stride;+ v += size_t(block_begin) * value_block_stride;++ const AccumType score_scale = AccumType(params.scale) * AccumType(M_LOG2E_F);+ // Masked scores are multiplied by score_scale below, so pre-divide this value.+ const AccumType masked_score_pre_scale = (static_cast<AccumType>(-1e9f) * M_LOG2E_F) / score_scale;+ const uint first_key = block_begin * uint(BK);++ auto accumulate_kv_block = [&](const uint kb, auto tail_flag) {+ constexpr bool IS_TAIL = decltype(tail_flag)::value;++ ScoreFragment score;+ score.clear();+ {+ auto key_source = fragment_source(k + size_t(kb) * key_block_stride, int(key_row_stride));+ if constexpr (IS_TAIL) {+ key_source = key_source.bounded(short(tail_keys), short(Ops::FRAGMENT_COLS));+ }+ METAL_PRAGMA_UNROLL+ for (short head_dim_fragment_idx = 0; head_dim_fragment_idx < HEAD_DIM_FRAGMENTS_PER_SLICE;+ ++head_dim_fragment_idx) {+ KeyFragment key_fragment;+ key_fragment.load_from(lane, key_source.advanced(head_dim_fragment_idx * Ops::FRAGMENT_ROWS));+ simdgroup_barrier(mem_flags::mem_none);+ fragment_mma(score, query_frags[head_dim_fragment_idx], key_fragment);+ }+ }++ if constexpr (D_SLICES > 1) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ score.store(lane, exchange_slot, int(BK));+ threadgroup_barrier(mem_flags::mem_threadgroup);++ thread AccumType* score_elements = score.elements();+ METAL_PRAGMA_UNROLL+ for (ushort j = 1; j < ushort(D_SLICES); ++j) {+ const ushort other = (slice + j) % ushort(D_SLICES);+ ScoreFragment partial;+ partial.load_from(+ lane,+ fragment_source(score_exchange + (band_exchange_base + other) * SCORE_TILE_ELEMENTS, int(BK))+ );+ thread AccumType* partial_elements = partial.elements();+ METAL_PRAGMA_UNROLL+ for (ushort e = 0; e < ScoreFragment::ELEMENTS_PER_FRAGMENT; ++e) {+ score_elements[e] += partial_elements[e];+ }+ }+ }++ const uint key_base = first_key + kb * uint(BK);+ const bool masked_block = IS_TAIL || ((is_causal || is_trie) && (key_base + uint(BK) > prefix_length));+ if (masked_block) {+ map_lane_rows(score, score_position, [&](ushort row, short col, AccumType value) {+ const uint key = key_base + uint(col);+ bool visible = key < key_length;+ if (is_trie) {+ visible =+ visible && (key < prefix_length || ((row_trie_mask[row] >> ulong(key - prefix_length)) & 1ul) != 0ul);+ } else if (is_causal) {+ visible = visible && (key <= last_visible_key[row]);+ }+ return visible ? value : masked_score_pre_scale;+ });+ }++ AccumType block_max[ROWS_PER_LANE];+ score.row_reduce(block_max, -INFINITY, [](AccumType a, AccumType b) { return metal::max(a, b); });+ METAL_PRAGMA_UNROLL+ for (int r = 0; r < ROWS_PER_LANE; ++r) {+ block_max[r] *= score_scale;+ }++ AccumType new_max[ROWS_PER_LANE];+ AccumType rescale[ROWS_PER_LANE];+ METAL_PRAGMA_UNROLL+ for (int r = 0; r < ROWS_PER_LANE; ++r) {+ new_max[r] = metal::max(max_score[r], block_max[r]);+ rescale[r] = fast::exp2(max_score[r] - new_max[r]);+ max_score[r] = new_max[r];+ sum_score[r] *= rescale[r];+ }++ score.map_rows(new_max, [score_scale](AccumType value, AccumType m) {+ return fast::exp2(value * score_scale - m);+ });++ AccumType block_sum[ROWS_PER_LANE];+ score.row_reduce(block_sum, AccumType(0), [](AccumType a, AccumType b) { return a + b; });+ METAL_PRAGMA_UNROLL+ for (int r = 0; r < ROWS_PER_LANE; ++r) {+ sum_score[r] += block_sum[r];+ }++ output.map_rows(rescale, [](AccumType value, AccumType f) { return value * f; });++ {+ auto value_source = fragment_source(v + size_t(kb) * value_block_stride, int(value_row_stride));+ if constexpr (IS_TAIL) {+ value_source = value_source.bounded(short(tail_keys), short(SLICE_COLS));+ }+ ValueFragment value_fragment;+ value_fragment.load_from(lane, value_source);+ simdgroup_barrier(mem_flags::mem_none);+ fragment_mma(output, score, value_fragment);+ }+ };++ for (uint kb = 0; kb < full_blocks; ++kb) {+ accumulate_kv_block(kb, uzu::false_type{});+ }+ if (owns_tail_block) {+ accumulate_kv_block(full_blocks, uzu::true_type{});+ }++ if (split_kv) {+ const uint tile_index = ((batch_idx * kv_heads + kv_head_idx) * m_tiles + m_tile_idx) * num_splits + split_idx;+ device float* partial_tile = partials + size_t(tile_index) * BLOCK_ROWS * BD + size_t(band) * TILE_ROWS * BD;+ output.store(lane, partial_tile + slice_offset, int(BD));++ if (slice == 0) {+ const short2 position = OutputFragment::get_position(lane);+ if (position.x == 0) {+ const uint row_base_in_tile = uint(tile_index) * uint(BLOCK_ROWS) + uint(band) * uint(TILE_ROWS);+ METAL_PRAGMA_UNROLL+ for (int r = 0; r < ROWS_PER_LANE; ++r) {+ const uint row = row_base_in_tile + uint(position.y + r * Ops::THREAD_ELEMENT_ROW_STRIDE);+ partial_maxs[row] = max_score[r];+ partial_sums[row] = sum_score[r];+ }+ }+ }+ } else {+ AccumType inverse_sum[ROWS_PER_LANE];+ METAL_PRAGMA_UNROLL+ for (int r = 0; r < ROWS_PER_LANE; ++r) {+ inverse_sum[r] = AccumType(1) / sum_score[r];+ }+ output.map_rows(inverse_sum, [](AccumType value, AccumType s) { return value * s; });++ device T* output_base = o + size_t(batch_idx) * params.o_strides[0];+ if (params.q_len % uint(TILE_ROWS) == 0u) {+ if (row_base < grouped_rows) {+ device T* destination = output_base + size_t(q_head_idx) * params.o_strides[1] ++ size_t(query_base) * output_row_stride + slice_offset;+ output.store(lane, destination, int(output_row_stride));+ }+ } else {+ const short2 position = OutputFragment::get_position(lane);+ thread AccumType* output_elements = output.elements();+ METAL_PRAGMA_UNROLL+ for (ushort r = 0; r < ROWS_PER_LANE; ++r) {+ const uint row = row_base + uint(position.y) + uint(r) * uint(Ops::THREAD_ELEMENT_ROW_STRIDE);+ if (row < grouped_rows) {+ const uint row_head = row / params.q_len;+ const uint row_position = row - row_head * params.q_len;+ device T* destination = output_base ++ (size_t(kv_head_idx) * params.gqa_factor + row_head) * params.o_strides[1] ++ size_t(row_position) * output_row_stride + slice_offset + position.x;+ METAL_PRAGMA_UNROLL+ for (ushort fragment_col = 0; fragment_col < VALUE_COL_FRAGMENTS; ++fragment_col) {+ METAL_PRAGMA_UNROLL+ for (ushort col = 0; col < Ops::THREAD_ELEMENT_COLS; ++col) {+ const ushort element = fragment_col * Ops::ELEMENTS_PER_THREAD + r * Ops::THREAD_ELEMENT_COLS + col;+ destination[fragment_col * Ops::FRAGMENT_COLS + col] = static_cast<T>(output_elements[element]);+ }+ }+ }+ }+ }+ }+}++template <typename T, uint BD>+VARIANTS(T, bfloat)+VARIANTS(BD, 128, 256)+KERNEL(AttentionGemmGroupedCombine)(+ const device float* partials,+ const device float* partial_maxs,+ const device float* partial_sums,+ device T* o,+ const constant AttnParams& params,+ const constant uint& grouped_rows,+ const constant uint& block_rows,+ const constant uint& m_tiles,+ const constant uint& kv_heads,+ const constant uint& batch_count,+ const constant uint& num_splits,+ const uint grouped_row_idx GROUPS(grouped_rows),+ const uint kv_head_idx GROUPS(kv_heads),+ const uint batch_idx GROUPS(batch_count),+ const uint column THREADS(BD)+) {+ const uint m_tile_idx = grouped_row_idx / block_rows;+ const uint row_in_tile = grouped_row_idx - m_tile_idx * block_rows;+ const uint tile_base = ((batch_idx * kv_heads + kv_head_idx) * m_tiles + m_tile_idx) * num_splits;++ const device float* row_maxs = partial_maxs + size_t(tile_base) * block_rows + row_in_tile;+ const device float* row_sums = partial_sums + size_t(tile_base) * block_rows + row_in_tile;+ const device float* row_partials = partials + size_t(tile_base) * block_rows * BD + size_t(row_in_tile) * BD + column;++ float global_max = -INFINITY;+ for (uint s = 0; s < num_splits; ++s) {+ if (row_sums[s * block_rows] > 0.0f) {+ global_max = metal::max(global_max, row_maxs[s * block_rows]);+ }+ }++ float numerator = 0.0f;+ float denominator = 0.0f;+ if (global_max > -INFINITY) {+ for (uint s = 0; s < num_splits; ++s) {+ const float split_sum = row_sums[s * block_rows];+ if (split_sum <= 0.0f) {+ continue;+ }+ const float rescale = fast::exp2(row_maxs[s * block_rows] - global_max);+ denominator += rescale * split_sum;+ numerator += rescale * row_partials[size_t(s) * block_rows * BD];+ }+ }++ const uint head_in_group = grouped_row_idx / params.q_len;+ const uint query_position = grouped_row_idx - head_in_group * params.q_len;+ const uint q_head_idx = kv_head_idx * params.gqa_factor + head_in_group;+ o += size_t(batch_idx) * params.o_strides[0] + size_t(q_head_idx) * params.o_strides[1] ++ size_t(query_position) * params.o_strides[2] + column;+ o[0] = static_cast<T>(denominator > 0.0f ? numerator / denominator : 0.0f);+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rsnew file mode 100644index 000000000..14fd5ebbc--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs@@ -0,0 +1,289 @@+use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};++use crate::{+ backends::{+ common::{+ Allocation, BufferArg, BufferArgMut, Encoder,+ gpu_types::AttnParams,+ kernel::{AttentionArguments, AttentionKernelConfig},+ },+ metal::{+ Metal,+ context::MetalContext,+ error::MetalError,+ kernel::{AttentionGemmGroupedCombineMetalKernel, AttentionGemmGroupedMetalKernel},+ },+ },+ data_type::DataType,+};++mod policy;+use policy::{MAX_TRIE_SUFFIX, choose_splits};++use super::MaskKind;++const SIMDGROUPS_PER_THREADGROUP: u32 = 8;+const SLICE_COLS: u32 = 64;+const TILE_ROWS: u32 = 16;+const BLOCK_K: u32 = 32;++struct AttentionGemmGroupedMetal {+ kernel: AttentionGemmGroupedMetalKernel,+ split_kernel: AttentionGemmGroupedMetalKernel,+ combine: AttentionGemmGroupedCombineMetalKernel,+ head_dim: u32,+ num_groups: u32,+ num_q_heads: u32,+ scale: Option<f32>,+ block_rows: u32,+ mask: MaskKind,+ gpu_core_count: u32,+}++impl AttentionGemmGroupedMetal {+ fn new_fixed(+ context: &MetalContext,+ head_dim: u32,+ num_groups: u32,+ num_q_heads: u32,+ scale: Option<f32>,+ mask: MaskKind,+ ) -> Result<Self, MetalError> {+ assert!(matches!(head_dim, 128 | 256), "head_dim must be 128 or 256");+ assert_eq!(num_q_heads % num_groups, 0, "num_q_heads must be divisible by num_groups");+ let slices = head_dim / SLICE_COLS;+ let block_rows = SIMDGROUPS_PER_THREADGROUP / slices * TILE_ROWS;+ let new_kernel = |split| {+ AttentionGemmGroupedMetalKernel::new(+ context,+ DataType::BF16,+ BLOCK_K,+ head_dim,+ slices,+ split,+ mask.is_causal(),+ mask.is_trie(),+ )+ };+ let combine = AttentionGemmGroupedCombineMetalKernel::new(context, DataType::BF16, head_dim)?;+ Ok(Self {+ kernel: new_kernel(false)?,+ split_kernel: new_kernel(true)?,+ combine,+ head_dim,+ num_groups,+ num_q_heads,+ scale,+ block_rows,+ mask,+ gpu_core_count: context.device_profile().gpu_core_count(),+ })+ }++ #[allow(clippy::too_many_arguments)]+ fn encode<'q, 'k, 'v, 'o, 't>(+ &self,+ queries: impl BufferArg<'q, Metal>,+ keys: impl BufferArg<'k, Metal>,+ values: impl BufferArg<'v, Metal>,+ output: impl BufferArgMut<'o, Metal>,+ trie: Option<impl BufferArg<'t, Metal>>,+ suffix_length: u32,+ kv_length: u32,+ q_replicas: u32,+ num_splits: u32,+ encoder: &mut Encoder<Metal>,+ ) -> Result<(), MetalError> {+ let params = self.params(suffix_length, kv_length);+ let grouped_rows = self.num_q_heads / self.num_groups * suffix_length;+ let m_tiles = grouped_rows.div_ceil(self.block_rows);+ assert_eq!(trie.is_some(), self.mask.is_trie(), "trie presence must match mask");+ assert!(num_splits >= 1, "num_splits must be at least one");+ assert!(num_splits <= params.nk, "num_splits must not exceed KV blocks");++ if num_splits == 1 {+ self.kernel.encode(+ queries,+ keys,+ values,+ Some(output),+ None::<&mut Allocation<Metal>>,+ None::<&mut Allocation<Metal>>,+ None::<&mut Allocation<Metal>>,+ trie,+ params,+ m_tiles,+ self.num_groups,+ q_replicas,+ 1,+ encoder,+ );+ return Ok(());+ }++ let partial_rows = q_replicas * self.num_groups * m_tiles * num_splits * self.block_rows;+ let mut partials = encoder.allocate_scratch_for_shape(&[partial_rows, self.head_dim], DataType::F32)?;+ let mut partial_maxs = encoder.allocate_scratch_for_shape(&[partial_rows], DataType::F32)?;+ let mut partial_sums = encoder.allocate_scratch_for_shape(&[partial_rows], DataType::F32)?;++ self.split_kernel.encode(+ queries,+ keys,+ values,+ None::<&mut Allocation<Metal>>,+ Some(&mut partials),+ Some(&mut partial_maxs),+ Some(&mut partial_sums),+ trie,+ params,+ m_tiles,+ self.num_groups,+ q_replicas,+ num_splits,+ encoder,+ );++ self.combine.encode(+ &partials,+ &partial_maxs,+ &partial_sums,+ output,+ params,+ grouped_rows,+ self.block_rows,+ m_tiles,+ self.num_groups,+ q_replicas,+ num_splits,+ encoder,+ );+ Ok(())+ }++ fn params(+ &self,+ suffix_length: u32,+ kv_length: u32,+ ) -> AttnParams {+ assert!(suffix_length > 0 && kv_length > 0, "lengths must be positive");+ assert!(kv_length >= suffix_length, "kv_length must be at least suffix_length");+ assert!(!self.mask.is_trie() || suffix_length <= MAX_TRIE_SUFFIX, "trie suffix is too long");++ let head_dim = self.head_dim;+ AttnParams {+ q_strides: [self.num_q_heads * suffix_length * head_dim, suffix_length * head_dim, head_dim],+ k_strides: [0, head_dim, self.num_groups * head_dim],+ v_strides: [0, head_dim, self.num_groups * head_dim],+ o_strides: [suffix_length * self.num_q_heads * head_dim, head_dim, self.num_q_heads * head_dim],+ gqa_factor: self.num_q_heads / self.num_groups,+ scale: self.scale.unwrap_or(1.0f32 / (head_dim as f32).sqrt()),+ q_len: suffix_length,+ k_len: kv_length,+ q_off: kv_length - suffix_length,+ nq_aligned: 0,+ q_rem: 0,+ nk: kv_length.div_ceil(BLOCK_K),+ nk_aligned: kv_length / BLOCK_K,+ k_rem: kv_length % BLOCK_K,+ }+ }+}++pub struct AttentionGemmGrouped {+ non_trie: Mutex<Option<AttentionGemmGroupedMetal>>,+ trie: Mutex<Option<AttentionGemmGroupedMetal>>,+ head_dim: u32,+ num_groups: u32,+ num_q_heads: u32,+ scale: Option<f32>,+}++impl AttentionGemmGrouped {+ fn get_or_create(+ &self,+ context: &MetalContext,+ mask: MaskKind,+ ) -> Result<MappedMutexGuard<'_, AttentionGemmGroupedMetal>, MetalError> {+ let cache = if mask.is_trie() {+ &self.trie+ } else {+ &self.non_trie+ };+ let mut cache = cache.lock();+ if cache.is_none() {+ *cache = Some(AttentionGemmGroupedMetal::new_fixed(+ context,+ self.head_dim,+ self.num_groups,+ self.num_q_heads,+ self.scale,+ mask,+ )?);+ }+ Ok(MutexGuard::map(cache, |cache| cache.as_mut().expect("attention pipeline was just initialized")))+ }++ pub fn is_supported(+ config: &AttentionKernelConfig,+ context: &MetalContext,+ ) -> bool {+ policy::is_supported(config, context)+ }++ pub fn new(config: &AttentionKernelConfig) -> Self {+ Self {+ non_trie: Mutex::new(None),+ trie: Mutex::new(None),+ head_dim: config.head_dim,+ num_groups: config.num_groups,+ num_q_heads: config.num_q_heads,+ scale: config.scale,+ }+ }++ pub fn should_encode(+ &self,+ mask: MaskKind,+ suffix_length: u32,+ kv_length: u32,+ ) -> bool {+ policy::should_encode(self.head_dim, mask, suffix_length, kv_length)+ }++ pub fn encode<'a, KT: BufferArg<'a, Metal>, VT: BufferArg<'a, Metal>>(+ &self,+ mask: MaskKind,+ arguments: AttentionArguments<'a, Metal, KT, VT>,+ encoder: &mut Encoder<Metal>,+ ) -> Result<Allocation<Metal>, MetalError> {+ let suffix_length = arguments.suffix_length;+ assert!(arguments.cache.ring_params().is_none(), "ring KV cache is unsupported");+ assert!(arguments.sinks.is_none(), "attention sinks are unsupported");++ let kv_length = arguments.cache.prefix_len() + suffix_length;+ let mut output =+ encoder.allocate_constant_for_shape(&[suffix_length, self.num_q_heads, self.head_dim], DataType::BF16)?;+ let core = self.get_or_create(encoder.context(), mask)?;+ let num_splits = choose_splits(+ core.head_dim,+ suffix_length,+ kv_length,+ (core.num_q_heads / core.num_groups * suffix_length).div_ceil(core.block_rows) * core.num_groups,+ BLOCK_K,+ core.gpu_core_count,+ );+ core.encode(+ arguments.queries,+ arguments.keys,+ arguments.values,+ &mut output,+ arguments.trie,+ suffix_length,+ kv_length,+ 1,+ num_splits,+ encoder,+ )?;+ Ok(output)+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rsnew file mode 100644index 000000000..b759f5964--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs@@ -0,0 +1,112 @@+use super::MaskKind;+use crate::{+ backends::{common::kernel::AttentionKernelConfig, metal::context::MetalContext},+ data_type::DataType,+};++const GEMM_GROUPED_HEAD_DIMS: [u32; 2] = [128, 256];+const GEMM_GROUPED_DECODE_SUFFIX_MIN: u32 = 16;+const GEMM_GROUPED_DECODE_SUFFIX_MAX: u32 = 64;+const GEMM_GROUPED_PREFILL_SUFFIX_MAX: u32 = 1024;+const GEMM_GROUPED_MIN_KV_LENGTH: u32 = 1024;+pub const MAX_TRIE_SUFFIX: u32 = 64;++pub fn is_supported(+ arguments: &AttentionKernelConfig,+ context: &MetalContext,+) -> bool {+ GEMM_GROUPED_HEAD_DIMS.contains(&arguments.head_dim)+ && arguments.data_type == DataType::BF16+ && context.supports_mxu()+ && !arguments.has_sinks+ && !arguments.is_kv_cache_ring+ && arguments.sliding_window_size.is_none()+ && arguments.scale.is_none_or(|scale| scale > 0.0)+ && arguments.num_groups > 0+ && arguments.num_q_heads.is_multiple_of(arguments.num_groups)+}++pub fn should_encode(+ head_dim: u32,+ mask: MaskKind,+ suffix_length: u32,+ kv_length: u32,+) -> bool {+ kv_length >= GEMM_GROUPED_MIN_KV_LENGTH+ && ((GEMM_GROUPED_DECODE_SUFFIX_MIN..=GEMM_GROUPED_DECODE_SUFFIX_MAX).contains(&suffix_length)+ || (head_dim == 256+ && mask == MaskKind::Causal+ && (GEMM_GROUPED_DECODE_SUFFIX_MAX + 1..=GEMM_GROUPED_PREFILL_SUFFIX_MAX).contains(&suffix_length)))+}++type MeasuredSplits = (u32, u32, &'static [(u32, u32)]);++const MEASURED_SUFFIX_MIN: u32 = 16;+const MEASURED_SUFFIX_MAX: u32 = 64;++// TODO: validate this table per chip (M1-M5) before changing the policy.+// TODO: add a simdgroup/non-MXU implementation before widening availability.+// TODO: measure the D128/D256 crossover before widening suffix ranges.+const MEASURED_TG_PER_CORE_TENTHS: &[MeasuredSplits] = &[+ (256, 16, &[(0, 24), (5120, 18), (131072, 39)]),+ (256, 32, &[(0, 24), (5120, 18), (32768, 60)]),+ (256, 64, &[(0, 24), (5120, 60)]),+ (128, 16, &[(0, 16), (51200, 40)]),+ (128, 32, &[(0, 16), (5120, 40), (51200, 80)]),+ (128, 64, &[(0, 32), (5120, 40), (32768, 80)]),+];+const TG_PER_CORE_FALLBACK: u32 = 6;+const TENTHS_PER_RATIO: u32 = 10;++fn tg_per_core_tenths_for(+ steps: &[(u32, u32)],+ kv_length: u32,+) -> u32 {+ steps.iter().rev().find(|(minimum_kv, _)| kv_length >= *minimum_kv).map_or(steps[0].1, |(_, tenths)| *tenths)+}++fn splits_for_ratio(+ tg_per_core_tenths: u32,+ gpu_core_count: u32,+ unsplit_threadgroups: u32,+) -> u32 {+ (tg_per_core_tenths * gpu_core_count.max(1)).div_ceil(TENTHS_PER_RATIO * unsplit_threadgroups.max(1))+}++fn tabled_splits(+ head_dim: u32,+ suffix_length: u32,+ kv_length: u32,+ unsplit_threadgroups: u32,+ gpu_core_count: u32,+) -> Option<u32> {+ if !(MEASURED_SUFFIX_MIN..=MEASURED_SUFFIX_MAX).contains(&suffix_length) {+ return None;+ }+ MEASURED_TG_PER_CORE_TENTHS+ .iter()+ .filter(|(row_head_dim, _, _)| *row_head_dim == head_dim)+ .min_by_key(|(_, row_suffix, _)| (row_suffix.abs_diff(suffix_length), u32::MAX - row_suffix))+ .map(|(_, _, steps)| {+ splits_for_ratio(tg_per_core_tenths_for(steps, kv_length), gpu_core_count, unsplit_threadgroups)+ })+}++pub fn choose_splits(+ head_dim: u32,+ suffix_length: u32,+ kv_length: u32,+ unsplit_threadgroups: u32,+ block_k: u32,+ gpu_core_count: u32,+) -> u32 {+ let splits = tabled_splits(head_dim, suffix_length, kv_length, unsplit_threadgroups, gpu_core_count)+ .unwrap_or_else(|| {+ splits_for_ratio(TENTHS_PER_RATIO * TG_PER_CORE_FALLBACK, gpu_core_count, unsplit_threadgroups)+ });+ splits.clamp(1, kv_length.div_ceil(block_k).max(1))+}++#[cfg(test)]+#[path = "../../../../../../unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs"]+mod tests;diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metalindex 851e8661b..b75c7709b 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal@@ -4,7 +4,7 @@#include "kv_cache_update.h"template <typename T>-VARIANTS(T, float, bfloat, half)+VARIANTS(T, float, bfloat)PUBLIC KERNEL(KVCacheUpdate) (device T* in_place_keys,device T* in_place_values,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rsnew file mode 100644index 000000000..452afdf94--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs@@ -0,0 +1,155 @@+mod fallback;+mod gemm;+mod gemm_grouped;+mod single_pass;+mod two_pass;++use crate::backends::{+ common::{+ Allocation, BufferArg, Encoder,+ kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig},+ },+ metal::{Metal, context::MetalContext, error::MetalError},+};++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum MaskKind {+ None,+ Causal,+ Trie,+}++impl MaskKind {+ fn for_attention(+ is_causal: bool,+ is_trie: bool,+ ) -> Option<Self> {+ match (is_causal, is_trie) {+ (true, true) => Some(Self::Trie),+ (true, false) => Some(Self::Causal),+ (false, false) => Some(Self::None),+ (false, true) => None,+ }+ }++ fn is_causal(self) -> bool {+ !matches!(self, Self::None)+ }++ fn is_trie(self) -> bool {+ matches!(self, Self::Trie)+ }+}++const SINGLE_PASS_KV_THRESHOLD: u32 = 1_024;+const D256_SINGLE_PASS_KV_THRESHOLD: u32 = 150_000;+const D128_SHORT_KV_GEMM_THRESHOLD: u32 = 2_048;++pub struct AttentionMetalKernel {+ head_dim: u32,+ is_causal: bool,+ grouped: Option<gemm_grouped::AttentionGemmGrouped>,+ gemm: Option<gemm::AttentionGemm>,+ fallback: Option<fallback::AttentionFallback>,+ two_pass: two_pass::AttentionTwoPass,+ single_pass: single_pass::AttentionSinglePass,+}++impl AttentionKernel for AttentionMetalKernel {+ type Backend = Metal;++ fn new(+ context: &MetalContext,+ config: AttentionKernelConfig,+ ) -> Result<Self, MetalError> {+ let grouped = gemm_grouped::AttentionGemmGrouped::is_supported(&config, context)+ .then(|| gemm_grouped::AttentionGemmGrouped::new(&config));+ let gemm = gemm::AttentionGemm::is_supported(&config).then(|| gemm::AttentionGemm::new(&config));+ let fallback = fallback::AttentionFallback::is_supported(&config)+ .then(|| fallback::AttentionFallback::new(&config, context))+ .transpose()?;+ Ok(Self {+ head_dim: config.head_dim,+ is_causal: config.is_causal,+ grouped,+ gemm,+ fallback,+ two_pass: two_pass::AttentionTwoPass::new(&config),+ single_pass: single_pass::AttentionSinglePass::new(&config),+ })+ }++ fn encode<'a, KT, VT>(+ &self,+ arguments: AttentionArguments<'a, Metal, KT, VT>,+ encoder: &mut Encoder<Metal>,+ ) -> Result<Allocation<Metal>, MetalError>+ where+ KT: BufferArg<'a, Metal>,+ VT: BufferArg<'a, Metal>,+ {+ encoder.push_debug_group("attention core");+ let result = self.encode_impl(arguments, encoder);+ encoder.pop_debug_group();+ result+ }+}++impl AttentionMetalKernel {+ fn encode_impl<'a, KT, VT>(+ &self,+ arguments: AttentionArguments<'a, Metal, KT, VT>,+ encoder: &mut Encoder<Metal>,+ ) -> Result<Allocation<Metal>, MetalError>+ where+ KT: BufferArg<'a, Metal>,+ VT: BufferArg<'a, Metal>,+ {+ let suffix_length = arguments.suffix_length;+ let kv_length = arguments.cache.prefix_len() + suffix_length;+ let is_trie = arguments.trie.is_some();+ let is_ring = arguments.cache.ring_params().is_some();+ let mask = MaskKind::for_attention(self.is_causal, is_trie);++ if !is_ring+ && let Some(mask) = mask+ && let Some(grouped) = &self.grouped+ {+ if grouped.should_encode(mask, suffix_length, kv_length) {+ return grouped.encode(mask, arguments, encoder);+ }++ // These measured sibling exceptions apply only when the+ // grouped core was built for this layer.+ if self.head_dim == 256 && (1..=16).contains(&suffix_length) && kv_length > D256_SINGLE_PASS_KV_THRESHOLD {+ return self.single_pass.encode(arguments, encoder);+ }+ if (self.head_dim == 256 && (9..=16).contains(&suffix_length))+ || (self.head_dim == 128+ && (9..=16).contains(&suffix_length)+ && kv_length > D128_SHORT_KV_GEMM_THRESHOLD)+ {+ return self.two_pass.encode(arguments, encoder);+ }+ }++ if suffix_length > 8 {+ if let Some(gemm) = &self.gemm {+ return gemm.encode(arguments, encoder);+ }+ if !is_trie && let Some(fallback) = &self.fallback {+ return fallback.encode(arguments, encoder);+ }+ }++ if kv_length > SINGLE_PASS_KV_THRESHOLD {+ self.two_pass.encode(arguments, encoder)+ } else {+ self.single_pass.encode(arguments, encoder)+ }+ }+}++#[cfg(test)]+#[path = "../../../../../unit/backends/metal/kernel/attention/mod.rs"]+mod tests;diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/qkv_norm.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/qkv_norm.metalindex df042a68d..373e729d7 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/qkv_norm.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/qkv_norm.metal@@ -11,10 +11,10 @@ using namespace metal;// - One SIMD-group (32 threads) processes one head.// - One threadgroup (one SIMD-group) is dispatched per head.template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>-VARIANTS(InputT, float, half, bfloat)-VARIANTS(ScaleT, float, half, bfloat)-VARIANTS(OutputT, float, half, bfloat)-VARIANTS(AccumT, float, half)+VARIANTS(InputT, float, bfloat)+VARIANTS(ScaleT, float, bfloat)+VARIANTS(OutputT, float, bfloat)+VARIANTS(AccumT, float)PUBLIC KERNEL(QKVNorm)(const device InputT* qkv_input OPTIONAL(!in_place),const device ScaleT* scales OPTIONAL(has_scales),diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metalindex 876609196..d49cb6b98 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal@@ -2,7 +2,7 @@#include "../common/dsl.h"template <typename T>-VARIANTS(T, float, half, bfloat)+VARIANTS(T, float, bfloat)PUBLIC KERNEL(SigmoidGate)(const device T* gate,device T* output,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rsnew file mode 100644index 000000000..ae72c18c5--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/single_pass.rs@@ -0,0 +1,81 @@+use std::collections::{HashMap, hash_map::Entry};++use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};++use crate::backends::{+ common::{+ Allocation, BufferArg, Encoder,+ kernel::{AttentionArguments, AttentionKernelConfig, AttentionSinglePassKernel},+ },+ metal::{Metal, context::MetalContext, error::MetalError, kernel::AttentionSinglePassMetalKernel},+};++pub struct AttentionSinglePass {+ kernels: Mutex<HashMap<bool, AttentionSinglePassMetalKernel>>,+ config: AttentionKernelConfig,+}++impl AttentionSinglePass {+ pub fn new(config: &AttentionKernelConfig) -> Self {+ Self {+ kernels: Mutex::new(HashMap::new()),+ config: *config,+ }+ }++ fn get_or_create(+ &self,+ context: &MetalContext,+ is_trie: bool,+ ) -> Result<MappedMutexGuard<'_, AttentionSinglePassMetalKernel>, MetalError> {+ let mut kernels = self.kernels.lock();+ if let Entry::Vacant(entry) = kernels.entry(is_trie) {+ let kernel = AttentionSinglePassMetalKernel::new(+ context,+ self.config.data_type,+ self.config.head_dim,+ self.config.has_sinks,+ self.config.is_kv_cache_ring,+ self.config.is_causal,+ is_trie,+ self.config.sliding_window_size.is_some(),+ )?;+ entry.insert(kernel);+ }+ Ok(MutexGuard::map(kernels, |kernels| kernels.get_mut(&is_trie).expect("kernel was just initialized")))+ }++ pub fn encode<'a, KT: BufferArg<'a, Metal>, VT: BufferArg<'a, Metal>>(+ &self,+ arguments: AttentionArguments<'a, Metal, KT, VT>,+ encoder: &mut Encoder<Metal>,+ ) -> Result<Allocation<Metal>, MetalError> {+ let config = self.config;+ let mut output = encoder.allocate_constant_for_shape(+ &[arguments.suffix_length, config.num_q_heads, config.head_dim],+ config.data_type,+ )?;+ let kernel = self.get_or_create(encoder.context(), arguments.trie.is_some())?;+ kernel.encode(+ arguments.queries,+ arguments.keys,+ arguments.values,+ &mut output,+ config.num_q_heads / config.num_groups,+ arguments.cache.prefix_len() + arguments.suffix_length,+ config.head_dim,+ config.num_groups * config.head_dim,+ config.head_dim,+ config.num_groups * config.head_dim,+ arguments.cache.ring_params(),+ config.scale.unwrap_or(1.0 / (config.head_dim as f32).sqrt()),+ arguments.trie,+ config.sliding_window_size,+ arguments.sinks,+ config.num_q_heads,+ arguments.suffix_length,+ encoder,+ );+ Ok(output)+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rsnew file mode 100644index 000000000..111a5bda2--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/two_pass.rs@@ -0,0 +1,120 @@+use std::collections::{HashMap, hash_map::Entry};++use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};++use crate::{+ backends::{+ common::{+ Allocation, BufferArg, Encoder,+ kernel::{AttentionArguments, AttentionKernelConfig},+ },+ metal::{+ Metal,+ context::MetalContext,+ error::MetalError,+ kernel::{AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel},+ },+ },+ data_type::DataType,+};++const PARTIAL_DATA_TYPE: DataType = DataType::F32;+const PARTIAL_BLOCKS: u32 = 32;++pub struct AttentionTwoPass {+ passes: Mutex<HashMap<bool, AttentionTwoPass1MetalKernel>>,+ second: Mutex<Option<AttentionTwoPass2MetalKernel>>,+ config: AttentionKernelConfig,+}++impl AttentionTwoPass {+ pub fn new(config: &AttentionKernelConfig) -> Self {+ Self {+ passes: Mutex::new(HashMap::new()),+ second: Mutex::new(None),+ config: *config,+ }+ }++ fn get_or_create(+ &self,+ context: &MetalContext,+ is_trie: bool,+ ) -> Result<MappedMutexGuard<'_, AttentionTwoPass1MetalKernel>, MetalError> {+ let mut passes = self.passes.lock();+ if let Entry::Vacant(entry) = passes.entry(is_trie) {+ entry.insert(AttentionTwoPass1MetalKernel::new(+ context,+ self.config.data_type,+ self.config.head_dim,+ self.config.has_sinks,+ self.config.is_kv_cache_ring,+ self.config.is_causal,+ is_trie,+ self.config.sliding_window_size.is_some(),+ )?);+ }+ Ok(MutexGuard::map(passes, |passes| passes.get_mut(&is_trie).expect("passes were just initialized")))+ }++ fn get_or_create_second(+ &self,+ context: &MetalContext,+ ) -> Result<MappedMutexGuard<'_, AttentionTwoPass2MetalKernel>, MetalError> {+ let mut second = self.second.lock();+ if second.is_none() {+ *second = Some(AttentionTwoPass2MetalKernel::new(context, self.config.data_type, self.config.head_dim)?);+ }+ Ok(MutexGuard::map(second, |second| second.as_mut().expect("pass was just initialized")))+ }++ pub fn encode<'a, KT: BufferArg<'a, Metal>, VT: BufferArg<'a, Metal>>(+ &self,+ arguments: AttentionArguments<'a, Metal, KT, VT>,+ encoder: &mut Encoder<Metal>,+ ) -> Result<Allocation<Metal>, MetalError> {+ let config = self.config;+ let mut partials = encoder.allocate_scratch_for_shape(+ &[arguments.suffix_length, config.num_q_heads, PARTIAL_BLOCKS, config.head_dim],+ PARTIAL_DATA_TYPE,+ )?;+ let mut sums = encoder.allocate_scratch_for_shape(+ &[arguments.suffix_length, config.num_q_heads, PARTIAL_BLOCKS],+ PARTIAL_DATA_TYPE,+ )?;+ let mut maxs = encoder.allocate_scratch_for_shape(+ &[arguments.suffix_length, config.num_q_heads, PARTIAL_BLOCKS],+ PARTIAL_DATA_TYPE,+ )?;+ let first = self.get_or_create(encoder.context(), arguments.trie.is_some())?;+ first.encode(+ arguments.queries,+ arguments.keys,+ arguments.values,+ &mut partials,+ &mut sums,+ &mut maxs,+ config.num_q_heads / config.num_groups,+ arguments.cache.prefix_len() + arguments.suffix_length,+ config.head_dim,+ config.num_groups * config.head_dim,+ config.head_dim,+ config.num_groups * config.head_dim,+ arguments.cache.ring_params(),+ config.scale.unwrap_or(1.0 / (config.head_dim as f32).sqrt()),+ config.num_q_heads,+ arguments.suffix_length,+ arguments.trie,+ config.sliding_window_size,+ arguments.sinks,+ encoder,+ );+ let mut output = encoder.allocate_constant_for_shape(+ &[arguments.suffix_length, config.num_q_heads, config.head_dim],+ config.data_type,+ )?;+ let second = self.get_or_create_second(encoder.context())?;+ second.encode(&partials, &sums, &maxs, &mut output, config.num_q_heads, arguments.suffix_length, encoder);+ Ok(output)+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/mod.rsindex f061464d8..7c2cf7a05 100644--- a/crates/uzu-engine/src/backends/metal/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/mod.rs@@ -9,7 +9,7 @@ use crate::backends::{metal::Metal,};-pub mod attention;+mod attention;pub mod gdn;pub mod matmul;mod radix_top_k_small;@@ -22,7 +22,7 @@ impl Kernels for MetalKernels {type Backend = Metal;autogen_kernels!();- type AttentionGemmCore = attention::AttentionGemmMetalCore;+ type AttentionKernel = attention::AttentionMetalKernel;type DeltaNetChunkedPrefill = gdn::chunked::MetalDeltaNetChunkedPrefill;type DeltaNetTreeVerify = gdn::tree_verify::MetalDeltaNetTreeVerify;type MatmulKernel = matmul::MatmulMetalKernel;diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rsdeleted file mode 100644index 1bb3f9278..000000000--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/mod.rs+++ /dev/null@@ -1,107 +0,0 @@-use crate::{- backends::common::{- Allocation, Backend, BufferArg, Encoder, Kernels,- kernel::attention_gemm::AttentionGemmCore as AttentionGemmCoreTrait,- },- data_type::DataType,- encodable_block::mixer::attention::{- KVCacheView,- core::{fallback::AttentionFallbackCore, single_pass::AttentionSinglePassCore, two_pass::AttentionTwoPassCore},- },-};--mod fallback;-mod single_pass;-mod two_pass;--pub struct AttentionCoreNewArguments {- pub head_dim: u32,- pub num_groups: u32,- pub num_q_heads: u32,- pub has_sinks: bool,- pub is_kv_cache_ring: bool,- pub is_causal: bool,- pub is_trie: bool,- pub sliding_window_size: Option<u32>,- pub scale: Option<f32>,- pub data_type: DataType,-}--pub struct AttentionCoreEncodeArguments<'a, B: Backend, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>> {- pub queries: &'a Allocation<B>,- pub keys: KT,- pub values: VT,- pub suffix_length: u32,- pub trie: Option<&'a Allocation<B>>,- pub sinks: Option<&'a Allocation<B>>,- pub cache: KVCacheView,-}--pub struct AttentionCores<B: Backend> {- gemm: Option<<B::Kernels as Kernels>::AttentionGemmCore>,- fallback: Option<AttentionFallbackCore<B>>,- two_pass: AttentionTwoPassCore<B>,- single_pass: AttentionSinglePassCore<B>,-}--impl<B: Backend> AttentionCores<B> {- pub fn new(- arguments: AttentionCoreNewArguments,- context: &B::Context,- ) -> Result<Self, B::Error> {- let gemm = if <<B::Kernels as Kernels>::AttentionGemmCore as AttentionGemmCoreTrait>::is_supported(- &arguments, context,- )? {- Some(<<B::Kernels as Kernels>::AttentionGemmCore as AttentionGemmCoreTrait>::new(context, &arguments)?)- } else {- None- };- let fallback = if arguments.head_dim == 512 && !arguments.is_trie {- Some(AttentionFallbackCore::new(&arguments, context)?)- } else {- None- };- let two_pass = AttentionTwoPassCore::new(&arguments, context)?;- let single_pass = AttentionSinglePassCore::new(&arguments, context)?;-- Ok(Self {- gemm,- fallback,- two_pass,- single_pass,- })- }- pub fn encode<'a, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>>(- &self,- arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,- encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, <B as Backend>::Error> {- encoder.push_debug_group("attention core");-- let output = if arguments.suffix_length > 8- && let Some(gemm) = &self.gemm- {- gemm.encode(arguments, encoder)- } else if arguments.suffix_length > 8- && let Some(fallback) = &self.fallback- {- fallback.encode(arguments, encoder)- } else if arguments.cache.prefix_len() + arguments.suffix_length > 1024 {- self.two_pass.encode(arguments, encoder)- } else {- self.single_pass.encode(arguments, encoder)- }?;-- encoder.pop_debug_group();-- Ok(output)- }-}--#[cfg(test)]-#[path = "../../../../../unit/encodable_block/attention_test.rs"]-mod tests;--#[cfg(test)]-#[path = "../../../../../unit/encodable_block/attention_gemm_test.rs"]-mod gemm_tests;diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rsdeleted file mode 100644index bffd0153b..000000000--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/single_pass.rs+++ /dev/null@@ -1,74 +0,0 @@-use crate::{- backends::common::{Allocation, Backend, BufferArg, Encoder, Kernels, kernel::AttentionSinglePassKernel},- data_type::DataType,- encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},-};--pub struct AttentionSinglePassCore<B: Backend> {- head_dim: u32,- num_groups: u32,- num_q_heads: u32,- sliding_window_size: Option<u32>,- scale: Option<f32>,- data_type: DataType,- kernel: <B::Kernels as Kernels>::AttentionSinglePassKernel,-}--impl<B: Backend> AttentionSinglePassCore<B> {- pub fn new(- arguments: &AttentionCoreNewArguments,- context: &B::Context,- ) -> Result<Self, B::Error> {- let kernel = <B::Kernels as Kernels>::AttentionSinglePassKernel::new(- context,- arguments.data_type,- arguments.head_dim,- arguments.has_sinks,- arguments.is_kv_cache_ring,- arguments.is_causal,- arguments.is_trie,- arguments.sliding_window_size.is_some(),- )?;-- Ok(Self {- head_dim: arguments.head_dim,- num_groups: arguments.num_groups,- num_q_heads: arguments.num_q_heads,- sliding_window_size: arguments.sliding_window_size,- scale: arguments.scale,- data_type: arguments.data_type,- kernel,- })- }-- pub fn encode<'a, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>>(- &self,- arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,- encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, B::Error> {- let mut output = encoder- .allocate_constant_for_shape(&[arguments.suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;- self.kernel.encode(- arguments.queries,- arguments.keys,- arguments.values,- &mut output,- self.num_q_heads / self.num_groups,- arguments.cache.prefix_len() + arguments.suffix_length,- self.head_dim,- self.num_groups * self.head_dim,- self.head_dim,- self.num_groups * self.head_dim,- arguments.cache.ring_params(),- self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),- arguments.trie,- self.sliding_window_size,- arguments.sinks,- self.num_q_heads,- arguments.suffix_length,- encoder,- );-- Ok(output)- }-}diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rsdeleted file mode 100644index ca072829b..000000000--- a/crates/uzu-engine/src/encodable_block/mixer/attention/core/two_pass.rs+++ /dev/null@@ -1,102 +0,0 @@-use crate::{- backends::common::{- Allocation, Backend, BufferArg, Encoder, Kernels,- kernel::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},- },- data_type::DataType,- encodable_block::mixer::attention::core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},-};--const INNER_DATA_TYPE: DataType = DataType::F32;-const TWO_PASS_BLOCKS: u32 = 32;--pub struct AttentionTwoPassCore<B: Backend> {- head_dim: u32,- num_groups: u32,- num_q_heads: u32,- sliding_window_size: Option<u32>,- scale: Option<f32>,- data_type: DataType,- pass_1: <B::Kernels as Kernels>::AttentionTwoPass1Kernel,- pass_2: <B::Kernels as Kernels>::AttentionTwoPass2Kernel,-}--impl<B: Backend> AttentionTwoPassCore<B> {- pub fn new(- arguments: &AttentionCoreNewArguments,- context: &B::Context,- ) -> Result<Self, B::Error> {- let pass_1 = <B::Kernels as Kernels>::AttentionTwoPass1Kernel::new(- context,- arguments.data_type,- arguments.head_dim,- arguments.has_sinks,- arguments.is_kv_cache_ring,- arguments.is_causal,- arguments.is_trie,- arguments.sliding_window_size.is_some(),- )?;-- let pass_2 =- <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, arguments.data_type, arguments.head_dim)?;-- Ok(Self {- head_dim: arguments.head_dim,- num_groups: arguments.num_groups,- num_q_heads: arguments.num_q_heads,- sliding_window_size: arguments.sliding_window_size,- scale: arguments.scale,- data_type: arguments.data_type,- pass_1,- pass_2,- })- }-- pub fn encode<'a, KT: BufferArg<'a, B>, VT: BufferArg<'a, B>>(- &self,- arguments: AttentionCoreEncodeArguments<'a, B, KT, VT>,- encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, B::Error> {- let mut partials = encoder.allocate_scratch_for_shape(- &[arguments.suffix_length, self.num_q_heads, TWO_PASS_BLOCKS, self.head_dim],- INNER_DATA_TYPE,- )?;- let mut sums = encoder.allocate_scratch_for_shape(- &[arguments.suffix_length, self.num_q_heads, TWO_PASS_BLOCKS],- INNER_DATA_TYPE,- )?;- let mut maxs = encoder.allocate_scratch_for_shape(- &[arguments.suffix_length, self.num_q_heads, TWO_PASS_BLOCKS],- INNER_DATA_TYPE,- )?;-- self.pass_1.encode(- arguments.queries,- arguments.keys,- arguments.values,- &mut partials,- &mut sums,- &mut maxs,- self.num_q_heads / self.num_groups,- arguments.cache.prefix_len() + arguments.suffix_length,- self.head_dim,- self.num_groups * self.head_dim,- self.head_dim,- self.num_groups * self.head_dim,- arguments.cache.ring_params(),- self.scale.unwrap_or(1.0f32 / (self.head_dim as f32).sqrt()),- self.num_q_heads,- arguments.suffix_length,- arguments.trie,- self.sliding_window_size,- arguments.sinks,- encoder,- );-- let mut output = encoder- .allocate_constant_for_shape(&[arguments.suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;- self.pass_2.encode(&partials, &sums, &maxs, &mut output, self.num_q_heads, arguments.suffix_length, encoder);-- Ok(output)- }-}diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rsindex 1aef18ff2..3e7927833 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mod.rs@@ -3,7 +3,7 @@ use thiserror::Error;use crate::{backends::common::{Allocation, Backend, Encoder, Kernels,- kernel::{AttentionPrepareKernel, SigmoidGateKernel},+ kernel::{AttentionKernel, AttentionKernelConfig, AttentionPrepareKernel, SigmoidGateKernel},},config::{rope::AnyRoPEConfig, token_mixer::attention::AttentionConfig},data_type::DataType,@@ -13,7 +13,6 @@ use crate::{mixer::{Mixer, MixerState,attention::{- core::{AttentionCoreNewArguments, AttentionCores},mode::LinearProjection,qkv_norm::{QKVNorm, QKVNormError},rope::PrecalculatedRoPE,@@ -24,7 +23,6 @@ use crate::{utils::maybe_mut::MaybeMut,};-pub mod core;mod mode;mod qkv_norm;mod state;@@ -44,8 +42,7 @@ pub struct Attention<B: Backend> {prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,gate_projection: Option<Box<dyn Linear<B>>>,sinks: Option<Allocation<B>>,- flat_core: AttentionCores<B>,- trie_core: AttentionCores<B>,+ kernel: <B::Kernels as Kernels>::AttentionKernel,gate_kernel: Option<<B::Kernels as Kernels>::SigmoidGateKernel>,out_projection: Box<dyn Linear<B>>,}@@ -167,37 +164,19 @@ impl<B: Backend> Attention<B> {let ring_capacity = sliding_window_size;let is_kv_cache_ring = ring_capacity.is_some();- let flat_core = AttentionCores::new(- AttentionCoreNewArguments {- head_dim,- num_groups,- num_q_heads,- has_sinks: sinks.is_some(),- is_kv_cache_ring,- is_causal,- is_trie: false,- sliding_window_size,- scale: config.scale,- data_type,- },+ let kernel = <B::Kernels as Kernels>::AttentionKernel::new(context,- )- .map_err(AttentionNewError::Backend)?;-- let trie_core = AttentionCores::new(- AttentionCoreNewArguments {+ AttentionKernelConfig {head_dim,num_groups,num_q_heads,has_sinks: sinks.is_some(),is_kv_cache_ring,is_causal,- is_trie: true,sliding_window_size,scale: config.scale,data_type,},- context,).map_err(AttentionNewError::Backend)?;@@ -230,8 +209,7 @@ impl<B: Backend> Attention<B> {prepare,gate_projection,sinks,- flat_core,- trie_core,+ kernel,gate_kernel,out_projection,},diff --git a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rsindex 218cbb124..863e5e21d 100644--- a/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/attention/mode.rs@@ -2,17 +2,14 @@ use crate::{backends::common::{Allocation, Backend, BufferArgMut, Encoder,gpu_types::trie::TrieNode,- kernel::{AttentionPrepareKernel, SigmoidGateKernel},+ kernel::{AttentionArguments, AttentionKernel, AttentionPrepareKernel, SigmoidGateKernel},},encodable_block::{batch_topology::BatchTopology,linear::Linear,mixer::{MixerState,- attention::{- Attention, KVCacheView, core::AttentionCoreEncodeArguments, qkv_norm::QKVNorm, rope::PrecalculatedRoPE,- state::AttentionState,- },+ attention::{Attention, KVCacheView, qkv_norm::QKVNorm, rope::PrecalculatedRoPE, state::AttentionState},},},utils::maybe_mut::MaybeMut,@@ -104,8 +101,8 @@ impl<B: Backend> Attention<B> {let cache = self.ring_capacity.map_or_else(|| KVCacheView::full(0), |_| KVCacheView::ring(0, 0));- self.flat_core.encode(- AttentionCoreEncodeArguments {+ self.kernel.encode(+ AttentionArguments {queries: &queries,keys: &keys,values: &values,@@ -163,16 +160,16 @@ impl<B: Backend> Attention<B> {state: &AttentionState<B>,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let (core, trie) = if batch_dim.is_flat() {- (&self.flat_core, None)+ let trie = if batch_dim.is_flat() {+ None} else {let mut trie = encoder.allocate_constant(batch_dim.size() as usize * size_of::<TrieNode>())?;trie.copyin(batch_dim.nodes());- (&self.trie_core, Some(trie))+ Some(trie)};- core.encode(- AttentionCoreEncodeArguments {+ self.kernel.encode(+ AttentionArguments {queries,keys: state.keys.as_ref(),values: state.values.as_ref(),diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rsindex 309210ccd..07bcfa28e 100644--- a/crates/uzu-engine/src/encodable_block/weaver.rs+++ b/crates/uzu-engine/src/encodable_block/weaver.rs@@ -9,8 +9,8 @@ use crate::{TreeIdx,},kernel::{- AncestorAttentionKernel, WeaverFrontierInsertChildrenKernel, WeaverFrontierSelectKernel,- WeaverTopChildrenKernel, radix_top_k_small::RadixTopKSmall,+ AncestorAttentionKernel, AttentionArguments, AttentionKernel, WeaverFrontierInsertChildrenKernel,+ WeaverFrontierSelectKernel, WeaverTopChildrenKernel, radix_top_k_small::RadixTopKSmall,},},config::{rope::AnyRoPEConfig, weaver::WeaverConfig},@@ -18,7 +18,7 @@ use crate::{encodable_block::{embedding::{Embedding, EmbeddingError},linear::{Linear, LinearBlockError},- mixer::attention::{KVCacheView, core::AttentionCoreEncodeArguments, rope::PrecalculatedRoPE},+ mixer::attention::{KVCacheView, rope::PrecalculatedRoPE},mlp::MlpBlockError,normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},weaver_layer::{PreparedPrefixAttention, WeaverLayer},@@ -330,7 +330,7 @@ impl<B: Backend> Weaver<B> {let attention_output = layer.prefix_attention.encode(- AttentionCoreEncodeArguments {+ AttentionArguments {queries: &queries,keys: &kv_cache,values: (&kv_cache, kv_plane_bytes),diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rsindex dacfd8d11..8c6a52468 100644--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs@@ -2,7 +2,7 @@ use crate::{array::size_for_shape,backends::common::{Allocation, AsBufferRangeMut, Backend, Encoder, Kernels,- kernel::{AncestorAttentionKernel, AttentionPrepareKernel},+ kernel::{AncestorAttentionKernel, AttentionKernel, AttentionKernelConfig, AttentionPrepareKernel},},config::{activation::{AnyActivation, silu::SiLU},@@ -11,10 +11,7 @@ use crate::{},encodable_block::{linear::Linear,- mixer::attention::{- core::{AttentionCoreNewArguments, AttentionCores},- rope::PrecalculatedRoPE,- },+ mixer::attention::rope::PrecalculatedRoPE,mlp::Mlp,normalization::{Normalization, PostLayerScalar, ShortcutMode},weaver::{DATA_TYPE, ROPE_DATA_TYPE, WeaverNewError},@@ -31,7 +28,7 @@ pub struct WeaverLayer<B: Backend> {pub pre_attention_norm: Normalization<B>,pub qkv_projection: Box<dyn Linear<B>>,attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,- pub prefix_attention: AttentionCores<B>,+ pub prefix_attention: <B::Kernels as Kernels>::AttentionKernel,pub ancestor_attention: <B::Kernels as Kernels>::AncestorAttentionKernel,out_projection: Box<dyn Linear<B>>,pre_mlp_norm: Normalization<B>,@@ -77,20 +74,19 @@ impl<B: Backend> WeaverLayer<B> {DATA_TYPE,¶meter_tree.subtree("out_projection"),)?;- let prefix_attention = AttentionCores::new(- AttentionCoreNewArguments {+ let prefix_attention = <B::Kernels as Kernels>::AttentionKernel::new(+ context,+ AttentionKernelConfig {head_dim,num_groups: num_heads,num_q_heads: num_heads,has_sinks: false,is_kv_cache_ring: false,is_causal: true,- is_trie: false,sliding_window_size: None,scale: Some(attention_scale),data_type: DATA_TYPE,},- context,).map_err(WeaverNewError::Backend)?;let attention_prepare =diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_fallback_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_fallback_test.rsdeleted file mode 100644index faf7d9789..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_fallback_test.rs+++ /dev/null@@ -1,209 +0,0 @@-use half::bf16;-use uzu_engine_macros::uzu_test;--use crate::{- backends::{- common::{- Allocation, Backend, Context, Encoder, Kernels,- kernel::{- AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, AttentionSinglePassKernel,- SoftmaxKernel,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},- },- },- cpu::Cpu,- },- data_type::DataType,- tests::{- assert::assert_eq_float,- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},- },-};--const NUM_HEADS: u32 = 4;-const NUM_GROUPS: u32 = 2;-const SEQ: u32 = 16;-const SUFFIX: u32 = 3;-const HEAD_DIM: u32 = 512;-const GQA: u32 = NUM_HEADS / NUM_GROUPS;--fn make_inputs() -> (Box<[bf16]>, Box<[bf16]>, Box<[bf16]>) {- let q = (0..(NUM_HEADS * SUFFIX * HEAD_DIM) as usize)- .map(|i| bf16::from_f32((i as f32 * 0.013 + 0.5).sin() * 0.5))- .collect();- let k = (0..(NUM_GROUPS * SEQ * HEAD_DIM) as usize)- .map(|i| bf16::from_f32((i as f32 * 0.007 + 1.0).cos() * 0.5))- .collect();- let v = (0..(NUM_GROUPS * SEQ * HEAD_DIM) as usize)- .map(|i| bf16::from_f32((i as f32 * 0.011 + 2.0).sin() * 0.5))- .collect();- (q, k, v)-}--// Ground truth: fused single-pass attention on the CPU backend.-fn reference(- q: &[bf16],- k: &[bf16],- v: &[bf16],- scale: f32,-) -> Vec<bf16> {- let ctx = <Cpu as Backend>::Context::new().unwrap();- let kernel = <<Cpu as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(- &ctx,- DataType::BF16,- HEAD_DIM,- false,- false,- true,- false,- false,- )- .unwrap();- let qa = alloc_allocation_with_data::<Cpu, bf16>(&ctx, q);- let ka = alloc_allocation_with_data::<Cpu, bf16>(&ctx, k);- let va = alloc_allocation_with_data::<Cpu, bf16>(&ctx, v);- let mut out = alloc_allocation::<Cpu, bf16>(&ctx, (NUM_HEADS * SUFFIX * HEAD_DIM) as usize);- let mut enc = Encoder::new(ctx.as_ref()).unwrap();- kernel.encode(- &qa,- &ka,- &va,- &mut out,- GQA,- SEQ,- SEQ * HEAD_DIM,- HEAD_DIM,- SEQ * HEAD_DIM,- HEAD_DIM,- None,- scale,- None::<&Allocation<Cpu>>,- None,- None::<&Allocation<Cpu>>,- NUM_HEADS,- SUFFIX,- &mut enc,- );- enc.end_encoding().submit().wait_until_completed().unwrap();- allocation_to_vec::<Cpu, bf16>(&out)-}--// Production fallback pipeline (matmul → scatter → softmax → matmul → scatter)-// driven directly via the kernel-cache traits, for backends other than CPU.-fn pipeline_output<B: Backend>(- q: &[bf16],- k: &[bf16],- v: &[bf16],- scale: f32,-) -> Vec<bf16> {- let ctx = B::Context::new().unwrap();- let softmax = <<B as Backend>::Kernels as Kernels>::SoftmaxKernel::new(&ctx, DataType::BF16, false).unwrap();- let scatter_scores = <<B as Backend>::Kernels as Kernels>::AttentionFallbackScatterScoresKernel::new(- &ctx,- DataType::BF16,- false,- true,- false,- false,- )- .unwrap();- let scatter_values =- <<B as Backend>::Kernels as Kernels>::AttentionFallbackScatterValuesKernel::new(&ctx, DataType::BF16).unwrap();- let mut matmul =- <<B as Backend>::Kernels as Kernels>::MatmulKernel::new(&ctx, DataType::BF16, DataType::BF16, DataType::BF16)- .unwrap();-- let qa = alloc_allocation_with_data::<B, bf16>(&ctx, q);- let ka = alloc_allocation_with_data::<B, bf16>(&ctx, k);- let va = alloc_allocation_with_data::<B, bf16>(&ctx, v);- let mut scores = alloc_allocation::<B, bf16>(&ctx, (NUM_HEADS * SUFFIX * SEQ) as usize);- let mut grp_s = alloc_allocation::<B, bf16>(&ctx, (GQA * SUFFIX * SEQ) as usize);- let mut grp_o = alloc_allocation::<B, bf16>(&ctx, (GQA * SUFFIX * HEAD_DIM) as usize);- let mut out = alloc_allocation::<B, bf16>(&ctx, (NUM_HEADS * SUFFIX * HEAD_DIM) as usize);-- let dt = DataType::BF16.size_in_bytes();- let q_row = HEAD_DIM as usize * dt;- let s_row = SEQ as usize * dt;- let head_stride = (SEQ * HEAD_DIM) as usize;- let mut enc = Encoder::new(ctx.as_ref()).unwrap();-- for g in 0..NUM_GROUPS {- matmul- .encode(- MatmulArguments {- a: MatmulA::FullPrecision {- values: &qa,- offset: g as usize * GQA as usize * SUFFIX as usize * q_row,- },- b: MatmulB::FullPrecision {- b: (&ka, g as usize * head_stride * dt),- },- b_leading_dimension: Some(HEAD_DIM),- b_transpose: true,- d: &mut grp_s,- d_transform: MatmulDOps {- ab_scale: scale,- ..MatmulDOps::none()- },- gather_indices: None,- m: GQA * SUFFIX,- n: SEQ,- k: HEAD_DIM,- },- &mut enc,- )- .expect("encode failed");- scatter_scores.encode(- &grp_s,- &mut scores,- None,- None::<&Allocation<B>>,- None,- g,- GQA,- SEQ,- SUFFIX,- GQA * SUFFIX * SEQ,- &mut enc,- );- }- softmax.encode(&mut scores, None::<&Allocation<B>>, SEQ, NUM_HEADS, SUFFIX, &mut enc);- for g in 0..NUM_GROUPS {- matmul- .encode(- MatmulArguments {- a: MatmulA::FullPrecision {- values: &scores,- offset: g as usize * GQA as usize * SUFFIX as usize * s_row,- },- b: MatmulB::FullPrecision {- b: (&va, g as usize * head_stride * dt),- },- b_leading_dimension: Some(HEAD_DIM),- b_transpose: false,- d: &mut grp_o,- d_transform: MatmulDOps::none(),- gather_indices: None,- m: GQA * SUFFIX,- n: HEAD_DIM,- k: SEQ,- },- &mut enc,- )- .expect("encode failed");- scatter_values.encode(&grp_o, &mut out, g, GQA, SUFFIX, NUM_HEADS, HEAD_DIM, GQA * SUFFIX * HEAD_DIM, &mut enc);- }- enc.end_encoding().submit().wait_until_completed().unwrap();- allocation_to_vec::<B, bf16>(&out)-}--#[uzu_test]-fn test_head_dim_512_bf16() {- let (q, k, v) = make_inputs();- let scale = 1.0 / (HEAD_DIM as f32).sqrt();- let expected = reference(&q, &k, &v, scale);- for_each_non_cpu_backend!(|B| {- let output = pipeline_output::<B>(&q, &k, &v, scale);- assert_eq_float::<bf16>(&expected, &output, 1e-2, "AttentionFallback bf16");- });-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rsdeleted file mode 100644index bda3b9bf2..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_single_pass_test.rs+++ /dev/null@@ -1,295 +0,0 @@-use std::fmt::{Debug, Display};--use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::uzu_test;--use crate::{- array::ArrayElement,- backends::{- common::{- Allocation, Backend, Context, Encoder, Kernels, gpu_types::ring::RingParams,- kernel::AttentionSinglePassKernel,- },- cpu::Cpu,- },- data_type::DataType,- tests::{- assert::assert_eq_float,- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},- },-};--struct Input<T: ArrayElement + Float> {- queries: Box<[T]>,- keys: Box<[T]>,- values: Box<[T]>,- num_heads: u32,- gqa_factor: u32,- sequence_length: u32,- suffix_length: u32,- head_dim: u32,- scale: f32,- do_causal: bool,- ring_params: Option<RingParams>,- sliding_window_size: Option<u32>,-}--fn get_input<T: ArrayElement + Float>(- num_heads: u32,- num_kv_heads: u32,- sequence_length: u32,- suffix_length: u32,- head_dim: u32,- do_causal: bool,-) -> Input<T> {- let gqa_factor = num_heads / num_kv_heads;-- // queries: [num_heads * suffix_length, head_dim]- let q_size = (num_heads * suffix_length * head_dim) as usize;- let mut queries = vec![T::zero(); q_size];- for i in 0..q_size {- queries[i] = T::from((i as f32 * 0.13 + 0.5).sin() * 0.5).unwrap();- }-- // keys: [num_kv_heads, sequence_length, head_dim]- let k_size = (num_kv_heads * sequence_length * head_dim) as usize;- let mut keys = vec![T::zero(); k_size];- for i in 0..k_size {- keys[i] = T::from((i as f32 * 0.07 + 1.0).cos() * 0.5).unwrap();- }-- // values: [num_kv_heads, sequence_length, head_dim]- let v_size = (num_kv_heads * sequence_length * head_dim) as usize;- let mut values = vec![T::zero(); v_size];- for i in 0..v_size {- values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();- }-- let scale = 1.0 / (head_dim as f32).sqrt();-- Input {- queries: queries.into_boxed_slice(),- keys: keys.into_boxed_slice(),- values: values.into_boxed_slice(),- num_heads,- gqa_factor,- sequence_length,- suffix_length,- head_dim,- scale,- do_causal,- ring_params: None,- sliding_window_size: None,- }-}--fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {- let context = B::Context::new().expect("Failed to create Context");-- let kernel = <<B as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(- &context,- T::data_type(),- input.head_dim,- false,- input.ring_params.is_some(),- input.do_causal,- false,- input.sliding_window_size.is_some(),- )- .expect("Failed to create AttentionSinglePassKernel");-- let queries_allocation = alloc_allocation_with_data::<B, T>(&context, &input.queries);- let keys_allocation = alloc_allocation_with_data::<B, T>(&context, &input.keys);- let values_allocation = alloc_allocation_with_data::<B, T>(&context, &input.values);-- let output_size = (input.suffix_length * input.num_heads * input.head_dim) as usize;- let mut output_allocation = alloc_allocation::<B, T>(&context, output_size);-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- &queries_allocation,- &keys_allocation,- &values_allocation,- &mut output_allocation,- input.gqa_factor,- input.sequence_length,- input.sequence_length * input.head_dim,- input.head_dim,- input.sequence_length * input.head_dim,- input.head_dim,- input.ring_params,- input.scale,- None::<&Allocation<B>>,- input.sliding_window_size,- None::<&Allocation<B>>,- input.num_heads,- input.suffix_length,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().unwrap();-- allocation_to_vec::<B, T>(&output_allocation)-}--fn test_internal<T: ArrayElement + Float + Debug + Display>(- input: &Input<T>,- expected: &[T],-) {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {- 1e-2- } else {- 1e-5- };-- for_each_non_cpu_backend!(|B| {- let output = get_output::<T, B>(input);- let msg = format!(- "AttentionSinglePass failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={})",- std::any::type_name::<B>(),- input.num_heads,- input.sequence_length,- input.suffix_length,- input.head_dim,- input.do_causal,- );- assert_eq_float::<T>(expected, &output, eps, &msg);- });-}--fn test_basic<T: ArrayElement + Float + Debug + Display>() {- // Non-causal, single token- let input = get_input::<T>(4, 4, 8, 1, 64, false);- let expected = get_output::<T, Cpu>(&input);- test_internal(&input, &expected);-- // Non-causal, multiple tokens- let input = get_input::<T>(4, 4, 8, 4, 64, false);- let expected = get_output::<T, Cpu>(&input);- test_internal(&input, &expected);-}--fn test_causal<T: ArrayElement + Float + Debug + Display>() {- // Causal, single token decode- let input = get_input::<T>(4, 4, 16, 1, 64, true);- let expected = get_output::<T, Cpu>(&input);- test_internal(&input, &expected);-- // Causal, multi-token prefill- let input = get_input::<T>(4, 4, 8, 4, 64, true);- let expected = get_output::<T, Cpu>(&input);- test_internal(&input, &expected);-}--fn test_gqa<T: ArrayElement + Float + Debug + Display>() {- // GQA: 8 query heads, 2 kv heads- let input = get_input::<T>(8, 2, 8, 1, 64, false);- let expected = get_output::<T, Cpu>(&input);- test_internal(&input, &expected);-- // GQA causal- let input = get_input::<T>(8, 2, 8, 4, 64, true);- let expected = get_output::<T, Cpu>(&input);- test_internal(&input, &expected);-}--fn test_noncausal_ring<T: ArrayElement + Float + Debug + Display>() {- let mut input = get_input::<T>(8, 2, 8, 4, 64, false);- input.ring_params = Some(RingParams {- ring_offset: 2,- ring_length: 4,- });- input.sliding_window_size = Some(4);- let expected = get_output::<T, Cpu>(&input);- test_internal(&input, &expected);-}--fn test_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {- let input = get_input::<T>(4, 4, 8, 2, head_dim, true);- let expected = get_output::<T, Cpu>(&input);- test_internal(&input, &expected);-}--// Basic tests-#[uzu_test]-fn test_basic_f32() {- test_basic::<f32>();-}--#[uzu_test]-fn test_basic_f16() {- test_basic::<f16>();-}--#[uzu_test]-fn test_basic_bf16() {- test_basic::<bf16>();-}--// Causal tests-#[uzu_test]-fn test_causal_f32() {- test_causal::<f32>();-}--#[uzu_test]-fn test_causal_f16() {- test_causal::<f16>();-}--#[uzu_test]-fn test_causal_bf16() {- test_causal::<bf16>();-}--// GQA tests-#[uzu_test]-fn test_gqa_f32() {- test_gqa::<f32>();-}--#[uzu_test]-fn test_gqa_f16() {- test_gqa::<f16>();-}--#[uzu_test]-fn test_gqa_bf16() {- test_gqa::<bf16>();-}--#[uzu_test]-fn test_noncausal_ring_bf16() {- test_noncausal_ring::<bf16>();-}--// Head dim 128-#[uzu_test]-fn test_head_dim_128_f32() {- test_head_dim::<f32>(128);-}--#[uzu_test]-fn test_head_dim_128_f16() {- test_head_dim::<f16>(128);-}--#[uzu_test]-fn test_head_dim_128_bf16() {- test_head_dim::<bf16>(128);-}--#[uzu_test]-fn test_head_dim_512_f32() {- test_head_dim::<f32>(512);-}--#[uzu_test]-fn test_head_dim_512_f16() {- test_head_dim::<f16>(512);-}--#[uzu_test]-fn test_head_dim_512_bf16() {- test_head_dim::<bf16>(512);-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_two_pass_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/attention_two_pass_test.rsdeleted file mode 100644index 193239007..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/attention/attention_two_pass_test.rs+++ /dev/null@@ -1,446 +0,0 @@-use std::fmt::{Debug, Display};--use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::uzu_test;--use crate::{- array::ArrayElement,- backends::{- common::{- Allocation, Backend, Context, Encoder, Kernels,- kernel::{AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},- },- cpu::Cpu,- },- data_type::DataType,- tests::{- assert::assert_eq_float,- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},- },-};--const TOTAL_BLOCKS_COUNT: u32 = 32;--// --- First pass types and helpers -----struct FirstPassInput<T: ArrayElement + Float> {- queries: Box<[T]>,- keys: Box<[T]>,- values: Box<[T]>,- num_heads: u32,- gqa_factor: u32,- sequence_length: u32,- suffix_length: u32,- head_dim: u32,- scale: f32,- do_causal: bool,-}--struct FirstPassOutput {- partials: Vec<f32>,- sums: Vec<f32>,- maxs: Vec<f32>,-}--fn get_first_pass_input<T: ArrayElement + Float>(- num_heads: u32,- num_kv_heads: u32,- sequence_length: u32,- suffix_length: u32,- head_dim: u32,- do_causal: bool,-) -> FirstPassInput<T> {- let gqa_factor = num_heads / num_kv_heads;-- let q_size = (num_heads * suffix_length * head_dim) as usize;- let mut queries = vec![T::zero(); q_size];- for i in 0..q_size {- queries[i] = T::from((i as f32 * 0.13 + 0.5).sin() * 0.5).unwrap();- }-- let k_size = (num_kv_heads * sequence_length * head_dim) as usize;- let mut keys = vec![T::zero(); k_size];- for i in 0..k_size {- keys[i] = T::from((i as f32 * 0.07 + 1.0).cos() * 0.5).unwrap();- }-- let v_size = (num_kv_heads * sequence_length * head_dim) as usize;- let mut values = vec![T::zero(); v_size];- for i in 0..v_size {- values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();- }-- let scale = 1.0 / (head_dim as f32).sqrt();-- FirstPassInput {- queries: queries.into_boxed_slice(),- keys: keys.into_boxed_slice(),- values: values.into_boxed_slice(),- num_heads,- gqa_factor,- sequence_length,- suffix_length,- head_dim,- scale,- do_causal,- }-}--fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassInput<T>) -> FirstPassOutput {- let context = B::Context::new().expect("Failed to create Context");-- let kernel = <<B as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(- &context,- T::data_type(),- input.head_dim,- false,- false,- input.do_causal,- false,- false,- )- .expect("Failed to create AttentionTwoPass1Kernel");-- let queries = alloc_allocation_with_data::<B, T>(&context, &input.queries);- let keys = alloc_allocation_with_data::<B, T>(&context, &input.keys);- let values = alloc_allocation_with_data::<B, T>(&context, &input.values);-- let total_offsets = (input.suffix_length * input.num_heads) as usize;- let partials_size = total_offsets * TOTAL_BLOCKS_COUNT as usize * input.head_dim as usize;- let sums_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;- let maxs_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;-- let mut partials = alloc_allocation::<B, f32>(&context, partials_size);- let mut sums = alloc_allocation::<B, f32>(&context, sums_size);- let mut maxs = alloc_allocation::<B, f32>(&context, maxs_size);-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- &queries,- &keys,- &values,- &mut partials,- &mut sums,- &mut maxs,- input.gqa_factor,- input.sequence_length,- input.sequence_length * input.head_dim,- input.head_dim,- input.sequence_length * input.head_dim,- input.head_dim,- None,- input.scale,- input.num_heads,- input.suffix_length,- None::<&Allocation<B>>,- None,- None::<&Allocation<B>>,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().unwrap();-- FirstPassOutput {- partials: allocation_to_vec(&partials),- sums: allocation_to_vec(&sums),- maxs: allocation_to_vec(&maxs),- }-}--// --- Second pass types and helpers -----struct SecondPassInput {- partials: Box<[f32]>,- sums: Box<[f32]>,- maxs: Box<[f32]>,- num_heads: u32,- suffix_length: u32,- head_dim: u32,-}--fn get_second_pass_input(- num_heads: u32,- suffix_length: u32,- head_dim: u32,-) -> SecondPassInput {- let total_offsets = (suffix_length * num_heads) as usize;- let partials_size = total_offsets * TOTAL_BLOCKS_COUNT as usize * head_dim as usize;- let sums_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;- let maxs_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;-- let mut partials = vec![0.0f32; partials_size];- for i in 0..partials_size {- partials[i] = (i as f32 * 0.03 + 0.1).sin() * 0.5;- }-- let mut sums = vec![0.0f32; sums_size];- for i in 0..sums_size {- // Ensure sums are positive (they represent sum of exp scores)- sums[i] = (i as f32 * 0.07 + 1.0).cos().abs() + 0.1;- }-- let mut maxs = vec![0.0f32; maxs_size];- for i in 0..maxs_size {- maxs[i] = (i as f32 * 0.05 + 0.5).sin() * 2.0;- }-- SecondPassInput {- partials: partials.into_boxed_slice(),- sums: sums.into_boxed_slice(),- maxs: maxs.into_boxed_slice(),- num_heads,- suffix_length,- head_dim,- }-}--fn get_second_pass_output<T: ArrayElement + Float, B: Backend>(input: &SecondPassInput) -> Vec<T> {- let context = B::Context::new().expect("Failed to create Context");-- let kernel =- <<B as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(&context, T::data_type(), input.head_dim)- .expect("Failed to create AttentionTwoPass2Kernel");-- let partials = alloc_allocation_with_data::<B, f32>(&context, &input.partials);- let sums = alloc_allocation_with_data::<B, f32>(&context, &input.sums);- let maxs = alloc_allocation_with_data::<B, f32>(&context, &input.maxs);-- let output_size = (input.suffix_length * input.num_heads * input.head_dim) as usize;- let mut output = alloc_allocation::<B, T>(&context, output_size);-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(&partials, &sums, &maxs, &mut output, input.num_heads, input.suffix_length, &mut encoder);- encoder.end_encoding().submit().wait_until_completed().unwrap();-- allocation_to_vec(&output)-}--// --- First pass tests -----fn test_first_pass_internal<T: ArrayElement + Float + Debug + Display>(- input: &FirstPassInput<T>,- expected: &FirstPassOutput,-) {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {- 1e-2- } else {- 1e-5- };-- for_each_non_cpu_backend!(|B| {- let output = get_first_pass_output::<T, B>(input);- let msg = format!(- "AttentionTwoPass1 failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={})",- std::any::type_name::<B>(),- input.num_heads,- input.sequence_length,- input.suffix_length,- input.head_dim,- input.do_causal,- );- assert_eq_float::<f32>(&expected.partials, &output.partials, eps, &format!("{msg} [partials]"));- assert_eq_float::<f32>(&expected.sums, &output.sums, eps, &format!("{msg} [sums]"));- assert_eq_float::<f32>(&expected.maxs, &output.maxs, eps, &format!("{msg} [maxs]"));- });-}--fn test_first_pass_basic<T: ArrayElement + Float + Debug + Display>() {- let input = get_first_pass_input::<T>(4, 4, 8, 1, 64, false);- let expected = get_first_pass_output::<T, Cpu>(&input);- test_first_pass_internal(&input, &expected);-- let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, false);- let expected = get_first_pass_output::<T, Cpu>(&input);- test_first_pass_internal(&input, &expected);-}--fn test_first_pass_causal<T: ArrayElement + Float + Debug + Display>() {- let input = get_first_pass_input::<T>(4, 4, 16, 1, 64, true);- let expected = get_first_pass_output::<T, Cpu>(&input);- test_first_pass_internal(&input, &expected);-- let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, true);- let expected = get_first_pass_output::<T, Cpu>(&input);- test_first_pass_internal(&input, &expected);-}--fn test_first_pass_gqa<T: ArrayElement + Float + Debug + Display>() {- let input = get_first_pass_input::<T>(8, 2, 8, 1, 64, false);- let expected = get_first_pass_output::<T, Cpu>(&input);- test_first_pass_internal(&input, &expected);-- let input = get_first_pass_input::<T>(8, 2, 8, 4, 64, true);- let expected = get_first_pass_output::<T, Cpu>(&input);- test_first_pass_internal(&input, &expected);-}--fn test_first_pass_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {- let input = get_first_pass_input::<T>(4, 4, 8, 2, head_dim, true);- let expected = get_first_pass_output::<T, Cpu>(&input);- test_first_pass_internal(&input, &expected);-}--// --- Second pass tests -----fn test_second_pass_internal<T: ArrayElement + Float + Debug + Display>(- input: &SecondPassInput,- expected: &[T],-) {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {- 1e-2- } else {- 1e-5- };-- for_each_non_cpu_backend!(|B| {- let output = get_second_pass_output::<T, B>(input);- let msg = format!(- "AttentionTwoPass2 failed (backend={}, heads={}, suffix={}, head_dim={})",- std::any::type_name::<B>(),- input.num_heads,- input.suffix_length,- input.head_dim,- );- assert_eq_float::<T>(expected, &output, eps, &msg);- });-}--fn test_second_pass_basic<T: ArrayElement + Float + Debug + Display>() {- let input = get_second_pass_input(4, 1, 64);- let expected = get_second_pass_output::<T, Cpu>(&input);- test_second_pass_internal::<T>(&input, &expected);-- let input = get_second_pass_input(4, 4, 64);- let expected = get_second_pass_output::<T, Cpu>(&input);- test_second_pass_internal::<T>(&input, &expected);-}--fn test_second_pass_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: u32) {- let input = get_second_pass_input(4, 2, head_dim);- let expected = get_second_pass_output::<T, Cpu>(&input);- test_second_pass_internal::<T>(&input, &expected);-}--// --- First pass test entries -----#[uzu_test]-fn test_first_pass_basic_f32() {- test_first_pass_basic::<f32>();-}--#[uzu_test]-fn test_first_pass_basic_f16() {- test_first_pass_basic::<f16>();-}--#[uzu_test]-fn test_first_pass_basic_bf16() {- test_first_pass_basic::<bf16>();-}--#[uzu_test]-fn test_first_pass_causal_f32() {- test_first_pass_causal::<f32>();-}--#[uzu_test]-fn test_first_pass_causal_f16() {- test_first_pass_causal::<f16>();-}--#[uzu_test]-fn test_first_pass_causal_bf16() {- test_first_pass_causal::<bf16>();-}--#[uzu_test]-fn test_first_pass_gqa_f32() {- test_first_pass_gqa::<f32>();-}--#[uzu_test]-fn test_first_pass_gqa_f16() {- test_first_pass_gqa::<f16>();-}--#[uzu_test]-fn test_first_pass_gqa_bf16() {- test_first_pass_gqa::<bf16>();-}--#[uzu_test]-fn test_first_pass_head_dim_128_f32() {- test_first_pass_head_dim::<f32>(128);-}--#[uzu_test]-fn test_first_pass_head_dim_128_f16() {- test_first_pass_head_dim::<f16>(128);-}--#[uzu_test]-fn test_first_pass_head_dim_128_bf16() {- test_first_pass_head_dim::<bf16>(128);-}--#[uzu_test]-fn test_first_pass_head_dim_512_f32() {- test_first_pass_head_dim::<f32>(512);-}--#[uzu_test]-fn test_first_pass_head_dim_512_f16() {- test_first_pass_head_dim::<f16>(512);-}--#[uzu_test]-fn test_first_pass_head_dim_512_bf16() {- test_first_pass_head_dim::<bf16>(512);-}--// --- Second pass test entries -----#[uzu_test]-fn test_second_pass_basic_f32() {- test_second_pass_basic::<f32>();-}--#[uzu_test]-fn test_second_pass_basic_f16() {- test_second_pass_basic::<f16>();-}--#[uzu_test]-fn test_second_pass_basic_bf16() {- test_second_pass_basic::<bf16>();-}--#[uzu_test]-fn test_second_pass_head_dim_128_f32() {- test_second_pass_head_dim::<f32>(128);-}--#[uzu_test]-fn test_second_pass_head_dim_128_f16() {- test_second_pass_head_dim::<f16>(128);-}--#[uzu_test]-fn test_second_pass_head_dim_128_bf16() {- test_second_pass_head_dim::<bf16>(128);-}--#[uzu_test]-fn test_second_pass_head_dim_512_f32() {- test_second_pass_head_dim::<f32>(512);-}--#[uzu_test]-fn test_second_pass_head_dim_512_f16() {- test_second_pass_head_dim::<f16>(512);-}--#[uzu_test]-fn test_second_pass_head_dim_512_bf16() {- test_second_pass_head_dim::<bf16>(512);-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/kv_cache_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rssimilarity index 97%rename from crates/uzu-engine/unit/backends/common/kernel/kv_cache_update_test.rsrename to crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rsindex 72d8d047a..1e5ee58f2 100644--- a/crates/uzu-engine/unit/backends/common/kernel/kv_cache_update_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs@@ -1,6 +1,6 @@use std::fmt::{Debug, Display};-use half::{bf16, f16};+use half::bf16;#[cfg(backend = "metal")]use ndarray::{Array, Array3};use num_traits::Float;@@ -215,11 +215,6 @@ fn test_single_copy_f32() {test_single_copy_internal::<f32>();}-#[uzu_test]-fn test_single_copy_f16() {- test_single_copy_internal::<f16>();-}-#[uzu_test]fn test_single_copy_bf16() {test_single_copy_internal::<bf16>();@@ -231,11 +226,6 @@ fn test_multi_copy_f32() {test_multi_copy_internal::<f32>();}-#[uzu_test]-fn test_multi_copy_f16() {- test_multi_copy_internal::<f16>();-}-#[uzu_test]fn test_multi_copy_bf16() {test_multi_copy_internal::<bf16>();@@ -247,11 +237,6 @@ fn test_no_copy_f32() {test_no_copy_internal::<f32>();}-#[uzu_test]-fn test_no_copy_f16() {- test_no_copy_internal::<f16>();-}-#[uzu_test]fn test_no_copy_bf16() {test_no_copy_internal::<bf16>();@@ -263,11 +248,6 @@ fn test_large_f32() {test_large_internal::<f32>();}-#[uzu_test]-fn test_large_f16() {- test_large_internal::<f16>();-}-#[uzu_test]fn test_large_bf16() {test_large_internal::<bf16>();diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/mod.rsindex 2d613a765..44cda3596 100644--- a/crates/uzu-engine/unit/backends/common/kernel/attention/mod.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/mod.rs@@ -1,5 +1,4 @@mod ancestor_attention_test;-mod attention_fallback_test;-mod attention_single_pass_test;-mod attention_two_pass_test;+mod kv_cache_update_test;+mod qkv_norm_test;mod sigmoid_gate_test;diff --git a/crates/uzu-engine/unit/backends/common/kernel/qkv_norm_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rssimilarity index 91%rename from crates/uzu-engine/unit/backends/common/kernel/qkv_norm_test.rsrename to crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rsindex b4335f276..6f7ce5b4a 100644--- a/crates/uzu-engine/unit/backends/common/kernel/qkv_norm_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/qkv_norm_test.rs@@ -1,6 +1,6 @@use std::fmt::{Debug, Display};-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use uzu_engine_macros::uzu_test;@@ -163,10 +163,9 @@ fn test_internal<input: &Input<InputT, ScaleT, OutputT>,expected: &[OutputT],) {- let eps = if matches!(InputT::data_type(), DataType::F16 | DataType::BF16)- || matches!(ScaleT::data_type(), DataType::F16 | DataType::BF16)- || matches!(OutputT::data_type(), DataType::F16 | DataType::BF16)- || matches!(AccumT::data_type(), DataType::F16 | DataType::BF16)+ let eps = if [InputT::data_type(), ScaleT::data_type(), OutputT::data_type(), AccumT::data_type()]+ .into_iter()+ .any(|data_type| data_type == DataType::BF16){1e-2} else {@@ -318,47 +317,17 @@ fn test_q_norm_f32_f32_f32_f32() {test_q_norm::<f32, f32, f32, f32>();}-#[uzu_test]-fn test_q_norm_f16_f16_f16_f32() {- test_q_norm::<f16, f16, f16, f32>();-}--#[uzu_test]-fn test_q_norm_f16_f16_f16_f16() {- test_q_norm::<f16, f16, f16, f16>();-}-#[uzu_test]fn test_q_norm_bf16_bf16_bf16_f32() {test_q_norm::<bf16, bf16, bf16, f32>();}-#[uzu_test]-fn test_q_norm_f32_f16_f32_f32() {- test_q_norm::<f32, f16, f32, f32>();-}--#[uzu_test]-fn test_q_norm_f16_f32_f16_f32() {- test_q_norm::<f16, f32, f16, f32>();-}-// K norm tests#[uzu_test]fn test_k_norm_f32_f32_f32_f32() {test_k_norm::<f32, f32, f32, f32>();}-#[uzu_test]-fn test_k_norm_f16_f16_f16_f32() {- test_k_norm::<f16, f16, f16, f32>();-}--#[uzu_test]-fn test_k_norm_f16_f16_f16_f16() {- test_k_norm::<f16, f16, f16, f16>();-}-#[uzu_test]fn test_k_norm_bf16_bf16_bf16_f32() {test_k_norm::<bf16, bf16, bf16, f32>();@@ -369,11 +338,6 @@ fn test_v_norm_no_scales_f32_f32_f32_f32() {test_v_norm_no_scales::<f32, f32, f32, f32>();}-#[uzu_test]-fn test_v_norm_no_scales_f16_f16_f16_f32() {- test_v_norm_no_scales::<f16, f16, f16, f32>();-}-#[uzu_test]fn test_v_norm_no_scales_bf16_bf16_bf16_f32() {test_v_norm_no_scales::<bf16, bf16, bf16, f32>();@@ -385,11 +349,6 @@ fn test_addressing_f32_f32_f32_f32() {test_addressing::<f32, f32, f32, f32>();}-#[uzu_test]-fn test_addressing_f16_f16_f16_f32() {- test_addressing::<f16, f16, f16, f32>();-}-#[uzu_test]fn test_v_addressing_f32_f32_f32_f32() {test_v_addressing::<f32, f32, f32, f32>();diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rsindex 2f3e7c179..5f668e5f7 100644--- a/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs@@ -1,6 +1,6 @@use std::fmt::Debug;-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use uzu_engine_macros::uzu_test;@@ -97,20 +97,6 @@ fn test_sigmoid_gate_f32() {});}-#[uzu_test]-fn test_sigmoid_gate_f16() {- run_test::<f16>(&Config {- num_heads: 8,- head_dim: 64,- suffix_length: 4,- });- run_test::<f16>(&Config {- num_heads: 16,- head_dim: 256,- suffix_length: 1,- });-}-#[uzu_test]fn test_sigmoid_gate_bf16() {run_test::<bf16>(&Config {diff --git a/crates/uzu-engine/unit/backends/common/kernel/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/mod.rsindex 6ab0887dd..e56e9b0d9 100644--- a/crates/uzu-engine/unit/backends/common/kernel/mod.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/mod.rs@@ -4,13 +4,11 @@ mod attention;mod embedding;mod gated_act_mul_test;mod gdn;-mod kv_cache_update_test;mod logit_transform_test;mod matmul;mod moe;mod normalization_test;mod pooling;-mod qkv_norm_test;mod radix_top_k_small_test;mod short_conv;mod ssm;diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rsnew file mode 100644index 000000000..33b42c2c5--- /dev/null+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs@@ -0,0 +1,49 @@+use uzu_engine_macros::uzu_test;++use super::{MaskKind, choose_splits, should_encode};++fn splits(+ cores: u32,+ head_dim: u32,+ suffix_length: u32,+ kv_length: u32,+) -> u32 {+ let (gqa, rows, groups) = if head_dim == 128 {+ (4, 64, 8)+ } else {+ (6, 32, 4)+ };+ let threadgroups = (gqa * suffix_length).div_ceil(rows) * groups;+ choose_splits(head_dim, suffix_length, kv_length, threadgroups, 32, cores)+}++#[uzu_test]+fn measured_and_fallback_boundaries() {+ for (cores, head_dim, suffix, kv, expected) in [+ (40, 256, 16, 5_120, 6),+ (40, 256, 32, 32_768, 10),+ (40, 128, 64, 32_768, 10),+ (40, 256, 17, 262_144, 10),+ (40, 256, 128, 262_144, 3),+ (40, 256, 16, 128, 4),+ (10, 256, 32, 262_144, 3),+ (10, 256, 128, 262_144, 1),+ ] {+ assert_eq!(splits(cores, head_dim, suffix, kv), expected);+ }+}++#[uzu_test]+fn should_encode_boundaries() {+ for (head_dim, mask, suffix_length, kv_length, expected) in [+ (128, MaskKind::Causal, 16, 1_024, true),+ (128, MaskKind::Causal, 32, 1_023, false),+ (256, MaskKind::Causal, 16, 1_023, false),+ (256, MaskKind::Causal, 8, 150_001, false),+ (256, MaskKind::Causal, 1_024, 1_024, true),+ (256, MaskKind::None, 1_024, 1_024, false),+ (256, MaskKind::Trie, 65, 1_024, false),+ ] {+ assert_eq!(should_encode(head_dim, mask, suffix_length, kv_length), expected);+ }+}diff --git a/crates/uzu-engine/unit/encodable_block/attention_gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rssimilarity index 66%rename from crates/uzu-engine/unit/encodable_block/attention_gemm_test.rsrename to crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rsindex d37dcb66e..ba21fe864 100644--- a/crates/uzu-engine/unit/encodable_block/attention_gemm_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs@@ -1,23 +1,24 @@use std::fmt::{Debug, Display};-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use uzu_engine_macros::uzu_test;use crate::{array::ArrayElement,backends::{- common::{Backend, Context, Encoder, Kernels, kernel::attention_gemm::AttentionGemmCore},+ common::{+ Backend, Context, Encoder, Kernels,+ kernel::{AttentionArguments, AttentionKernel, AttentionKernelConfig},+ },cpu::Cpu,+ metal::Metal,},data_type::DataType,- encodable_block::mixer::attention::{- KVCacheView,- core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments, AttentionCores},- },+ encodable_block::mixer::attention::KVCacheView,tests::{assert::assert_eq_float,- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},},};@@ -30,10 +31,13 @@ struct Input<T: ArrayElement + Float> {sequence_length: usize,suffix_length: usize,head_dim: usize,- scale: f32,do_causal: bool,}+fn attention_config<T: ArrayElement + Float>(input: &Input<T>) -> AttentionKernelConfig {+ super::default_attention_config::<T>(input.head_dim, input.num_heads, input.num_kv_heads, input.do_causal)+}+fn get_test_data<T: ArrayElement + Float>(num_heads: usize,num_kv_heads: usize,@@ -60,8 +64,6 @@ fn get_test_data<T: ArrayElement + Float>(values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();}- let scale = 1.0 / (head_dim as f32).sqrt();-let input = Input {queries: queries.into_boxed_slice(),keys: keys.into_boxed_slice(),@@ -71,7 +73,6 @@ fn get_test_data<T: ArrayElement + Float>(sequence_length,suffix_length,head_dim,- scale,do_causal,};@@ -82,25 +83,14 @@ fn get_test_data<T: ArrayElement + Float>(fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");- let new_arguments = AttentionCoreNewArguments {- head_dim: input.head_dim as u32,- num_groups: input.num_kv_heads as u32,- num_q_heads: input.num_heads as u32,- has_sinks: false,- is_kv_cache_ring: false,- is_causal: input.do_causal,- is_trie: false,- sliding_window_size: None,- scale: Some(input.scale),- data_type: T::data_type(),- };+ let config = attention_config(input);let queries_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.queries);let keys_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.keys);let values_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.values);let segment_prefix_length = input.sequence_length - input.suffix_length;- let args = AttentionCoreEncodeArguments {+ let args = AttentionArguments {queries: &queries_allocation,keys: &keys_allocation,values: &values_allocation,@@ -111,52 +101,71 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {};let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- let pooled_output =- if <<B as Backend>::Kernels as Kernels>::AttentionGemmCore::is_supported(&new_arguments, context.as_ref())- .expect("Failed to query AttentionGemm support")- {- let core = <<B as Backend>::Kernels as Kernels>::AttentionGemmCore::new(context.as_ref(), &new_arguments)- .expect("Failed to create AttentionGemmCore");- core.encode(args, &mut encoder).expect("Failed to encode AttentionGemm")- } else {- let core =- AttentionCores::<B>::new(new_arguments, context.as_ref()).expect("Failed to create AttentionCore");- core.encode(args, &mut encoder).expect("Failed to encode AttentionCore")- };+ let kernel = <B::Kernels as Kernels>::AttentionKernel::new(context.as_ref(), config)+ .expect("Failed to create attention kernel");+ let pooled_output = kernel.encode(args, &mut encoder).expect("Failed to encode attention");let mut output_allocation =alloc_allocation::<B, T>(context.as_ref(), input.suffix_length * input.num_heads * input.head_dim);encoder.encode_copy(&pooled_output, .., &mut output_allocation, ..);- let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(pooled_output);+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(completed);allocation_to_vec::<B, T>(&output_allocation)}+fn get_gemm_output<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {+ let context = <Metal as Backend>::Context::new().expect("Failed to create Metal context");+ let config = attention_config(input);+ let queries = alloc_allocation_with_data::<Metal, T>(context.as_ref(), &input.queries);+ let keys = alloc_allocation_with_data::<Metal, T>(context.as_ref(), &input.keys);+ let values = alloc_allocation_with_data::<Metal, T>(context.as_ref(), &input.values);+ let cache = KVCacheView::full((input.sequence_length - input.suffix_length) as u32);+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ let pooled = super::gemm::AttentionGemm::new(&config)+ .encode(+ AttentionArguments {+ queries: &queries,+ keys: &keys,+ values: &values,+ suffix_length: input.suffix_length as u32,+ trie: None,+ sinks: None,+ cache,+ },+ &mut encoder,+ )+ .expect("Failed to encode AttentionGemm");+ let mut output =+ alloc_allocation::<Metal, T>(context.as_ref(), input.suffix_length * input.num_heads * input.head_dim);+ encoder.encode_copy(&pooled, .., &mut output, ..);+ drop(pooled);+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(completed);+ allocation_to_vec::<Metal, T>(&output)+}+fn test_internal<T: ArrayElement + Float + Debug + Display>(input: &Input<T>,expected: &[T],) {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {+ let eps = if T::data_type() == DataType::BF16 {1e-2} else {1e-5};- for_each_non_cpu_backend!(|B| {- let output = get_output::<T, B>(input);- let msg = format!(- "AttentionGemm failed (backend={}, heads={}, kv_heads={}, seq={}, suffix={}, head_dim={}, causal={})",- std::any::type_name::<B>(),- input.num_heads,- input.num_kv_heads,- input.sequence_length,- input.suffix_length,- input.head_dim,- input.do_causal,- );- assert_eq_float::<T>(expected, &output, eps, &msg);- });+ let output = get_gemm_output(input);+ let msg = format!(+ "AttentionGemm failed (heads={}, kv_heads={}, seq={}, suffix={}, head_dim={}, causal={})",+ input.num_heads,+ input.num_kv_heads,+ input.sequence_length,+ input.suffix_length,+ input.head_dim,+ input.do_causal,+ );+ assert_eq_float::<T>(expected, &output, eps, &msg);}fn test_basic<T: ArrayElement + Float + Debug + Display>() {@@ -201,11 +210,6 @@ fn test_basic_f32() {test_basic::<f32>();}-#[uzu_test]-fn test_basic_f16() {- test_basic::<f16>();-}-#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();@@ -216,11 +220,6 @@ fn test_causal_f32() {test_causal::<f32>();}-#[uzu_test]-fn test_causal_f16() {- test_causal::<f16>();-}-#[uzu_test]fn test_causal_bf16() {test_causal::<bf16>();@@ -231,11 +230,6 @@ fn test_gqa_f32() {test_gqa::<f32>();}-#[uzu_test]-fn test_gqa_f16() {- test_gqa::<f16>();-}-#[uzu_test]fn test_gqa_bf16() {test_gqa::<bf16>();@@ -246,11 +240,6 @@ fn test_head_dim_128_f32() {test_head_dim::<f32>(128);}-#[uzu_test]-fn test_head_dim_128_f16() {- test_head_dim::<f16>(128);-}-#[uzu_test]fn test_head_dim_128_bf16() {test_head_dim::<bf16>(128);@@ -261,11 +250,6 @@ fn test_unaligned_f32() {test_unaligned::<f32>();}-#[uzu_test]-fn test_unaligned_f16() {- test_unaligned::<f16>();-}-#[uzu_test]fn test_unaligned_bf16() {test_unaligned::<bf16>();@@ -273,7 +257,7 @@ fn test_unaligned_bf16() {#[uzu_test]fn test_prefill_mxu() {- let (input, expected) = get_test_data::<f16>(4, 4, 128, 128, 64, true);+ let (input, expected) = get_test_data::<bf16>(4, 4, 128, 128, 64, true);test_internal(&input, &expected);let (input, expected) = get_test_data::<bf16>(8, 2, 100, 96, 128, true);test_internal(&input, &expected);diff --git a/crates/uzu-engine/unit/encodable_block/attention_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rssimilarity index 60%rename from crates/uzu-engine/unit/encodable_block/attention_test.rsrename to crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rsindex 52981e5ba..273fc8db6 100644--- a/crates/uzu-engine/unit/encodable_block/attention_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs@@ -1,26 +1,23 @@-#![cfg(backend = "metal")]-+use half::bf16;use ndarray::{Array4, s};-use test_tag::tag;use uzu_engine_macros::uzu_test;use crate::{backends::{common::{Allocation, Backend, Context, Encoder, Kernels,- kernel::{- AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel,- attention_gemm::AttentionGemmCore,- },+ gpu_types::trie::TrieNode as GpuTrieNode,+ kernel::{AttentionArguments, AttentionKernel},},+ cpu::Cpu,metal::Metal,},- data_type::DataType,- encodable_block::mixer::attention::{- KVCacheView,- core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments},+ encodable_block::{mixer::attention::KVCacheView, sampling::PRng},+ tests::{+ assert::assert_eq_float,+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, submit_encoder},},- tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, submit_encoder},+ trie::TrieNode,};fn reference_attention(@@ -215,45 +212,37 @@ fn convert_kernel_output(}fn run_single_pass_attention(- kernel: &<<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel,+ kernel: &super::single_pass::AttentionSinglePass,context: &<Metal as Backend>::Context,queries: &Array4<f32>,keys: &Array4<f32>,values: &Array4<f32>,sinks: Option<&[f32]>,- scale: f32,+ _scale: f32,) -> Result<Array4<f32>, Box<dyn std::error::Error>> {let (batch_size, num_heads, seq_len, head_dim) = queries.dim();- let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();+ let (_batch_size, _num_kv_heads, _seq_len, _head_dim) = keys.dim();let query_buffer = create_query_allocation(queries, context);let key_cache_buffer = create_attention_cache_allocation(keys, seq_len, context);let value_cache_buffer = create_attention_cache_allocation(values, seq_len, context);let sinks_buffer = sinks.map(|sinks| create_sinks_allocation(sinks, context));- let mut output_buffer = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);-let mut encoder = Encoder::new(context).expect("Failed to create encoder");-- kernel.encode(- &query_buffer,- &key_cache_buffer,- &value_cache_buffer,- &mut output_buffer,- (num_heads / num_kv_heads) as u32,- seq_len as u32,- head_dim as u32,- (num_kv_heads * head_dim) as u32,- head_dim as u32,- (num_kv_heads * head_dim) as u32,- None,- scale,- None::<&Allocation<Metal>>,- None,- sinks_buffer.as_ref(),- num_heads as u32,- seq_len as u32,+ let pooled_output = kernel.encode(+ AttentionArguments {+ queries: &query_buffer,+ keys: &key_cache_buffer,+ values: &value_cache_buffer,+ suffix_length: seq_len as u32,+ trie: None,+ sinks: sinks_buffer.as_ref(),+ cache: KVCacheView::full(0),+ },&mut encoder,- );+ )?;+ let mut output_buffer = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);+ encoder.encode_copy(&pooled_output, .., &mut output_buffer, ..);+ drop(pooled_output);submit_encoder(encoder);let output_slice: Vec<f32> = allocation_to_vec(&output_buffer);@@ -263,47 +252,26 @@ fn run_single_pass_attention(}fn create_single_pass_kernel(- context: &<Metal as Backend>::Context,head_dim: usize,+ num_q_heads: usize,+ num_groups: usize,has_sinks: bool,is_causal: bool,-) -> <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel {- <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(- context,- DataType::F32,- head_dim as u32,- has_sinks,- false,- is_causal,- false,- false,- )- .expect("Failed to create attention single pass kernel")+) -> super::single_pass::AttentionSinglePass {+ let mut config = super::default_attention_config::<f32>(head_dim, num_q_heads, num_groups, is_causal);+ config.has_sinks = has_sinks;+ config.scale = None;+ super::single_pass::AttentionSinglePass::new(&config)}-fn create_two_pass_kernels(- context: &<Metal as Backend>::Context,+fn create_two_pass_kernel(head_dim: usize,+ num_q_heads: usize,+ num_groups: usize,is_causal: bool,-) -> (- <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel,- <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel,-) {- (- <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(- context,- DataType::F32,- head_dim as u32,- false,- false,- is_causal,- false,- false,- )- .expect("Failed to create AttentionTwoPass1Kernel"),- <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, DataType::F32, head_dim as u32)- .expect("Failed to create AttentionTwoPass2Kernel"),- )+) -> super::two_pass::AttentionTwoPass {+ let config = super::default_attention_config::<f32>(head_dim, num_q_heads, num_groups, is_causal);+ super::two_pass::AttentionTwoPass::new(&config)}fn run_gemm_attention(@@ -318,21 +286,10 @@ fn run_gemm_attention(let (batch_size, num_heads, seq_len, head_dim) = queries.dim();let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();- let core = <<Metal as Backend>::Kernels as Kernels>::AttentionGemmCore::new(- context,- &AttentionCoreNewArguments {- head_dim: head_dim as u32,- num_groups: num_kv_heads as u32,- num_q_heads: num_heads as u32,- has_sinks: sinks.is_some(),- is_kv_cache_ring: false,- is_causal,- is_trie: false,- sliding_window_size: None,- scale: Some(scale),- data_type: DataType::F32,- },- )?;+ let mut config = super::default_attention_config::<f32>(head_dim, num_heads, num_kv_heads, is_causal);+ config.has_sinks = sinks.is_some();+ config.scale = Some(scale);+ let kernel = super::gemm::AttentionGemm::new(&config);let query_allocation = create_query_allocation(queries, context);let key_allocation = create_attention_cache_allocation(keys, seq_len, context);@@ -341,7 +298,7 @@ fn run_gemm_attention(let sinks_allocation = sinks.map(|sinks| create_sinks_allocation(sinks, context));let mut encoder = Encoder::new(context).expect("Failed to create encoder");- let args = AttentionCoreEncodeArguments {+ let args = AttentionArguments {queries: &query_allocation,keys: &key_allocation,values: &value_allocation,@@ -351,7 +308,7 @@ fn run_gemm_attention(cache: KVCacheView::full(0),};- let pooled_output = core.encode(args, &mut encoder)?;+ let pooled_output = kernel.encode(args, &mut encoder)?;let mut output_allocation = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);encoder.encode_copy(&pooled_output, .., &mut output_allocation, ..);let completed = encoder.end_encoding().submit().wait_until_completed()?;@@ -397,53 +354,13 @@ fn test_single_pass_attention_basic() {let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);- let kernel = create_single_pass_kernel(&context, head_dim, false, false);+ let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, false, false);let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale).expect("single-pass attention");compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention").unwrap();}-#[uzu_test]-fn test_gemm_attention_basic() {- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");-- let batch_size = 1;- let num_heads = 4;- let num_kv_heads = 4;- let seq_len = 32;- let head_dim = 64;- let scale = 1.0 / (head_dim as f32).sqrt();-- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);-- let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);- let kernel_output =- run_gemm_attention(&context, &queries, &keys, &values, None, scale, false).expect("run gemm attention");-- compare_results(&kernel_output, &reference_output, 1e-2, "Gemm attention").unwrap();-}--#[uzu_test]-fn test_gemm_attention_f32_head_dim_128() {- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");-- let batch_size = 1;- let num_heads = 4;- let num_kv_heads = 4;- let seq_len = 32;- let head_dim = 128;- let scale = 1.0 / (head_dim as f32).sqrt();-- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);-- let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);- let kernel_output = run_gemm_attention(&context, &queries, &keys, &values, None, scale, false)- .expect("run gemm attention f32 head_dim=128");-- compare_results(&kernel_output, &reference_output, 1e-2, "Gemm attention f32 head_dim=128").unwrap();-}-#[uzu_test]fn test_matrix_attention_matches_vector_and_cpu_seq256() {let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");@@ -459,7 +376,7 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);let reference_output = reference_attention(&queries, &keys, &values, None, scale, is_causal);- let kernel = create_single_pass_kernel(&context, head_dim, false, is_causal);+ let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, false, is_causal);let vector_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale).expect("run vector attention");let matrix_output =@@ -493,7 +410,7 @@ fn test_single_pass_attention_with_sinks() {let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 444);let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();- let kernel = create_single_pass_kernel(&context, head_dim, true, false);+ let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, true, false);let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale)@@ -516,7 +433,7 @@ fn test_single_pass_attention_with_sinks_long_sequence() {let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 777);let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();- let kernel = create_single_pass_kernel(&context, head_dim, true, false);+ let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, true, false);let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale)@@ -539,7 +456,7 @@ fn test_single_pass_attention_gqa() {let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);- let kernel = create_single_pass_kernel(&context, head_dim, false, false);+ let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, false, false);let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale).expect("single-pass attention GQA");@@ -548,65 +465,36 @@ fn test_single_pass_attention_gqa() {}fn run_two_pass_attention(- kernel_pass1: &<<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel,- kernel_pass2: &<<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel,+ kernel: &super::two_pass::AttentionTwoPass,context: &<Metal as Backend>::Context,queries: &Array4<f32>,keys: &Array4<f32>,values: &Array4<f32>,sinks: Option<&[f32]>,- scale: f32,+ _scale: f32,) -> Result<Array4<f32>, Box<dyn std::error::Error>> {let (batch_size, num_heads, seq_len, head_dim) = queries.dim();- let (_, num_kv_heads, _, _) = keys.dim();-let queries_buffer = create_query_allocation(queries, context);let keys_buffer = create_attention_cache_allocation(keys, seq_len, context);let values_buffer = create_attention_cache_allocation(values, seq_len, context);let sinks_buffer = sinks.map(|sinks| create_sinks_allocation(sinks, context));-- let total_blocks_count = 32;- let partials_size = num_heads * seq_len * total_blocks_count * head_dim;- let sums_maxs_size = num_heads * seq_len * total_blocks_count;-- let mut partials_buffer = alloc_allocation::<Metal, f32>(context, partials_size);- let mut sums_buffer = alloc_allocation::<Metal, f32>(context, sums_maxs_size);- let mut maxs_buffer = alloc_allocation::<Metal, f32>(context, sums_maxs_size);- let mut output_buffer = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);-let mut encoder = Encoder::new(context).expect("Failed to create encoder");- kernel_pass1.encode(- &queries_buffer,- &keys_buffer,- &values_buffer,- &mut partials_buffer,- &mut sums_buffer,- &mut maxs_buffer,- (num_heads / num_kv_heads) as u32,- seq_len as u32,- head_dim as u32,- (num_kv_heads * head_dim) as u32,- head_dim as u32,- (num_kv_heads * head_dim) as u32,- None,- scale,- num_heads as u32,- seq_len as u32,- None::<&Allocation<Metal>>,- None,- sinks_buffer.as_ref(),- &mut encoder,- );- kernel_pass2.encode(- &partials_buffer,- &sums_buffer,- &maxs_buffer,- &mut output_buffer,- num_heads as u32,- seq_len as u32,+ let pooled_output = kernel.encode(+ AttentionArguments {+ queries: &queries_buffer,+ keys: &keys_buffer,+ values: &values_buffer,+ suffix_length: seq_len as u32,+ trie: None,+ sinks: sinks_buffer.as_ref(),+ cache: KVCacheView::full(0),+ },&mut encoder,- );+ )?;+ let mut output_buffer = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);+ encoder.encode_copy(&pooled_output, .., &mut output_buffer, ..);+ drop(pooled_output);submit_encoder(encoder);let output_slice: Vec<f32> = allocation_to_vec(&output_buffer);@@ -631,10 +519,9 @@ fn test_two_pass_attention() {let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);- let (kernel_pass1, kernel_pass2) = create_two_pass_kernels(&context, head_dim, is_causal);+ let kernel = create_two_pass_kernel(head_dim, num_heads, num_kv_heads, is_causal);let kernel_output =- run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale)- .expect("two-pass attention");+ run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale).expect("two-pass attention");compare_results(&kernel_output, &reference_output, 1e-2, "Two-pass attention").unwrap();}@@ -655,100 +542,202 @@ fn test_two_pass_attention_gqa() {let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);- let (kernel_pass1, kernel_pass2) = create_two_pass_kernels(&context, head_dim, is_causal);- let kernel_output =- run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale)- .expect("two-pass attention GQA");+ let kernel = create_two_pass_kernel(head_dim, num_heads, num_kv_heads, is_causal);+ let kernel_output = run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)+ .expect("two-pass attention GQA");compare_results(&kernel_output, &reference_output, 1e-2, "Two-pass attention GQA").unwrap();}-#[tag(heavy)]+type AttentionShape = (usize, usize, usize, usize, usize, bool);++fn attention_data(shape: AttentionShape) -> (Vec<bf16>, Vec<bf16>) {+ let (head_dim, _, num_groups, suffix_length, prefix_length, _) = shape;+ let row = num_groups * head_dim;+ let count = (prefix_length + suffix_length) * row;+ (fill_attention(count, 1.0), fill_attention(count, 2.0))+}++fn fill_attention(+ count: usize,+ phase: f32,+) -> Vec<bf16> {+ (0..count).map(|i| bf16::from_f32(((i as f32) * 0.017 + phase).sin() * 0.5)).collect()+}++fn run_attention<B: Backend>(+ context: &B::Context,+ shape: AttentionShape,+ trie_nodes: Option<&[GpuTrieNode]>,+) -> Vec<bf16> {+ let (keys, values) = attention_data(shape);+ let (_, _, _, _, prefix_length, _) = shape;+ let (head_dim, num_q_heads, num_groups, _, _, is_causal) = shape;+ let config = super::default_attention_config::<bf16>(head_dim, num_q_heads, num_groups, is_causal);+ let kernel = <B::Kernels as Kernels>::AttentionKernel::new(context, config).expect("attention kernel");+ run_attention_with_kernel::<B>(+ context,+ &kernel,+ shape,+ &keys,+ &values,+ KVCacheView::full(prefix_length as u32),+ trie_nodes,+ )+}++fn run_attention_with_kernel<B: Backend>(+ context: &B::Context,+ kernel: &<B::Kernels as Kernels>::AttentionKernel,+ shape: AttentionShape,+ keys_data: &[bf16],+ values_data: &[bf16],+ cache: KVCacheView,+ trie_nodes: Option<&[GpuTrieNode]>,+) -> Vec<bf16> {+ let (head_dim, num_q_heads, _, suffix_length, _, _) = shape;+ let queries =+ alloc_allocation_with_data::<B, bf16>(context, &fill_attention(num_q_heads * suffix_length * head_dim, 0.5));+ let keys = alloc_allocation_with_data::<B, bf16>(context, keys_data);+ let values = alloc_allocation_with_data::<B, bf16>(context, values_data);+ let trie = trie_nodes.map(|nodes| {+ let words: Vec<u32> = nodes.iter().flat_map(|node| [node.trie_start, node.trie_end, node.height]).collect();+ alloc_allocation_with_data::<B, u32>(context, &words)+ });+ let arguments = AttentionArguments {+ queries: &queries,+ keys: &keys,+ values: &values,+ suffix_length: suffix_length as u32,+ trie: trie.as_ref(),+ sinks: None,+ cache,+ };+ let mut encoder = Encoder::<B>::new(context).expect("encoder");+ let pooled = kernel.encode(arguments, &mut encoder).expect("encode");+ let mut output = alloc_allocation::<B, bf16>(context, suffix_length * num_q_heads * head_dim);+ encoder.encode_copy(&pooled, .., &mut output, ..);+ let completed = encoder.end_encoding().submit().wait_until_completed().expect("submit");+ drop(pooled);+ drop(completed);+ allocation_to_vec::<B, bf16>(&output)+}+#[uzu_test]-fn perf_two_pass_attention() {- use std::time::Instant;+fn attention_kernel_matches_cpu() {+ let trie: Vec<GpuTrieNode> =+ TrieNode::flat(0, &[0, 1, 2, 3, 4], &PRng::new(0)).linearize().token_subtrie_ranges().collect();+ let cpu_context = <Cpu as Backend>::Context::new().expect("CPU attention context");+ let metal_context = <Metal as Backend>::Context::new().expect("Metal attention context");+ for &(head_dim, num_q_heads, num_groups, suffix_length, prefix_length, causal, use_trie) in &[+ (512, 8, 8, 9, 0, false, false),+ (128, 8, 2, 16, 1024, false, false),+ (256, 6, 1, 32, 1024, true, false),+ (256, 6, 1, 31, 1024, true, true),+ (512, 8, 8, 1, 0, false, false),+ (512, 8, 8, 1, 1024, false, false),+ (64, 8, 8, 9, 0, false, false),+ ] {+ let nodes = use_trie.then_some(trie.as_slice());+ let shape = (head_dim, num_q_heads, num_groups, suffix_length, prefix_length, causal);+ let expected = run_attention::<Cpu>(cpu_context.as_ref(), shape, nodes);+ let actual = run_attention::<Metal>(metal_context.as_ref(), shape, nodes);+ let label = format!("attention kernel D{head_dim} S{suffix_length} L{prefix_length} causal={causal}");+ assert_eq_float::<bf16>(&expected, &actual, 1e-2, &label);+ }+}- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");+#[uzu_test]+fn attention_kernel_reuses_instance_for_flat_and_trie() {+ let shape = (256, 6, 1, 5, 1024, true);+ let trie: Vec<GpuTrieNode> =+ TrieNode::flat(0, &[0, 1, 2, 3, 4], &PRng::new(0)).linearize().token_subtrie_ranges().collect();+ let context = <Metal as Backend>::Context::new().expect("Metal attention context");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionKernel::new(+ context.as_ref(),+ super::default_attention_config::<bf16>(shape.0, shape.1, shape.2, shape.5),+ )+ .expect("attention kernel");+ let (keys, values) = attention_data(shape);+ let cache = KVCacheView::full(1024);+ let flat = run_attention_with_kernel::<Metal>(context.as_ref(), &kernel, shape, &keys, &values, cache, None);+ let trie_output =+ run_attention_with_kernel::<Metal>(context.as_ref(), &kernel, shape, &keys, &values, cache, Some(&trie));+ let cpu_context = <Cpu as Backend>::Context::new().expect("CPU attention context");+ let cpu_kernel = <<Cpu as Backend>::Kernels as Kernels>::AttentionKernel::new(+ cpu_context.as_ref(),+ super::default_attention_config::<bf16>(shape.0, shape.1, shape.2, shape.5),+ )+ .expect("CPU attention kernel");+ let expected_flat =+ run_attention_with_kernel::<Cpu>(cpu_context.as_ref(), &cpu_kernel, shape, &keys, &values, cache, None);+ let expected_trie =+ run_attention_with_kernel::<Cpu>(cpu_context.as_ref(), &cpu_kernel, shape, &keys, &values, cache, Some(&trie));+ assert_eq_float::<bf16>(&expected_flat, &flat, 1e-2, "flat mask specialization");+ assert_eq_float::<bf16>(&expected_trie, &trie_output, 1e-2, "trie mask specialization");+}- // ---- Problem sizes requiring two-pass ----- let batch_size = 1;- let num_heads = 32;- let num_kv_heads = 32;- let seq_len = 8192; // Large sequence length (prefix + suffix)- let suffix_length = 1; // Only processing 1 new token (realistic inference)- let head_dim = 128;- let scale = 1.0 / (head_dim as f32).sqrt();- let is_causal = false;-- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);- let (kernel_pass1, kernel_pass2) = create_two_pass_kernels(&context, head_dim, is_causal);-- let suffix_start = seq_len - suffix_length;- let queries_suffix = queries.slice(s![.., .., suffix_start.., ..]).to_owned();- let queries_buffer = create_query_allocation(&queries_suffix, &context);- let keys_buffer = create_attention_cache_allocation(&keys, seq_len, &context);- let values_buffer = create_attention_cache_allocation(&values, seq_len, &context);-- let total_blocks_count = 32;- let partials_size = num_heads * suffix_length * total_blocks_count * head_dim;- let sums_maxs_size = num_heads * suffix_length * total_blocks_count;-- let mut partials_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), partials_size);- let mut sums_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), sums_maxs_size);- let mut maxs_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), sums_maxs_size);- let mut output_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), num_heads * suffix_length * head_dim);-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");-- let sinks_allocation: Option<Allocation<Metal>> = None;- kernel_pass1.encode(- &queries_buffer,- &keys_buffer,- &values_buffer,- &mut partials_buffer,- &mut sums_buffer,- &mut maxs_buffer,- (num_heads / num_kv_heads) as u32,- seq_len as u32,- head_dim as u32,- (num_kv_heads * head_dim) as u32,- head_dim as u32,- (num_kv_heads * head_dim) as u32,- None,- scale,- num_heads as u32,- suffix_length as u32,- None::<&Allocation<Metal>>,+#[uzu_test]+fn attention_kernel_ring_matches_full_on_wrap() {+ const HEAD_DIM: usize = 64;+ const NUM_HEADS: usize = 8;+ const CAPACITY: usize = 8;+ const CONTEXT: usize = 12;+ const SUFFIX: usize = 2;+ let shape = (HEAD_DIM, NUM_HEADS, NUM_HEADS, SUFFIX, CONTEXT, true);+ let row = NUM_HEADS * HEAD_DIM;+ let (full_keys, full_values) = attention_data(shape);+ let mut ring_keys = vec![bf16::ZERO; (CAPACITY + SUFFIX) * row];+ let mut ring_values = vec![bf16::ZERO; (CAPACITY + SUFFIX) * row];+ let ring_offset = (CONTEXT - CAPACITY) % CAPACITY;+ for index in 0..CAPACITY {+ let source = CONTEXT - CAPACITY + index;+ let destination = (ring_offset + index) % CAPACITY;+ ring_keys[destination * row..(destination + 1) * row]+ .copy_from_slice(&full_keys[source * row..(source + 1) * row]);+ ring_values[destination * row..(destination + 1) * row]+ .copy_from_slice(&full_values[source * row..(source + 1) * row]);+ }+ for index in 0..SUFFIX {+ let source = CONTEXT + index;+ let destination = CAPACITY + index;+ ring_keys[destination * row..(destination + 1) * row]+ .copy_from_slice(&full_keys[source * row..(source + 1) * row]);+ ring_values[destination * row..(destination + 1) * row]+ .copy_from_slice(&full_values[source * row..(source + 1) * row]);+ }++ let context = <Metal as Backend>::Context::new().expect("Metal attention context");+ let full_kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionKernel::new(context.as_ref(), {+ let mut config = super::default_attention_config::<bf16>(shape.0, shape.1, shape.2, shape.5);+ config.sliding_window_size = Some(CAPACITY as u32);+ config+ })+ .expect("full attention kernel");+ let ring_kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionKernel::new(context.as_ref(), {+ let mut config = super::default_attention_config::<bf16>(shape.0, shape.1, shape.2, shape.5);+ config.is_kv_cache_ring = true;+ config.sliding_window_size = Some(CAPACITY as u32);+ config+ })+ .expect("ring attention kernel");+ let full = run_attention_with_kernel::<Metal>(+ context.as_ref(),+ &full_kernel,+ shape,+ &full_keys,+ &full_values,+ KVCacheView::full(CONTEXT as u32),None,- sinks_allocation.as_ref(),- &mut encoder,);- kernel_pass2.encode(- &partials_buffer,- &sums_buffer,- &maxs_buffer,- &mut output_buffer,- num_heads as u32,- suffix_length as u32,- &mut encoder,- );- let host_timer = Instant::now();- let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();- let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;-- let gpu_time_ms = completed.gpu_execution_time().as_secs_f64() * 1e3;- println!(- "Two-pass attention perf (heads={}, prefix={}, suffix={}, head_dim={}): GPU={:.2} ms, Host-side={:.2} ms",- num_heads,- seq_len - suffix_length,- suffix_length,- head_dim,- gpu_time_ms,- host_elapsed_ms+ let ring = run_attention_with_kernel::<Metal>(+ context.as_ref(),+ &ring_kernel,+ shape,+ &ring_keys,+ &ring_values,+ KVCacheView::ring(CAPACITY as u32, ring_offset as u32),+ None,);-- let output_slice: Vec<f32> = allocation_to_vec(&output_buffer);- for &val in output_slice.iter().take(100) {- assert!(val.is_finite(), "Output contains non-finite values");- }+ assert_eq_float::<bf16>(&full, &ring, 1e-2, "wrapped ring attention");}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rsnew file mode 100644index 000000000..888095db5--- /dev/null+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/mod.rs@@ -0,0 +1,26 @@+#![cfg(backend = "metal")]++use super::{gemm, single_pass, two_pass};+use crate::{array::ArrayElement, backends::common::kernel::AttentionKernelConfig};++mod gemm_test;+mod kernel_test;++fn default_attention_config<T: ArrayElement>(+ head_dim: usize,+ num_q_heads: usize,+ num_groups: usize,+ is_causal: bool,+) -> AttentionKernelConfig {+ AttentionKernelConfig {+ head_dim: head_dim as u32,+ num_groups: num_groups as u32,+ num_q_heads: num_q_heads as u32,+ has_sinks: false,+ is_kv_cache_ring: false,+ is_causal,+ sliding_window_size: None,+ scale: Some(1.0 / (head_dim as f32).sqrt()),+ data_type: T::data_type(),+ }+}