A high-performance inference engine for AI models
Remove f32 attention (#946)
CCYeh432da1b7bf42
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metalindex dbb45dfe0..4af0b3324 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal@@ -11,7 +11,7 @@ using namespace uzu::ring;using namespace uzu::trie;template <typename T>-VARIANTS(T, float, bfloat)+VARIANTS(T, bfloat)KERNEL(AttentionFallbackScatterScores)(const device T* group_scores,device T* scores,@@ -59,7 +59,7 @@ KERNEL(AttentionFallbackScatterScores)(}template <typename T>-VARIANTS(T, float, bfloat)+VARIANTS(T, bfloat)KERNEL(AttentionFallbackScatterValues)(const device T* group_output,device T* out,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metalindex b31d7b054..0cdfb142d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal@@ -37,12 +37,11 @@ struct AttentionGemmLayout {};template <typename T, uint BK, uint BD, bool USE_MXU>-VARIANTS(T, float, bfloat)+VARIANTS(T, bfloat)VARIANTS(BK, 16, 32)VARIANTS(BD, 64, 128, 256)VARIANTS(USE_MXU, false, true)CONSTRAINT(!USE_MXU || BK == 32)-CONSTRAINT(!USE_MXU || T != "float")CONSTRAINT(!USE_MXU || BD != 256)KERNEL(AttentionGemm)(const device T* q,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metalindex 42869cc1d..75651d57e 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -13,7 +13,7 @@ using namespace uzu::ring;using namespace uzu::trie;template <typename T, uint HEAD_DIM>-VARIANTS(T, float, bfloat)+VARIANTS(T, bfloat)VARIANTS(HEAD_DIM, 64, 128, 256, 512)PUBLIC KERNEL(AttentionSinglePass)(const device T* queries,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metalindex a34d453ef..bf76100bf 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -19,7 +19,7 @@ using namespace uzu::trie;// Each simdgroup independently handles one query head and derives its KV head// via integer division. Simdgroups sharing a KV head benefit from L1 cache.template <typename T, uint HEAD_DIM>-VARIANTS(T, float, bfloat)+VARIANTS(T, bfloat)VARIANTS(HEAD_DIM, 64, 128, 256, 512)KERNEL(AttentionTwoPass1)(const device T* queries,@@ -140,7 +140,7 @@ KERNEL(AttentionTwoPass1)(}template <typename T, uint HEAD_DIM>-VARIANTS(T, float, bfloat)+VARIANTS(T, bfloat)VARIANTS(HEAD_DIM, 64, 128, 256, 512)KERNEL(AttentionTwoPass2)(const device float* partials,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rsindex b8a0b01c9..114940f4a 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs@@ -39,7 +39,7 @@ pub struct AttentionFallback {impl AttentionFallback {pub fn is_supported(config: &AttentionKernelConfig) -> bool {- config.head_dim == HEAD_DIM && matches!(config.data_type, DataType::BF16 | DataType::F32)+ config.head_dim == HEAD_DIM && config.data_type == DataType::BF16}pub fn new(diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rsindex 957e25657..6c4ea891f 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs@@ -55,7 +55,7 @@ fn retile_params(impl AttentionGemm {pub fn is_supported(config: &AttentionKernelConfig) -> bool {- matches!(config.head_dim, 64 | 128 | 256) && matches!(config.data_type, DataType::BF16 | DataType::F32)+ matches!(config.head_dim, 64 | 128 | 256) && config.data_type == DataType::BF16}pub fn new(config: &AttentionKernelConfig) -> Self {@@ -127,10 +127,8 @@ impl AttentionGemm {let mut output = command_buffer.allocate_scratch_for_shape(&[arguments.suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;- let use_mxu = arguments.suffix_length >= 64- && matches!(self.data_type, DataType::BF16)- && matches!(self.head_dim, 64 | 128)- && command_buffer.context().supports_mxu;+ let use_mxu =+ arguments.suffix_length >= 64 && matches!(self.head_dim, 64 | 128) && command_buffer.context().supports_mxu;let (bq, bk) = if use_mxu {(64, 32)} else {diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metalindex b75c7709b..9fdfab026 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal@@ -4,7 +4,7 @@#include "kv_cache_update.h"template <typename T>-VARIANTS(T, float, bfloat)+VARIANTS(T, bfloat)PUBLIC KERNEL(KVCacheUpdate) (device T* in_place_keys,device T* in_place_values,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metalindex bb66922da..e4ef85676 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal@@ -3,7 +3,7 @@#include "attention_gate.h"template <typename T>-VARIANTS(T, float, bfloat)+VARIANTS(T, bfloat)PUBLIC KERNEL(SigmoidGate)(const device T* gate,device T* output,diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rsindex 9b722d016..845c24522 100644--- a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs@@ -1,8 +1,6 @@use std::fmt::{Debug, Display};use half::bf16;-#[cfg(backend = "metal")]-use ndarray::{Array, Array3};use num_traits::Float;use uzu_engine_macros::uzu_test;@@ -20,8 +18,6 @@ use crate::{helpers::{buffer_to_vec, create_buffer_with_data, for_each_non_cpu_backend},},};-#[cfg(backend = "metal")]-use crate::{backends::metal::Metal, data_type::DataType};struct Input<T: ArrayElement + Float> {keys: Box<[T]>,@@ -109,6 +105,10 @@ fn get_test_data_multi_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vecsource: 0,destination: 5,},+ Copy {+ source: 3,+ destination: 6,+ },];let input = Input {@@ -216,163 +216,25 @@ fn test_large_internal<T: ArrayElement + Float + Debug + Display>() {}// Single copy tests-#[uzu_test]-fn test_single_copy_f32() {- test_single_copy_internal::<f32>();-}-#[uzu_test]fn test_single_copy_bf16() {test_single_copy_internal::<bf16>();}// Multi copy tests-#[uzu_test]-fn test_multi_copy_f32() {- test_multi_copy_internal::<f32>();-}-#[uzu_test]fn test_multi_copy_bf16() {test_multi_copy_internal::<bf16>();}// No copy tests-#[uzu_test]-fn test_no_copy_f32() {- test_no_copy_internal::<f32>();-}-#[uzu_test]fn test_no_copy_bf16() {test_no_copy_internal::<bf16>();}// Large dimension tests-#[uzu_test]-fn test_large_f32() {- test_large_internal::<f32>();-}-#[uzu_test]fn test_large_bf16() {test_large_internal::<bf16>();}--#[cfg(backend = "metal")]-fn apply_copies_3d<T: Clone>(- array: &mut Array3<T>,- copies: &[Copy],-) {- let (_seq_len, num_heads, head_dim) = array.dim();- for head in 0..num_heads {- for channel in 0..head_dim {- for copy in copies {- let src = copy.source as usize;- let dst = copy.destination as usize;- array[(dst, head, channel)] = array[(src, head, channel)].clone();- }- }- }-}--#[cfg(backend = "metal")]-#[uzu_test]-fn test_random_pattern_f32() {- let context = match <Metal as Backend>::Context::new() {- Ok(context) => context,- Err(error) => {- eprintln!("Failed to create Metal context: {error:?}. Skipping test.");- return;- },- };-- let kernel = <<Metal as Backend>::Kernels as Kernels>::KVCacheUpdateKernel::new(&context, DataType::F32)- .expect("Failed to create KVCacheUpdateKernel");-- let num_heads = 3usize;- let seq_len = 15usize;- let head_dim = 7usize;- let element_dim = num_heads * head_dim;-- let key_data = Array3::<f32>::from_shape_fn((seq_len, num_heads, head_dim), |(token, head, channel)| {- (token * 1_000_000 + head * 100 + channel * 10) as f32- });-- let value_data = Array3::<f32>::from_shape_fn((seq_len, num_heads, head_dim), |(token, head, channel)| {- (token * 1_000_000 + head * 100 + channel * 10 + 1_000) as f32- });-- let copies = vec![- Copy {- source: 0,- destination: 14,- },- Copy {- source: 3,- destination: 11,- },- Copy {- source: 6,- destination: 8,- },- Copy {- source: 9,- destination: 5,- },- Copy {- source: 12,- destination: 2,- },- Copy {- source: 2,- destination: 12,- },- Copy {- source: 5,- destination: 9,- },- Copy {- source: 8,- destination: 6,- },- Copy {- source: 11,- destination: 3,- },- Copy {- source: 14,- destination: 0,- },- ];-- let mut expected_keys = key_data.clone();- let mut expected_values = value_data.clone();- apply_copies_3d(&mut expected_keys, &copies);- apply_copies_3d(&mut expected_values, &copies);-- let mut key_buffer = create_buffer_with_data::<Metal, f32>(&context, key_data.as_slice().unwrap());- let mut value_buffer = create_buffer_with_data::<Metal, f32>(&context, value_data.as_slice().unwrap());-- let mut command_buffer = context.create_command_buffer(None, None).unwrap();- kernel.encode(- &mut key_buffer,- &mut value_buffer,- &copies,- copies.len() as u32,- element_dim as u32,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let key_values: Vec<f32> = buffer_to_vec(&key_buffer);- let key_result = Array::from_shape_vec((seq_len, num_heads, head_dim), key_values)- .expect("Failed to convert key result to ndarray");-- let value_values: Vec<f32> = buffer_to_vec(&value_buffer);- let value_result = Array::from_shape_vec((seq_len, num_heads, head_dim), value_values)- .expect("Failed to convert value result to ndarray");-- assert_eq!(key_result, expected_keys);- assert_eq!(value_result, expected_values);-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rsindex 0a8b121e1..fd13f9aff 100644--- a/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs@@ -84,11 +84,7 @@ fn run_test<T: ArrayElement + Float + Debug>(config: &Config) {}).collect::<Vec<_>>();- let rtol = if std::mem::size_of::<T>() <= 2 {- 0.01- } else {- 1e-5- };+ let rtol = 0.01;let assert_output = |result: &[T], backend: &str| {for (i, (got, exp)) in result.iter().zip(expected.iter()).enumerate() {@@ -118,39 +114,23 @@ fn run_test<T: ArrayElement + Float + Debug>(config: &Config) {}#[uzu_test]-fn test_sigmoid_gate_f32() {- run_test::<f32>(&Config {+fn test_sigmoid_gate_bf16() {+ run_test::<bf16>(&Config {num_heads: 8,num_kv_heads: 2,head_dim: 64,suffix_length: 4,});- run_test::<f32>(&Config {+ run_test::<bf16>(&Config {num_heads: 16,num_kv_heads: 4,head_dim: 256,suffix_length: 1,});- run_test::<f32>(&Config {+ run_test::<bf16>(&Config {num_heads: 2,num_kv_heads: 1,head_dim: 64,suffix_length: 8,});}--#[uzu_test]-fn test_sigmoid_gate_bf16() {- run_test::<bf16>(&Config {- num_heads: 8,- num_kv_heads: 2,- head_dim: 64,- suffix_length: 4,- });- run_test::<bf16>(&Config {- num_heads: 16,- num_kv_heads: 4,- head_dim: 256,- suffix_length: 1,- });-}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rsindex 48a16d455..a5f6b753c 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs@@ -14,7 +14,6 @@ use crate::{cpu::Cpu,metal::Metal,},- data_type::DataType,encodable_block::mixer::attention::KVCacheView,tests::{assert::assert_eq_float,@@ -151,11 +150,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(input: &Input<T>,expected: &[T],) {- let eps = if T::data_type() == DataType::BF16 {- 1e-2- } else {- 1e-5- };+ let eps = 1e-2;let output = get_gemm_output(input);let msg = format!(@@ -207,51 +202,26 @@ fn test_unaligned<T: ArrayElement + Float + Debug + Display>() {test_internal(&input, &expected);}-#[uzu_test]-fn test_basic_f32() {- test_basic::<f32>();-}-#[uzu_test]fn test_basic_bf16() {test_basic::<bf16>();}-#[uzu_test]-fn test_causal_f32() {- test_causal::<f32>();-}-#[uzu_test]fn test_causal_bf16() {test_causal::<bf16>();}-#[uzu_test]-fn test_gqa_f32() {- test_gqa::<f32>();-}-#[uzu_test]fn test_gqa_bf16() {test_gqa::<bf16>();}-#[uzu_test]-fn test_head_dim_128_f32() {- test_head_dim::<f32>(128);-}-#[uzu_test]fn test_head_dim_128_bf16() {test_head_dim::<bf16>(128);}-#[uzu_test]-fn test_unaligned_f32() {- test_unaligned::<f32>();-}-#[uzu_test]fn test_unaligned_bf16() {test_unaligned::<bf16>();diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rsindex 9598b8344..9740132bd 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs@@ -1,7 +1,7 @@use half::bf16;-use ndarray::{Array4, s};use uzu_engine_macros::uzu_test;+use super::{gemm, single_pass, two_pass};use crate::{array::ArrayElement,backends::{@@ -21,538 +21,6 @@ use crate::{trie::TrieNode,};-fn reference_attention(- queries: &Array4<f32>,- keys: &Array4<f32>,- values: &Array4<f32>,- sinks: Option<&[f32]>,- scale: f32,- is_causal: bool,-) -> Array4<f32> {- let (batch_size, num_heads, sequence_length, head_dim) = queries.dim();- let (_, num_kv_heads, kv_sequence_length, _) = keys.dim();- let repeats_per_kv_head = num_heads / num_kv_heads;- let scaled_queries = queries.mapv(|value| value * scale);- let mut output = Array4::zeros((batch_size, num_heads, sequence_length, head_dim));-- for batch_index in 0..batch_size {- if repeats_per_kv_head > 1 {- for kv_head_index in 0..num_kv_heads {- for repeat_index in 0..repeats_per_kv_head {- let head_index = kv_head_index * repeats_per_kv_head + repeat_index;- assign_attention_head_output(- &mut output,- &scaled_queries,- keys,- values,- sinks,- is_causal,- batch_index,- head_index,- kv_head_index,- sequence_length,- kv_sequence_length,- );- }- }- } else {- for head_index in 0..num_heads {- assign_attention_head_output(- &mut output,- &scaled_queries,- keys,- values,- sinks,- is_causal,- batch_index,- head_index,- head_index,- sequence_length,- kv_sequence_length,- );- }- }- }-- output-}--fn assign_attention_head_output(- output: &mut Array4<f32>,- scaled_queries: &Array4<f32>,- keys: &Array4<f32>,- values: &Array4<f32>,- sinks: Option<&[f32]>,- is_causal: bool,- batch_index: usize,- head_index: usize,- kv_head_index: usize,- sequence_length: usize,- kv_sequence_length: usize,-) {- let queries = scaled_queries.slice(s![batch_index, head_index, .., ..]).to_owned();- let keys = keys.slice(s![batch_index, kv_head_index, .., ..]).to_owned();- let values = values.slice(s![batch_index, kv_head_index, .., ..]).to_owned();- let mut scores = queries.dot(&keys.t());-- if is_causal {- for query_index in 0..sequence_length {- for key_index in (query_index + 1)..kv_sequence_length {- scores[[query_index, key_index]] = f32::NEG_INFINITY;- }- }- }-- for query_index in 0..sequence_length {- let sink_logit = sinks.map(|sink_values| sink_values[head_index]);- let row = scores.row(query_index);- let max_score =- row.iter().fold(sink_logit.unwrap_or(f32::NEG_INFINITY), |current_max, &score| current_max.max(score));-- let mut sum_exp = sink_logit.map(|sink_value| (sink_value - max_score).exp()).unwrap_or(0.0);- for key_index in 0..kv_sequence_length {- scores[[query_index, key_index]] = (scores[[query_index, key_index]] - max_score).exp();- sum_exp += scores[[query_index, key_index]];- }- scores.row_mut(query_index).mapv_inplace(|value| value / sum_exp);- }-- let head_output = scores.dot(&values);- output.slice_mut(s![batch_index, head_index, .., ..]).assign(&head_output);-}--fn create_test_data(- batch_size: usize,- num_heads: usize,- num_kv_heads: usize,- sequence_length: usize,- head_dim: usize,- seed: u64,-) -> (Array4<f32>, Array4<f32>, Array4<f32>) {- use rand::{RngExt, SeedableRng, rngs::StdRng};-- let mut random = StdRng::seed_from_u64(seed);-- let queries =- Array4::from_shape_fn((batch_size, num_heads, sequence_length, head_dim), |_| random.random_range(-0.5..0.5));- let keys = Array4::from_shape_fn((batch_size, num_kv_heads, sequence_length, head_dim), |_| {- random.random_range(-0.5..0.5)- });- let values = Array4::from_shape_fn((batch_size, num_kv_heads, sequence_length, head_dim), |_| {- random.random_range(-0.5..0.5)- });-- (queries, keys, values)-}--fn create_query_buffer(- queries: &Array4<f32>,- context: &<Metal as Backend>::Context,-) -> <Metal as Backend>::GlobalBuffer {- let (_batch_size, num_heads, sequence_length, head_dim) = queries.dim();- let mut values = vec![0.0_f32; num_heads * sequence_length * head_dim];-- for head_index in 0..num_heads {- for sequence_index in 0..sequence_length {- for dim_index in 0..head_dim {- let flat_index = head_index * sequence_length * head_dim + sequence_index * head_dim + dim_index;- values[flat_index] = queries[[0, head_index, sequence_index, dim_index]];- }- }- }-- create_buffer_with_data::<Metal, _>(context, &values)-}--fn create_attention_cache_buffer(- values: &Array4<f32>,- max_sequence_length: usize,- context: &<Metal as Backend>::Context,-) -> <Metal as Backend>::GlobalBuffer {- let (_batch_size, num_kv_heads, sequence_length, head_dim) = values.dim();- let mut cache = vec![0.0_f32; max_sequence_length * num_kv_heads * head_dim];-- for sequence_index in 0..sequence_length {- for head_index in 0..num_kv_heads {- for dim_index in 0..head_dim {- let flat_index = sequence_index * num_kv_heads * head_dim + head_index * head_dim + dim_index;- cache[flat_index] = values[[0, head_index, sequence_index, dim_index]];- }- }- }-- create_buffer_with_data::<Metal, _>(context, &cache)-}--fn create_sinks_buffer(- sinks: &[f32],- context: &<Metal as Backend>::Context,-) -> <Metal as Backend>::GlobalBuffer {- create_buffer_with_data::<Metal, _>(context, sinks)-}--fn convert_kernel_output(- output: &[f32],- batch_size: usize,- num_heads: usize,- sequence_length: usize,- head_dim: usize,-) -> Array4<f32> {- let mut kernel_output = Array4::zeros((batch_size, num_heads, sequence_length, head_dim));-- for head_index in 0..num_heads {- for sequence_index in 0..sequence_length {- for dim_index in 0..head_dim {- let flat_index = (sequence_index * num_heads + head_index) * head_dim + dim_index;- kernel_output[[0, head_index, sequence_index, dim_index]] = output[flat_index];- }- }- }-- kernel_output-}--fn run_single_pass_attention(- kernel: &super::single_pass::AttentionSinglePass,- context: &<Metal as Backend>::Context,- queries: &Array4<f32>,- keys: &Array4<f32>,- values: &Array4<f32>,- sinks: Option<&[f32]>,- _scale: f32,-) -> Result<Array4<f32>, Box<dyn std::error::Error>> {- let (batch_size, num_heads, seq_len, head_dim) = queries.dim();- let (_batch_size, _num_kv_heads, _seq_len, _head_dim) = keys.dim();-- let query_buffer = create_query_buffer(queries, context);- let key_cache_buffer = create_attention_cache_buffer(keys, seq_len, context);- let value_cache_buffer = create_attention_cache_buffer(values, seq_len, context);- let sinks_buffer = sinks.map(|sinks| create_sinks_buffer(sinks, context));- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- let pooled_output = kernel.encode(- AttentionArguments {- queries: &query_buffer,- keys: &key_cache_buffer,- values: &value_cache_buffer,- suffix_length: seq_len as u32,- trie: None::<&<Metal as Backend>::GlobalBuffer>,- sinks: sinks_buffer.as_ref(),- cache: KVCacheView::full(0),- gate: None::<&<Metal as Backend>::GlobalBuffer>,- },- &mut command_buffer,- )?;- let mut output_buffer = create_buffer::<Metal, f32>(context, num_heads * seq_len * head_dim);- command_buffer.encode_copy(&pooled_output, &mut output_buffer);- drop(pooled_output);- submit_command_buffer(command_buffer);-- let output_slice: Vec<f32> = buffer_to_vec(&output_buffer);- let kernel_output = convert_kernel_output(&output_slice, batch_size, num_heads, seq_len, head_dim);-- Ok(kernel_output)-}--fn create_single_pass_kernel(- head_dim: usize,- num_q_heads: usize,- num_groups: usize,- has_sinks: bool,- is_causal: bool,-) -> super::single_pass::AttentionSinglePass {- let mut config = super::default_attention_config::<f32>(head_dim, num_q_heads, num_groups, is_causal);- config.has_sinks = has_sinks;- config.scale = None;- super::single_pass::AttentionSinglePass::new(&config)-}--fn create_two_pass_kernel(- head_dim: usize,- num_q_heads: usize,- num_groups: usize,- is_causal: bool,-) -> super::two_pass::AttentionTwoPass {- let config = super::default_attention_config::<f32>(head_dim, num_q_heads, num_groups, is_causal);- super::two_pass::AttentionTwoPass::new(&config)-}--fn run_gemm_attention(- context: &<Metal as Backend>::Context,- queries: &Array4<f32>,- keys: &Array4<f32>,- values: &Array4<f32>,- sinks: Option<&[f32]>,- scale: f32,- is_causal: bool,-) -> Result<Array4<f32>, Box<dyn std::error::Error>> {- let (batch_size, num_heads, seq_len, head_dim) = queries.dim();- let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();-- let mut config = super::default_attention_config::<f32>(head_dim, num_heads, num_kv_heads, is_causal);- config.has_sinks = sinks.is_some();- config.scale = Some(scale);- let kernel = super::gemm::AttentionGemm::new(&config);-- let query_buffer = create_query_buffer(queries, context);- let key_buffer = create_attention_cache_buffer(keys, seq_len, context);- let value_buffer = create_attention_cache_buffer(values, seq_len, context);-- let sinks_buffer = sinks.map(|sinks| create_sinks_buffer(sinks, context));- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");-- let args = AttentionArguments {- queries: &query_buffer,- keys: &key_buffer,- values: &value_buffer,- suffix_length: seq_len as u32,- trie: None::<&<Metal as Backend>::GlobalBuffer>,- sinks: sinks_buffer.as_ref(),- cache: KVCacheView::full(0),- gate: None::<&<Metal as Backend>::GlobalBuffer>,- };-- let pooled_output = kernel.encode(args, &mut command_buffer)?;- let mut output_buffer = create_buffer::<Metal, f32>(context, num_heads * seq_len * head_dim);- command_buffer.encode_copy(&pooled_output, &mut output_buffer);- let completed = command_buffer.end_encoding().submit().wait_until_completed()?;- drop(pooled_output);- drop(completed);-- let output: Vec<f32> = buffer_to_vec(&output_buffer);-- let kernel_output = convert_kernel_output(&output, batch_size, num_heads, seq_len, head_dim);-- Ok(kernel_output)-}--fn compare_results(- kernel_output: &Array4<f32>,- reference_output: &Array4<f32>,- tolerance: f32,- test_name: &str,-) -> Result<(), String> {- let max_diff =- kernel_output.iter().zip(reference_output.iter()).map(|(lhs, rhs)| (lhs - rhs).abs()).fold(0.0_f32, f32::max);-- if max_diff >= tolerance {- return Err(format!(- "{} output differs from reference by more than {}: max_diff = {}",- test_name, tolerance, max_diff- ));- }-- Ok(())-}--#[uzu_test]-fn test_single_pass_attention_basic() {- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");-- let batch_size = 1;- let num_heads = 4;- let num_kv_heads = 4;- let seq_len = 8;- let head_dim = 64;- let scale = 1.0 / (head_dim as f32).sqrt();-- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);- let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);- let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, false, false);- let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)- .expect("single-pass attention");-- compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention").unwrap();-}--#[uzu_test]-fn test_matrix_attention_matches_vector_and_cpu_seq256() {- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");-- let batch_size = 1;- let num_heads = 8;- let num_kv_heads = 2;- let seq_len = 256;- let head_dim = 64;- let scale = 1.0 / (head_dim as f32).sqrt();- let is_causal = true;-- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);-- let reference_output = reference_attention(&queries, &keys, &values, None, scale, is_causal);- let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, false, is_causal);- let vector_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)- .expect("run vector attention");- let matrix_output =- run_gemm_attention(&context, &queries, &keys, &values, None, scale, is_causal).expect("run matrix attention");- let tol_cpu = 5e-2;- compare_results(&vector_output, &reference_output, tol_cpu, "vector single-pass attention vs CPU").unwrap();- compare_results(&matrix_output, &reference_output, tol_cpu, "matrix attention vs CPU").unwrap();-- let max_diff_vector_matrix =- vector_output.iter().zip(matrix_output.iter()).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max);- let tol_vector_matrix = 5e-2;- assert!(- max_diff_vector_matrix <= tol_vector_matrix,- "vector single-pass attention and matrix attention differ too much: max_diff = {} (tol={})",- max_diff_vector_matrix,- tol_vector_matrix- );-}--#[uzu_test]-fn test_single_pass_attention_with_sinks() {- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");-- let batch_size = 1;- let num_heads = 4;- let num_kv_heads = 4;- let seq_len = 8;- let head_dim = 64;- let scale = 1.0 / (head_dim as f32).sqrt();-- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 444);-- let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();- let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, true, false);-- let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);- let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale)- .expect("single-pass attention with sinks");-- compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention with sinks").unwrap();-}--#[uzu_test]-fn test_single_pass_attention_with_sinks_long_sequence() {- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");-- let batch_size = 1;- let num_heads = 4;- let num_kv_heads = 4;- let seq_len = 64;- let head_dim = 64;- let scale = 1.0 / (head_dim as f32).sqrt();-- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 777);-- let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();- let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, true, false);-- let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);- let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale)- .expect("single-pass attention with sinks (long sequence)");-- compare_results(&kernel_output, &reference_output, 5e-2, "Single-pass attention with sinks (long sequence)")- .unwrap();-}--#[uzu_test]-fn test_single_pass_attention_gqa() {- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");-- let batch_size = 1;- let num_heads = 8;- let num_kv_heads = 2;- let seq_len = 8;- let head_dim = 64;- let scale = 1.0 / (head_dim as f32).sqrt();-- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);-- let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, false, false);- let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)- .expect("single-pass attention GQA");-- let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);- compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention GQA").unwrap();-}--fn run_two_pass_attention(- kernel: &super::two_pass::AttentionTwoPass,- context: &<Metal as Backend>::Context,- queries: &Array4<f32>,- keys: &Array4<f32>,- values: &Array4<f32>,- sinks: Option<&[f32]>,- _scale: f32,-) -> Result<Array4<f32>, Box<dyn std::error::Error>> {- let (batch_size, num_heads, seq_len, head_dim) = queries.dim();- let queries_buffer = create_query_buffer(queries, context);- let keys_buffer = create_attention_cache_buffer(keys, seq_len, context);- let values_buffer = create_attention_cache_buffer(values, seq_len, context);- let sinks_buffer = sinks.map(|sinks| create_sinks_buffer(sinks, context));- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");-- let pooled_output = kernel.encode(- AttentionArguments {- queries: &queries_buffer,- keys: &keys_buffer,- values: &values_buffer,- suffix_length: seq_len as u32,- trie: None::<&<Metal as Backend>::GlobalBuffer>,- sinks: sinks_buffer.as_ref(),- cache: KVCacheView::full(0),- gate: None::<&<Metal as Backend>::GlobalBuffer>,- },- &mut command_buffer,- )?;- let mut output_buffer = create_buffer::<Metal, f32>(context, num_heads * seq_len * head_dim);- command_buffer.encode_copy(&pooled_output, &mut output_buffer);- drop(pooled_output);- submit_command_buffer(command_buffer);-- let output_slice: Vec<f32> = buffer_to_vec(&output_buffer);- let kernel_output = convert_kernel_output(&output_slice, batch_size, num_heads, seq_len, head_dim);-- Ok(kernel_output)-}--#[uzu_test]-fn test_two_pass_attention() {- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");-- let batch_size = 1;- let num_heads = 8;- let num_kv_heads: usize = 8;- let seq_len = 2048;- let head_dim = 64;- let scale = 1.0 / (head_dim as f32).sqrt();- let is_causal = false; // Non-causal attention for this test-- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);-- let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);-- let kernel = create_two_pass_kernel(head_dim, num_heads, num_kv_heads, is_causal);- let kernel_output =- run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale).expect("two-pass attention");-- compare_results(&kernel_output, &reference_output, 1e-2, "Two-pass attention").unwrap();-}--#[uzu_test]-fn test_two_pass_attention_gqa() {- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");-- let batch_size = 1;- let num_heads = 8;- let num_kv_heads = 2;- let seq_len = 4096;- let head_dim = 64;- let scale = 1.0 / (head_dim as f32).sqrt();- let is_causal = false; // Non-causal attention for this test-- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);-- let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);-- let kernel = create_two_pass_kernel(head_dim, num_heads, num_kv_heads, is_causal);- let kernel_output = run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)- .expect("two-pass attention GQA");-- compare_results(&kernel_output, &reference_output, 1e-2, "Two-pass attention GQA").unwrap();-}-type AttentionShape = (usize, usize, usize, usize, usize, bool);fn attention_data(shape: AttentionShape) -> (Vec<bf16>, Vec<bf16>) {@@ -573,11 +41,13 @@ fn run_attention<B: Backend>(context: &B::Context,shape: AttentionShape,trie_nodes: Option<&[GpuTrieNode]>,+ sinks_data: Option<&[bf16]>,) -> Vec<bf16> {let (keys, values) = attention_data(shape);let (_, _, _, _, prefix_length, _) = shape;let (head_dim, num_q_heads, num_groups, _, _, is_causal) = shape;- let config = super::default_attention_config::<bf16>(head_dim, num_q_heads, num_groups, is_causal);+ let mut config = super::default_attention_config::<bf16>(head_dim, num_q_heads, num_groups, is_causal);+ config.has_sinks = sinks_data.is_some();let kernel = <B::Kernels as Kernels>::AttentionKernel::new(context, config).expect("attention kernel");run_attention_with_kernel::<B>(context,@@ -587,6 +57,7 @@ fn run_attention<B: Backend>(&values,KVCacheView::full(prefix_length as u32),trie_nodes,+ sinks_data,)}@@ -598,12 +69,14 @@ fn run_attention_with_kernel<B: Backend>(values_data: &[bf16],cache: KVCacheView,trie_nodes: Option<&[GpuTrieNode]>,+ sinks_data: Option<&[bf16]>,) -> Vec<bf16> {let (head_dim, num_q_heads, _, suffix_length, _, _) = shape;let queries =create_buffer_with_data::<B, bf16>(context, &fill_attention(num_q_heads * suffix_length * head_dim, 0.5));let keys = create_buffer_with_data::<B, bf16>(context, keys_data);let values = create_buffer_with_data::<B, bf16>(context, values_data);+ let sinks = sinks_data.map(|data| create_buffer_with_data::<B, bf16>(context, data));let trie = trie_nodes.map(|nodes| {let words: Vec<u32> = nodes.iter().flat_map(|node| [node.trie_start, node.trie_end, node.height]).collect();create_buffer_with_data::<B, u32>(context, &words)@@ -614,7 +87,7 @@ fn run_attention_with_kernel<B: Backend>(values: &values,suffix_length: suffix_length as u32,trie: trie.as_ref(),- sinks: None,+ sinks: sinks.as_ref(),cache,gate: None::<&B::GlobalBuffer>,};@@ -628,6 +101,72 @@ fn run_attention_with_kernel<B: Backend>(buffer_to_vec::<B, bf16>(&output)}+enum DirectKernel {+ SinglePass,+ TwoPass,+ Gemm,+}++fn run_direct_metal(+ context: &<Metal as Backend>::Context,+ shape: AttentionShape,+ sinks_data: Option<&[bf16]>,+ route: DirectKernel,+) -> Vec<bf16> {+ let (head_dim, num_q_heads, num_groups, suffix_length, prefix_length, is_causal) = shape;+ let mut config = super::default_attention_config::<bf16>(head_dim, num_q_heads, num_groups, is_causal);+ config.has_sinks = sinks_data.is_some();+ let (keys_data, values_data) = attention_data(shape);+ let queries =+ create_buffer_with_data::<Metal, bf16>(context, &fill_attention(num_q_heads * suffix_length * head_dim, 0.5));+ let keys = create_buffer_with_data::<Metal, bf16>(context, &keys_data);+ let values = create_buffer_with_data::<Metal, bf16>(context, &values_data);+ let sinks = sinks_data.map(|data| create_buffer_with_data::<Metal, bf16>(context, data));+ let arguments = AttentionArguments {+ queries: &queries,+ keys: &keys,+ values: &values,+ suffix_length: suffix_length as u32,+ trie: None::<&<Metal as Backend>::GlobalBuffer>,+ sinks: sinks.as_ref(),+ cache: KVCacheView::full(prefix_length as u32),+ gate: None::<&<Metal as Backend>::GlobalBuffer>,+ };+ let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");+ let pooled = match route {+ DirectKernel::SinglePass => {+ single_pass::AttentionSinglePass::new(&config).encode(arguments, &mut command_buffer)+ },+ DirectKernel::TwoPass => two_pass::AttentionTwoPass::new(&config).encode(arguments, &mut command_buffer),+ DirectKernel::Gemm => gemm::AttentionGemm::new(&config).encode(arguments, &mut command_buffer),+ }+ .expect("direct attention");+ let mut output = create_buffer::<Metal, bf16>(context, suffix_length * num_q_heads * head_dim);+ command_buffer.encode_copy(&pooled, &mut output);+ submit_command_buffer(command_buffer);+ buffer_to_vec::<Metal, bf16>(&output)+}++#[uzu_test]+fn direct_bf16_attention_matches_cpu() {+ let cpu = <Cpu as Backend>::Context::new().expect("CPU context");+ let metal = <Metal as Backend>::Context::new().expect("Metal context");+ for (shape, route, has_sinks) in [+ ((64, 4, 4, 8, 0, false), DirectKernel::SinglePass, false),+ ((64, 4, 4, 64, 0, false), DirectKernel::SinglePass, true),+ ((64, 8, 2, 8, 0, false), DirectKernel::SinglePass, false),+ ((64, 8, 2, 256, 0, true), DirectKernel::SinglePass, false),+ ((64, 8, 2, 256, 0, true), DirectKernel::Gemm, false),+ ((64, 8, 8, 128, 0, false), DirectKernel::TwoPass, false),+ ((64, 8, 2, 64, 1024, false), DirectKernel::TwoPass, false),+ ] {+ let sinks = has_sinks.then(|| vec![bf16::from_f32(1.0); shape.1]);+ let expected = run_attention::<Cpu>(cpu.as_ref(), shape, None, sinks.as_deref());+ let actual = run_direct_metal(metal.as_ref(), shape, sinks.as_deref(), route);+ assert_eq_float::<bf16>(&expected, &actual, 1e-2, "direct BF16 attention");+ }+}+#[uzu_test]fn attention_kernel_matches_cpu() {const TRIE_SUFFIX_LENGTH: usize = 31;@@ -648,11 +187,18 @@ fn attention_kernel_matches_cpu() {] {let nodes = use_trie.then_some(trie.as_slice());let shape = (head_dim, num_q_heads, num_groups, suffix_length, prefix_length, causal);- let expected = run_attention::<Cpu>(cpu_context.as_ref(), shape, nodes);- let actual = run_attention::<Metal>(metal_context.as_ref(), shape, nodes);+ let expected = run_attention::<Cpu>(cpu_context.as_ref(), shape, nodes, None);+ let actual = run_attention::<Metal>(metal_context.as_ref(), shape, nodes, None);let label = format!("attention kernel D{head_dim} S{suffix_length} L{prefix_length} causal={causal}");assert_eq_float::<bf16>(&expected, &actual, 1e-2, &label);}+ let sinks = vec![bf16::from_f32(1.0); 8];+ for prefix_length in [32, 2048] {+ let shape = (512, 8, 2, 1, prefix_length, false);+ let expected = run_attention::<Cpu>(cpu_context.as_ref(), shape, None, Some(&sinks));+ let actual = run_attention::<Metal>(metal_context.as_ref(), shape, None, Some(&sinks));+ assert_eq_float::<bf16>(&expected, &actual, 1e-2, "BF16 GQA with sinks");+ }}#[uzu_test]@@ -668,9 +214,9 @@ fn attention_kernel_reuses_instance_for_flat_and_trie() {.expect("attention kernel");let (keys, values) = attention_data(shape);let cache = KVCacheView::full(1024);- let flat = run_attention_with_kernel::<Metal>(context.as_ref(), &kernel, shape, &keys, &values, cache, None);+ let flat = run_attention_with_kernel::<Metal>(context.as_ref(), &kernel, shape, &keys, &values, cache, None, None);let trie_output =- run_attention_with_kernel::<Metal>(context.as_ref(), &kernel, shape, &keys, &values, cache, Some(&trie));+ run_attention_with_kernel::<Metal>(context.as_ref(), &kernel, shape, &keys, &values, cache, Some(&trie), None);let cpu_context = <Cpu as Backend>::Context::new().expect("CPU attention context");let cpu_kernel = <<Cpu as Backend>::Kernels as Kernels>::AttentionKernel::new(cpu_context.as_ref(),@@ -678,9 +224,17 @@ fn attention_kernel_reuses_instance_for_flat_and_trie() {).expect("CPU attention kernel");let expected_flat =- run_attention_with_kernel::<Cpu>(cpu_context.as_ref(), &cpu_kernel, shape, &keys, &values, cache, None);- let expected_trie =- run_attention_with_kernel::<Cpu>(cpu_context.as_ref(), &cpu_kernel, shape, &keys, &values, cache, Some(&trie));+ run_attention_with_kernel::<Cpu>(cpu_context.as_ref(), &cpu_kernel, shape, &keys, &values, cache, None, None);+ let expected_trie = run_attention_with_kernel::<Cpu>(+ cpu_context.as_ref(),+ &cpu_kernel,+ shape,+ &keys,+ &values,+ cache,+ Some(&trie),+ None,+ );assert_eq_float::<bf16>(&expected_flat, &flat, 1e-2, "flat mask specialization");assert_eq_float::<bf16>(&expected_trie, &trie_output, 1e-2, "trie mask specialization");}@@ -737,6 +291,7 @@ fn attention_kernel_ring_matches_full_on_wrap() {&full_values,KVCacheView::full(CONTEXT as u32),None,+ None,);let ring = run_attention_with_kernel::<Metal>(context.as_ref(),@@ -746,6 +301,7 @@ fn attention_kernel_ring_matches_full_on_wrap() {&ring_values,KVCacheView::ring(CAPACITY as u32, ring_offset as u32),None,+ None,);assert_eq_float::<bf16>(&full, &ring, 1e-2, "wrapped ring attention");}