Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Remove f32 attention (#946)

CCYeh432da1b7bf42
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal
index dbb45dfe0..4af0b3324 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_fallback.metal
@@ -11,7 +11,7 @@ using namespace uzu::ring;
using namespace uzu::trie;
template <typename T>
-VARIANTS(T, float, bfloat)
+VARIANTS(T, bfloat)
KERNEL(AttentionFallbackScatterScores)(
const device T* group_scores,
device T* scores,
@@ -59,7 +59,7 @@ KERNEL(AttentionFallbackScatterScores)(
}
template <typename T>
-VARIANTS(T, float, bfloat)
+VARIANTS(T, bfloat)
KERNEL(AttentionFallbackScatterValues)(
const device T* group_output,
device T* out,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal
index b31d7b054..0cdfb142d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -37,12 +37,11 @@ struct AttentionGemmLayout {
};
template <typename T, uint BK, uint BD, bool USE_MXU>
-VARIANTS(T, float, bfloat)
+VARIANTS(T, bfloat)
VARIANTS(BK, 16, 32)
VARIANTS(BD, 64, 128, 256)
VARIANTS(USE_MXU, false, true)
CONSTRAINT(!USE_MXU || BK == 32)
-CONSTRAINT(!USE_MXU || T != "float")
CONSTRAINT(!USE_MXU || BD != 256)
KERNEL(AttentionGemm)(
const device T* q,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal
index 42869cc1d..75651d57e 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -13,7 +13,7 @@ using namespace uzu::ring;
using namespace uzu::trie;
template <typename T, uint HEAD_DIM>
-VARIANTS(T, float, bfloat)
+VARIANTS(T, bfloat)
VARIANTS(HEAD_DIM, 64, 128, 256, 512)
PUBLIC KERNEL(AttentionSinglePass)(
const device T* queries,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal
index a34d453ef..bf76100bf 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -19,7 +19,7 @@ using namespace uzu::trie;
// Each simdgroup independently handles one query head and derives its KV head
// via integer division. Simdgroups sharing a KV head benefit from L1 cache.
template <typename T, uint HEAD_DIM>
-VARIANTS(T, float, bfloat)
+VARIANTS(T, bfloat)
VARIANTS(HEAD_DIM, 64, 128, 256, 512)
KERNEL(AttentionTwoPass1)(
const device T* queries,
@@ -140,7 +140,7 @@ KERNEL(AttentionTwoPass1)(
}
template <typename T, uint HEAD_DIM>
-VARIANTS(T, float, bfloat)
+VARIANTS(T, bfloat)
VARIANTS(HEAD_DIM, 64, 128, 256, 512)
KERNEL(AttentionTwoPass2)(
const device float* partials,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
index b8a0b01c9..114940f4a 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
@@ -39,7 +39,7 @@ pub struct AttentionFallback {
impl AttentionFallback {
pub fn is_supported(config: &AttentionKernelConfig) -> bool {
- config.head_dim == HEAD_DIM && matches!(config.data_type, DataType::BF16 | DataType::F32)
+ config.head_dim == HEAD_DIM && config.data_type == DataType::BF16
}
pub fn new(
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs
index 957e25657..6c4ea891f 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm.rs
@@ -55,7 +55,7 @@ fn retile_params(
impl AttentionGemm {
pub fn is_supported(config: &AttentionKernelConfig) -> bool {
- matches!(config.head_dim, 64 | 128 | 256) && matches!(config.data_type, DataType::BF16 | DataType::F32)
+ matches!(config.head_dim, 64 | 128 | 256) && config.data_type == DataType::BF16
}
pub fn new(config: &AttentionKernelConfig) -> Self {
@@ -127,10 +127,8 @@ impl AttentionGemm {
let mut output = command_buffer
.allocate_scratch_for_shape(&[arguments.suffix_length, self.num_q_heads, self.head_dim], self.data_type)?;
- let use_mxu = arguments.suffix_length >= 64
- && matches!(self.data_type, DataType::BF16)
- && matches!(self.head_dim, 64 | 128)
- && command_buffer.context().supports_mxu;
+ let use_mxu =
+ arguments.suffix_length >= 64 && matches!(self.head_dim, 64 | 128) && command_buffer.context().supports_mxu;
let (bq, bk) = if use_mxu {
(64, 32)
} else {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal
index b75c7709b..9fdfab026 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/kv_cache_update.metal
@@ -4,7 +4,7 @@
#include "kv_cache_update.h"
template <typename T>
-VARIANTS(T, float, bfloat)
+VARIANTS(T, bfloat)
PUBLIC KERNEL(KVCacheUpdate) (
device T* in_place_keys,
device T* in_place_values,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal
index bb66922da..e4ef85676 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/sigmoid_gate.metal
@@ -3,7 +3,7 @@
#include "attention_gate.h"
template <typename T>
-VARIANTS(T, float, bfloat)
+VARIANTS(T, bfloat)
PUBLIC KERNEL(SigmoidGate)(
const device T* gate,
device T* output,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
index 9b722d016..845c24522 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/kv_cache_update_test.rs
@@ -1,8 +1,6 @@
use std::fmt::{Debug, Display};
use half::bf16;
-#[cfg(backend = "metal")]
-use ndarray::{Array, Array3};
use num_traits::Float;
use uzu_engine_macros::uzu_test;
@@ -20,8 +18,6 @@ use crate::{
helpers::{buffer_to_vec, create_buffer_with_data, for_each_non_cpu_backend},
},
};
-#[cfg(backend = "metal")]
-use crate::{backends::metal::Metal, data_type::DataType};
struct Input<T: ArrayElement + Float> {
keys: Box<[T]>,
@@ -109,6 +105,10 @@ fn get_test_data_multi_copy<T: ArrayElement + Float>() -> (Input<T>, Vec<T>, Vec
source: 0,
destination: 5,
},
+ Copy {
+ source: 3,
+ destination: 6,
+ },
];
let input = Input {
@@ -216,163 +216,25 @@ fn test_large_internal<T: ArrayElement + Float + Debug + Display>() {
}
// Single copy tests
-#[uzu_test]
-fn test_single_copy_f32() {
- test_single_copy_internal::<f32>();
-}
-
#[uzu_test]
fn test_single_copy_bf16() {
test_single_copy_internal::<bf16>();
}
// Multi copy tests
-#[uzu_test]
-fn test_multi_copy_f32() {
- test_multi_copy_internal::<f32>();
-}
-
#[uzu_test]
fn test_multi_copy_bf16() {
test_multi_copy_internal::<bf16>();
}
// No copy tests
-#[uzu_test]
-fn test_no_copy_f32() {
- test_no_copy_internal::<f32>();
-}
-
#[uzu_test]
fn test_no_copy_bf16() {
test_no_copy_internal::<bf16>();
}
// Large dimension tests
-#[uzu_test]
-fn test_large_f32() {
- test_large_internal::<f32>();
-}
-
#[uzu_test]
fn test_large_bf16() {
test_large_internal::<bf16>();
}
-
-#[cfg(backend = "metal")]
-fn apply_copies_3d<T: Clone>(
- array: &mut Array3<T>,
- copies: &[Copy],
-) {
- let (_seq_len, num_heads, head_dim) = array.dim();
- for head in 0..num_heads {
- for channel in 0..head_dim {
- for copy in copies {
- let src = copy.source as usize;
- let dst = copy.destination as usize;
- array[(dst, head, channel)] = array[(src, head, channel)].clone();
- }
- }
- }
-}
-
-#[cfg(backend = "metal")]
-#[uzu_test]
-fn test_random_pattern_f32() {
- let context = match <Metal as Backend>::Context::new() {
- Ok(context) => context,
- Err(error) => {
- eprintln!("Failed to create Metal context: {error:?}. Skipping test.");
- return;
- },
- };
-
- let kernel = <<Metal as Backend>::Kernels as Kernels>::KVCacheUpdateKernel::new(&context, DataType::F32)
- .expect("Failed to create KVCacheUpdateKernel");
-
- let num_heads = 3usize;
- let seq_len = 15usize;
- let head_dim = 7usize;
- let element_dim = num_heads * head_dim;
-
- let key_data = Array3::<f32>::from_shape_fn((seq_len, num_heads, head_dim), |(token, head, channel)| {
- (token * 1_000_000 + head * 100 + channel * 10) as f32
- });
-
- let value_data = Array3::<f32>::from_shape_fn((seq_len, num_heads, head_dim), |(token, head, channel)| {
- (token * 1_000_000 + head * 100 + channel * 10 + 1_000) as f32
- });
-
- let copies = vec![
- Copy {
- source: 0,
- destination: 14,
- },
- Copy {
- source: 3,
- destination: 11,
- },
- Copy {
- source: 6,
- destination: 8,
- },
- Copy {
- source: 9,
- destination: 5,
- },
- Copy {
- source: 12,
- destination: 2,
- },
- Copy {
- source: 2,
- destination: 12,
- },
- Copy {
- source: 5,
- destination: 9,
- },
- Copy {
- source: 8,
- destination: 6,
- },
- Copy {
- source: 11,
- destination: 3,
- },
- Copy {
- source: 14,
- destination: 0,
- },
- ];
-
- let mut expected_keys = key_data.clone();
- let mut expected_values = value_data.clone();
- apply_copies_3d(&mut expected_keys, &copies);
- apply_copies_3d(&mut expected_values, &copies);
-
- let mut key_buffer = create_buffer_with_data::<Metal, f32>(&context, key_data.as_slice().unwrap());
- let mut value_buffer = create_buffer_with_data::<Metal, f32>(&context, value_data.as_slice().unwrap());
-
- let mut command_buffer = context.create_command_buffer(None, None).unwrap();
- kernel.encode(
- &mut key_buffer,
- &mut value_buffer,
- &copies,
- copies.len() as u32,
- element_dim as u32,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let key_values: Vec<f32> = buffer_to_vec(&key_buffer);
- let key_result = Array::from_shape_vec((seq_len, num_heads, head_dim), key_values)
- .expect("Failed to convert key result to ndarray");
-
- let value_values: Vec<f32> = buffer_to_vec(&value_buffer);
- let value_result = Array::from_shape_vec((seq_len, num_heads, head_dim), value_values)
- .expect("Failed to convert value result to ndarray");
-
- assert_eq!(key_result, expected_keys);
- assert_eq!(value_result, expected_values);
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs b/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs
index 0a8b121e1..fd13f9aff 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/attention/sigmoid_gate_test.rs
@@ -84,11 +84,7 @@ fn run_test<T: ArrayElement + Float + Debug>(config: &Config) {
})
.collect::<Vec<_>>();
- let rtol = if std::mem::size_of::<T>() <= 2 {
- 0.01
- } else {
- 1e-5
- };
+ let rtol = 0.01;
let assert_output = |result: &[T], backend: &str| {
for (i, (got, exp)) in result.iter().zip(expected.iter()).enumerate() {
@@ -118,39 +114,23 @@ fn run_test<T: ArrayElement + Float + Debug>(config: &Config) {
}
#[uzu_test]
-fn test_sigmoid_gate_f32() {
- run_test::<f32>(&Config {
+fn test_sigmoid_gate_bf16() {
+ run_test::<bf16>(&Config {
num_heads: 8,
num_kv_heads: 2,
head_dim: 64,
suffix_length: 4,
});
- run_test::<f32>(&Config {
+ run_test::<bf16>(&Config {
num_heads: 16,
num_kv_heads: 4,
head_dim: 256,
suffix_length: 1,
});
- run_test::<f32>(&Config {
+ run_test::<bf16>(&Config {
num_heads: 2,
num_kv_heads: 1,
head_dim: 64,
suffix_length: 8,
});
}
-
-#[uzu_test]
-fn test_sigmoid_gate_bf16() {
- run_test::<bf16>(&Config {
- num_heads: 8,
- num_kv_heads: 2,
- head_dim: 64,
- suffix_length: 4,
- });
- run_test::<bf16>(&Config {
- num_heads: 16,
- num_kv_heads: 4,
- head_dim: 256,
- suffix_length: 1,
- });
-}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs
index 48a16d455..a5f6b753c 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_test.rs
@@ -14,7 +14,6 @@ use crate::{
cpu::Cpu,
metal::Metal,
},
- data_type::DataType,
encodable_block::mixer::attention::KVCacheView,
tests::{
assert::assert_eq_float,
@@ -151,11 +150,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
input: &Input<T>,
expected: &[T],
) {
- let eps = if T::data_type() == DataType::BF16 {
- 1e-2
- } else {
- 1e-5
- };
+ let eps = 1e-2;
let output = get_gemm_output(input);
let msg = format!(
@@ -207,51 +202,26 @@ fn test_unaligned<T: ArrayElement + Float + Debug + Display>() {
test_internal(&input, &expected);
}
-#[uzu_test]
-fn test_basic_f32() {
- test_basic::<f32>();
-}
-
#[uzu_test]
fn test_basic_bf16() {
test_basic::<bf16>();
}
-#[uzu_test]
-fn test_causal_f32() {
- test_causal::<f32>();
-}
-
#[uzu_test]
fn test_causal_bf16() {
test_causal::<bf16>();
}
-#[uzu_test]
-fn test_gqa_f32() {
- test_gqa::<f32>();
-}
-
#[uzu_test]
fn test_gqa_bf16() {
test_gqa::<bf16>();
}
-#[uzu_test]
-fn test_head_dim_128_f32() {
- test_head_dim::<f32>(128);
-}
-
#[uzu_test]
fn test_head_dim_128_bf16() {
test_head_dim::<bf16>(128);
}
-#[uzu_test]
-fn test_unaligned_f32() {
- test_unaligned::<f32>();
-}
-
#[uzu_test]
fn test_unaligned_bf16() {
test_unaligned::<bf16>();
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
index 9598b8344..9740132bd 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
@@ -1,7 +1,7 @@
use half::bf16;
-use ndarray::{Array4, s};
use uzu_engine_macros::uzu_test;
+use super::{gemm, single_pass, two_pass};
use crate::{
array::ArrayElement,
backends::{
@@ -21,538 +21,6 @@ use crate::{
trie::TrieNode,
};
-fn reference_attention(
- queries: &Array4<f32>,
- keys: &Array4<f32>,
- values: &Array4<f32>,
- sinks: Option<&[f32]>,
- scale: f32,
- is_causal: bool,
-) -> Array4<f32> {
- let (batch_size, num_heads, sequence_length, head_dim) = queries.dim();
- let (_, num_kv_heads, kv_sequence_length, _) = keys.dim();
- let repeats_per_kv_head = num_heads / num_kv_heads;
- let scaled_queries = queries.mapv(|value| value * scale);
- let mut output = Array4::zeros((batch_size, num_heads, sequence_length, head_dim));
-
- for batch_index in 0..batch_size {
- if repeats_per_kv_head > 1 {
- for kv_head_index in 0..num_kv_heads {
- for repeat_index in 0..repeats_per_kv_head {
- let head_index = kv_head_index * repeats_per_kv_head + repeat_index;
- assign_attention_head_output(
- &mut output,
- &scaled_queries,
- keys,
- values,
- sinks,
- is_causal,
- batch_index,
- head_index,
- kv_head_index,
- sequence_length,
- kv_sequence_length,
- );
- }
- }
- } else {
- for head_index in 0..num_heads {
- assign_attention_head_output(
- &mut output,
- &scaled_queries,
- keys,
- values,
- sinks,
- is_causal,
- batch_index,
- head_index,
- head_index,
- sequence_length,
- kv_sequence_length,
- );
- }
- }
- }
-
- output
-}
-
-fn assign_attention_head_output(
- output: &mut Array4<f32>,
- scaled_queries: &Array4<f32>,
- keys: &Array4<f32>,
- values: &Array4<f32>,
- sinks: Option<&[f32]>,
- is_causal: bool,
- batch_index: usize,
- head_index: usize,
- kv_head_index: usize,
- sequence_length: usize,
- kv_sequence_length: usize,
-) {
- let queries = scaled_queries.slice(s![batch_index, head_index, .., ..]).to_owned();
- let keys = keys.slice(s![batch_index, kv_head_index, .., ..]).to_owned();
- let values = values.slice(s![batch_index, kv_head_index, .., ..]).to_owned();
- let mut scores = queries.dot(&keys.t());
-
- if is_causal {
- for query_index in 0..sequence_length {
- for key_index in (query_index + 1)..kv_sequence_length {
- scores[[query_index, key_index]] = f32::NEG_INFINITY;
- }
- }
- }
-
- for query_index in 0..sequence_length {
- let sink_logit = sinks.map(|sink_values| sink_values[head_index]);
- let row = scores.row(query_index);
- let max_score =
- row.iter().fold(sink_logit.unwrap_or(f32::NEG_INFINITY), |current_max, &score| current_max.max(score));
-
- let mut sum_exp = sink_logit.map(|sink_value| (sink_value - max_score).exp()).unwrap_or(0.0);
- for key_index in 0..kv_sequence_length {
- scores[[query_index, key_index]] = (scores[[query_index, key_index]] - max_score).exp();
- sum_exp += scores[[query_index, key_index]];
- }
- scores.row_mut(query_index).mapv_inplace(|value| value / sum_exp);
- }
-
- let head_output = scores.dot(&values);
- output.slice_mut(s![batch_index, head_index, .., ..]).assign(&head_output);
-}
-
-fn create_test_data(
- batch_size: usize,
- num_heads: usize,
- num_kv_heads: usize,
- sequence_length: usize,
- head_dim: usize,
- seed: u64,
-) -> (Array4<f32>, Array4<f32>, Array4<f32>) {
- use rand::{RngExt, SeedableRng, rngs::StdRng};
-
- let mut random = StdRng::seed_from_u64(seed);
-
- let queries =
- Array4::from_shape_fn((batch_size, num_heads, sequence_length, head_dim), |_| random.random_range(-0.5..0.5));
- let keys = Array4::from_shape_fn((batch_size, num_kv_heads, sequence_length, head_dim), |_| {
- random.random_range(-0.5..0.5)
- });
- let values = Array4::from_shape_fn((batch_size, num_kv_heads, sequence_length, head_dim), |_| {
- random.random_range(-0.5..0.5)
- });
-
- (queries, keys, values)
-}
-
-fn create_query_buffer(
- queries: &Array4<f32>,
- context: &<Metal as Backend>::Context,
-) -> <Metal as Backend>::GlobalBuffer {
- let (_batch_size, num_heads, sequence_length, head_dim) = queries.dim();
- let mut values = vec![0.0_f32; num_heads * sequence_length * head_dim];
-
- for head_index in 0..num_heads {
- for sequence_index in 0..sequence_length {
- for dim_index in 0..head_dim {
- let flat_index = head_index * sequence_length * head_dim + sequence_index * head_dim + dim_index;
- values[flat_index] = queries[[0, head_index, sequence_index, dim_index]];
- }
- }
- }
-
- create_buffer_with_data::<Metal, _>(context, &values)
-}
-
-fn create_attention_cache_buffer(
- values: &Array4<f32>,
- max_sequence_length: usize,
- context: &<Metal as Backend>::Context,
-) -> <Metal as Backend>::GlobalBuffer {
- let (_batch_size, num_kv_heads, sequence_length, head_dim) = values.dim();
- let mut cache = vec![0.0_f32; max_sequence_length * num_kv_heads * head_dim];
-
- for sequence_index in 0..sequence_length {
- for head_index in 0..num_kv_heads {
- for dim_index in 0..head_dim {
- let flat_index = sequence_index * num_kv_heads * head_dim + head_index * head_dim + dim_index;
- cache[flat_index] = values[[0, head_index, sequence_index, dim_index]];
- }
- }
- }
-
- create_buffer_with_data::<Metal, _>(context, &cache)
-}
-
-fn create_sinks_buffer(
- sinks: &[f32],
- context: &<Metal as Backend>::Context,
-) -> <Metal as Backend>::GlobalBuffer {
- create_buffer_with_data::<Metal, _>(context, sinks)
-}
-
-fn convert_kernel_output(
- output: &[f32],
- batch_size: usize,
- num_heads: usize,
- sequence_length: usize,
- head_dim: usize,
-) -> Array4<f32> {
- let mut kernel_output = Array4::zeros((batch_size, num_heads, sequence_length, head_dim));
-
- for head_index in 0..num_heads {
- for sequence_index in 0..sequence_length {
- for dim_index in 0..head_dim {
- let flat_index = (sequence_index * num_heads + head_index) * head_dim + dim_index;
- kernel_output[[0, head_index, sequence_index, dim_index]] = output[flat_index];
- }
- }
- }
-
- kernel_output
-}
-
-fn run_single_pass_attention(
- kernel: &super::single_pass::AttentionSinglePass,
- context: &<Metal as Backend>::Context,
- queries: &Array4<f32>,
- keys: &Array4<f32>,
- values: &Array4<f32>,
- sinks: Option<&[f32]>,
- _scale: f32,
-) -> Result<Array4<f32>, Box<dyn std::error::Error>> {
- let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
- let (_batch_size, _num_kv_heads, _seq_len, _head_dim) = keys.dim();
-
- let query_buffer = create_query_buffer(queries, context);
- let key_cache_buffer = create_attention_cache_buffer(keys, seq_len, context);
- let value_cache_buffer = create_attention_cache_buffer(values, seq_len, context);
- let sinks_buffer = sinks.map(|sinks| create_sinks_buffer(sinks, context));
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- let pooled_output = kernel.encode(
- AttentionArguments {
- queries: &query_buffer,
- keys: &key_cache_buffer,
- values: &value_cache_buffer,
- suffix_length: seq_len as u32,
- trie: None::<&<Metal as Backend>::GlobalBuffer>,
- sinks: sinks_buffer.as_ref(),
- cache: KVCacheView::full(0),
- gate: None::<&<Metal as Backend>::GlobalBuffer>,
- },
- &mut command_buffer,
- )?;
- let mut output_buffer = create_buffer::<Metal, f32>(context, num_heads * seq_len * head_dim);
- command_buffer.encode_copy(&pooled_output, &mut output_buffer);
- drop(pooled_output);
- submit_command_buffer(command_buffer);
-
- let output_slice: Vec<f32> = buffer_to_vec(&output_buffer);
- let kernel_output = convert_kernel_output(&output_slice, batch_size, num_heads, seq_len, head_dim);
-
- Ok(kernel_output)
-}
-
-fn create_single_pass_kernel(
- head_dim: usize,
- num_q_heads: usize,
- num_groups: usize,
- has_sinks: bool,
- is_causal: bool,
-) -> super::single_pass::AttentionSinglePass {
- let mut config = super::default_attention_config::<f32>(head_dim, num_q_heads, num_groups, is_causal);
- config.has_sinks = has_sinks;
- config.scale = None;
- super::single_pass::AttentionSinglePass::new(&config)
-}
-
-fn create_two_pass_kernel(
- head_dim: usize,
- num_q_heads: usize,
- num_groups: usize,
- is_causal: bool,
-) -> super::two_pass::AttentionTwoPass {
- let config = super::default_attention_config::<f32>(head_dim, num_q_heads, num_groups, is_causal);
- super::two_pass::AttentionTwoPass::new(&config)
-}
-
-fn run_gemm_attention(
- context: &<Metal as Backend>::Context,
- queries: &Array4<f32>,
- keys: &Array4<f32>,
- values: &Array4<f32>,
- sinks: Option<&[f32]>,
- scale: f32,
- is_causal: bool,
-) -> Result<Array4<f32>, Box<dyn std::error::Error>> {
- let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
- let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();
-
- let mut config = super::default_attention_config::<f32>(head_dim, num_heads, num_kv_heads, is_causal);
- config.has_sinks = sinks.is_some();
- config.scale = Some(scale);
- let kernel = super::gemm::AttentionGemm::new(&config);
-
- let query_buffer = create_query_buffer(queries, context);
- let key_buffer = create_attention_cache_buffer(keys, seq_len, context);
- let value_buffer = create_attention_cache_buffer(values, seq_len, context);
-
- let sinks_buffer = sinks.map(|sinks| create_sinks_buffer(sinks, context));
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
-
- let args = AttentionArguments {
- queries: &query_buffer,
- keys: &key_buffer,
- values: &value_buffer,
- suffix_length: seq_len as u32,
- trie: None::<&<Metal as Backend>::GlobalBuffer>,
- sinks: sinks_buffer.as_ref(),
- cache: KVCacheView::full(0),
- gate: None::<&<Metal as Backend>::GlobalBuffer>,
- };
-
- let pooled_output = kernel.encode(args, &mut command_buffer)?;
- let mut output_buffer = create_buffer::<Metal, f32>(context, num_heads * seq_len * head_dim);
- command_buffer.encode_copy(&pooled_output, &mut output_buffer);
- let completed = command_buffer.end_encoding().submit().wait_until_completed()?;
- drop(pooled_output);
- drop(completed);
-
- let output: Vec<f32> = buffer_to_vec(&output_buffer);
-
- let kernel_output = convert_kernel_output(&output, batch_size, num_heads, seq_len, head_dim);
-
- Ok(kernel_output)
-}
-
-fn compare_results(
- kernel_output: &Array4<f32>,
- reference_output: &Array4<f32>,
- tolerance: f32,
- test_name: &str,
-) -> Result<(), String> {
- let max_diff =
- kernel_output.iter().zip(reference_output.iter()).map(|(lhs, rhs)| (lhs - rhs).abs()).fold(0.0_f32, f32::max);
-
- if max_diff >= tolerance {
- return Err(format!(
- "{} output differs from reference by more than {}: max_diff = {}",
- test_name, tolerance, max_diff
- ));
- }
-
- Ok(())
-}
-
-#[uzu_test]
-fn test_single_pass_attention_basic() {
- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
-
- let batch_size = 1;
- let num_heads = 4;
- let num_kv_heads = 4;
- let seq_len = 8;
- let head_dim = 64;
- let scale = 1.0 / (head_dim as f32).sqrt();
-
- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
- let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, false, false);
- let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)
- .expect("single-pass attention");
-
- compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention").unwrap();
-}
-
-#[uzu_test]
-fn test_matrix_attention_matches_vector_and_cpu_seq256() {
- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
-
- let batch_size = 1;
- let num_heads = 8;
- let num_kv_heads = 2;
- let seq_len = 256;
- let head_dim = 64;
- let scale = 1.0 / (head_dim as f32).sqrt();
- let is_causal = true;
-
- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);
-
- let reference_output = reference_attention(&queries, &keys, &values, None, scale, is_causal);
- let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, false, is_causal);
- let vector_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)
- .expect("run vector attention");
- let matrix_output =
- run_gemm_attention(&context, &queries, &keys, &values, None, scale, is_causal).expect("run matrix attention");
- let tol_cpu = 5e-2;
- compare_results(&vector_output, &reference_output, tol_cpu, "vector single-pass attention vs CPU").unwrap();
- compare_results(&matrix_output, &reference_output, tol_cpu, "matrix attention vs CPU").unwrap();
-
- let max_diff_vector_matrix =
- vector_output.iter().zip(matrix_output.iter()).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max);
- let tol_vector_matrix = 5e-2;
- assert!(
- max_diff_vector_matrix <= tol_vector_matrix,
- "vector single-pass attention and matrix attention differ too much: max_diff = {} (tol={})",
- max_diff_vector_matrix,
- tol_vector_matrix
- );
-}
-
-#[uzu_test]
-fn test_single_pass_attention_with_sinks() {
- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
-
- let batch_size = 1;
- let num_heads = 4;
- let num_kv_heads = 4;
- let seq_len = 8;
- let head_dim = 64;
- let scale = 1.0 / (head_dim as f32).sqrt();
-
- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 444);
-
- let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();
- let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, true, false);
-
- let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);
- let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale)
- .expect("single-pass attention with sinks");
-
- compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention with sinks").unwrap();
-}
-
-#[uzu_test]
-fn test_single_pass_attention_with_sinks_long_sequence() {
- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
-
- let batch_size = 1;
- let num_heads = 4;
- let num_kv_heads = 4;
- let seq_len = 64;
- let head_dim = 64;
- let scale = 1.0 / (head_dim as f32).sqrt();
-
- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 777);
-
- let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();
- let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, true, false);
-
- let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);
- let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale)
- .expect("single-pass attention with sinks (long sequence)");
-
- compare_results(&kernel_output, &reference_output, 5e-2, "Single-pass attention with sinks (long sequence)")
- .unwrap();
-}
-
-#[uzu_test]
-fn test_single_pass_attention_gqa() {
- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
-
- let batch_size = 1;
- let num_heads = 8;
- let num_kv_heads = 2;
- let seq_len = 8;
- let head_dim = 64;
- let scale = 1.0 / (head_dim as f32).sqrt();
-
- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
-
- let kernel = create_single_pass_kernel(head_dim, num_heads, num_kv_heads, false, false);
- let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)
- .expect("single-pass attention GQA");
-
- let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
- compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention GQA").unwrap();
-}
-
-fn run_two_pass_attention(
- kernel: &super::two_pass::AttentionTwoPass,
- context: &<Metal as Backend>::Context,
- queries: &Array4<f32>,
- keys: &Array4<f32>,
- values: &Array4<f32>,
- sinks: Option<&[f32]>,
- _scale: f32,
-) -> Result<Array4<f32>, Box<dyn std::error::Error>> {
- let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
- let queries_buffer = create_query_buffer(queries, context);
- let keys_buffer = create_attention_cache_buffer(keys, seq_len, context);
- let values_buffer = create_attention_cache_buffer(values, seq_len, context);
- let sinks_buffer = sinks.map(|sinks| create_sinks_buffer(sinks, context));
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
-
- let pooled_output = kernel.encode(
- AttentionArguments {
- queries: &queries_buffer,
- keys: &keys_buffer,
- values: &values_buffer,
- suffix_length: seq_len as u32,
- trie: None::<&<Metal as Backend>::GlobalBuffer>,
- sinks: sinks_buffer.as_ref(),
- cache: KVCacheView::full(0),
- gate: None::<&<Metal as Backend>::GlobalBuffer>,
- },
- &mut command_buffer,
- )?;
- let mut output_buffer = create_buffer::<Metal, f32>(context, num_heads * seq_len * head_dim);
- command_buffer.encode_copy(&pooled_output, &mut output_buffer);
- drop(pooled_output);
- submit_command_buffer(command_buffer);
-
- let output_slice: Vec<f32> = buffer_to_vec(&output_buffer);
- let kernel_output = convert_kernel_output(&output_slice, batch_size, num_heads, seq_len, head_dim);
-
- Ok(kernel_output)
-}
-
-#[uzu_test]
-fn test_two_pass_attention() {
- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
-
- let batch_size = 1;
- let num_heads = 8;
- let num_kv_heads: usize = 8;
- let seq_len = 2048;
- let head_dim = 64;
- let scale = 1.0 / (head_dim as f32).sqrt();
- let is_causal = false; // Non-causal attention for this test
-
- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
-
- let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
-
- let kernel = create_two_pass_kernel(head_dim, num_heads, num_kv_heads, is_causal);
- let kernel_output =
- run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale).expect("two-pass attention");
-
- compare_results(&kernel_output, &reference_output, 1e-2, "Two-pass attention").unwrap();
-}
-
-#[uzu_test]
-fn test_two_pass_attention_gqa() {
- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
-
- let batch_size = 1;
- let num_heads = 8;
- let num_kv_heads = 2;
- let seq_len = 4096;
- let head_dim = 64;
- let scale = 1.0 / (head_dim as f32).sqrt();
- let is_causal = false; // Non-causal attention for this test
-
- let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
-
- let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
-
- let kernel = create_two_pass_kernel(head_dim, num_heads, num_kv_heads, is_causal);
- let kernel_output = run_two_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)
- .expect("two-pass attention GQA");
-
- compare_results(&kernel_output, &reference_output, 1e-2, "Two-pass attention GQA").unwrap();
-}
-
type AttentionShape = (usize, usize, usize, usize, usize, bool);
fn attention_data(shape: AttentionShape) -> (Vec<bf16>, Vec<bf16>) {
@@ -573,11 +41,13 @@ fn run_attention<B: Backend>(
context: &B::Context,
shape: AttentionShape,
trie_nodes: Option<&[GpuTrieNode]>,
+ sinks_data: Option<&[bf16]>,
) -> Vec<bf16> {
let (keys, values) = attention_data(shape);
let (_, _, _, _, prefix_length, _) = shape;
let (head_dim, num_q_heads, num_groups, _, _, is_causal) = shape;
- let config = super::default_attention_config::<bf16>(head_dim, num_q_heads, num_groups, is_causal);
+ let mut config = super::default_attention_config::<bf16>(head_dim, num_q_heads, num_groups, is_causal);
+ config.has_sinks = sinks_data.is_some();
let kernel = <B::Kernels as Kernels>::AttentionKernel::new(context, config).expect("attention kernel");
run_attention_with_kernel::<B>(
context,
@@ -587,6 +57,7 @@ fn run_attention<B: Backend>(
&values,
KVCacheView::full(prefix_length as u32),
trie_nodes,
+ sinks_data,
)
}
@@ -598,12 +69,14 @@ fn run_attention_with_kernel<B: Backend>(
values_data: &[bf16],
cache: KVCacheView,
trie_nodes: Option<&[GpuTrieNode]>,
+ sinks_data: Option<&[bf16]>,
) -> Vec<bf16> {
let (head_dim, num_q_heads, _, suffix_length, _, _) = shape;
let queries =
create_buffer_with_data::<B, bf16>(context, &fill_attention(num_q_heads * suffix_length * head_dim, 0.5));
let keys = create_buffer_with_data::<B, bf16>(context, keys_data);
let values = create_buffer_with_data::<B, bf16>(context, values_data);
+ let sinks = sinks_data.map(|data| create_buffer_with_data::<B, bf16>(context, data));
let trie = trie_nodes.map(|nodes| {
let words: Vec<u32> = nodes.iter().flat_map(|node| [node.trie_start, node.trie_end, node.height]).collect();
create_buffer_with_data::<B, u32>(context, &words)
@@ -614,7 +87,7 @@ fn run_attention_with_kernel<B: Backend>(
values: &values,
suffix_length: suffix_length as u32,
trie: trie.as_ref(),
- sinks: None,
+ sinks: sinks.as_ref(),
cache,
gate: None::<&B::GlobalBuffer>,
};
@@ -628,6 +101,72 @@ fn run_attention_with_kernel<B: Backend>(
buffer_to_vec::<B, bf16>(&output)
}
+enum DirectKernel {
+ SinglePass,
+ TwoPass,
+ Gemm,
+}
+
+fn run_direct_metal(
+ context: &<Metal as Backend>::Context,
+ shape: AttentionShape,
+ sinks_data: Option<&[bf16]>,
+ route: DirectKernel,
+) -> Vec<bf16> {
+ let (head_dim, num_q_heads, num_groups, suffix_length, prefix_length, is_causal) = shape;
+ let mut config = super::default_attention_config::<bf16>(head_dim, num_q_heads, num_groups, is_causal);
+ config.has_sinks = sinks_data.is_some();
+ let (keys_data, values_data) = attention_data(shape);
+ let queries =
+ create_buffer_with_data::<Metal, bf16>(context, &fill_attention(num_q_heads * suffix_length * head_dim, 0.5));
+ let keys = create_buffer_with_data::<Metal, bf16>(context, &keys_data);
+ let values = create_buffer_with_data::<Metal, bf16>(context, &values_data);
+ let sinks = sinks_data.map(|data| create_buffer_with_data::<Metal, bf16>(context, data));
+ let arguments = AttentionArguments {
+ queries: &queries,
+ keys: &keys,
+ values: &values,
+ suffix_length: suffix_length as u32,
+ trie: None::<&<Metal as Backend>::GlobalBuffer>,
+ sinks: sinks.as_ref(),
+ cache: KVCacheView::full(prefix_length as u32),
+ gate: None::<&<Metal as Backend>::GlobalBuffer>,
+ };
+ let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
+ let pooled = match route {
+ DirectKernel::SinglePass => {
+ single_pass::AttentionSinglePass::new(&config).encode(arguments, &mut command_buffer)
+ },
+ DirectKernel::TwoPass => two_pass::AttentionTwoPass::new(&config).encode(arguments, &mut command_buffer),
+ DirectKernel::Gemm => gemm::AttentionGemm::new(&config).encode(arguments, &mut command_buffer),
+ }
+ .expect("direct attention");
+ let mut output = create_buffer::<Metal, bf16>(context, suffix_length * num_q_heads * head_dim);
+ command_buffer.encode_copy(&pooled, &mut output);
+ submit_command_buffer(command_buffer);
+ buffer_to_vec::<Metal, bf16>(&output)
+}
+
+#[uzu_test]
+fn direct_bf16_attention_matches_cpu() {
+ let cpu = <Cpu as Backend>::Context::new().expect("CPU context");
+ let metal = <Metal as Backend>::Context::new().expect("Metal context");
+ for (shape, route, has_sinks) in [
+ ((64, 4, 4, 8, 0, false), DirectKernel::SinglePass, false),
+ ((64, 4, 4, 64, 0, false), DirectKernel::SinglePass, true),
+ ((64, 8, 2, 8, 0, false), DirectKernel::SinglePass, false),
+ ((64, 8, 2, 256, 0, true), DirectKernel::SinglePass, false),
+ ((64, 8, 2, 256, 0, true), DirectKernel::Gemm, false),
+ ((64, 8, 8, 128, 0, false), DirectKernel::TwoPass, false),
+ ((64, 8, 2, 64, 1024, false), DirectKernel::TwoPass, false),
+ ] {
+ let sinks = has_sinks.then(|| vec![bf16::from_f32(1.0); shape.1]);
+ let expected = run_attention::<Cpu>(cpu.as_ref(), shape, None, sinks.as_deref());
+ let actual = run_direct_metal(metal.as_ref(), shape, sinks.as_deref(), route);
+ assert_eq_float::<bf16>(&expected, &actual, 1e-2, "direct BF16 attention");
+ }
+}
+
#[uzu_test]
fn attention_kernel_matches_cpu() {
const TRIE_SUFFIX_LENGTH: usize = 31;
@@ -648,11 +187,18 @@ fn attention_kernel_matches_cpu() {
] {
let nodes = use_trie.then_some(trie.as_slice());
let shape = (head_dim, num_q_heads, num_groups, suffix_length, prefix_length, causal);
- let expected = run_attention::<Cpu>(cpu_context.as_ref(), shape, nodes);
- let actual = run_attention::<Metal>(metal_context.as_ref(), shape, nodes);
+ let expected = run_attention::<Cpu>(cpu_context.as_ref(), shape, nodes, None);
+ let actual = run_attention::<Metal>(metal_context.as_ref(), shape, nodes, None);
let label = format!("attention kernel D{head_dim} S{suffix_length} L{prefix_length} causal={causal}");
assert_eq_float::<bf16>(&expected, &actual, 1e-2, &label);
}
+ let sinks = vec![bf16::from_f32(1.0); 8];
+ for prefix_length in [32, 2048] {
+ let shape = (512, 8, 2, 1, prefix_length, false);
+ let expected = run_attention::<Cpu>(cpu_context.as_ref(), shape, None, Some(&sinks));
+ let actual = run_attention::<Metal>(metal_context.as_ref(), shape, None, Some(&sinks));
+ assert_eq_float::<bf16>(&expected, &actual, 1e-2, "BF16 GQA with sinks");
+ }
}
#[uzu_test]
@@ -668,9 +214,9 @@ fn attention_kernel_reuses_instance_for_flat_and_trie() {
.expect("attention kernel");
let (keys, values) = attention_data(shape);
let cache = KVCacheView::full(1024);
- let flat = run_attention_with_kernel::<Metal>(context.as_ref(), &kernel, shape, &keys, &values, cache, None);
+ let flat = run_attention_with_kernel::<Metal>(context.as_ref(), &kernel, shape, &keys, &values, cache, None, None);
let trie_output =
- run_attention_with_kernel::<Metal>(context.as_ref(), &kernel, shape, &keys, &values, cache, Some(&trie));
+ run_attention_with_kernel::<Metal>(context.as_ref(), &kernel, shape, &keys, &values, cache, Some(&trie), None);
let cpu_context = <Cpu as Backend>::Context::new().expect("CPU attention context");
let cpu_kernel = <<Cpu as Backend>::Kernels as Kernels>::AttentionKernel::new(
cpu_context.as_ref(),
@@ -678,9 +224,17 @@ fn attention_kernel_reuses_instance_for_flat_and_trie() {
)
.expect("CPU attention kernel");
let expected_flat =
- run_attention_with_kernel::<Cpu>(cpu_context.as_ref(), &cpu_kernel, shape, &keys, &values, cache, None);
- let expected_trie =
- run_attention_with_kernel::<Cpu>(cpu_context.as_ref(), &cpu_kernel, shape, &keys, &values, cache, Some(&trie));
+ run_attention_with_kernel::<Cpu>(cpu_context.as_ref(), &cpu_kernel, shape, &keys, &values, cache, None, None);
+ let expected_trie = run_attention_with_kernel::<Cpu>(
+ cpu_context.as_ref(),
+ &cpu_kernel,
+ shape,
+ &keys,
+ &values,
+ cache,
+ Some(&trie),
+ None,
+ );
assert_eq_float::<bf16>(&expected_flat, &flat, 1e-2, "flat mask specialization");
assert_eq_float::<bf16>(&expected_trie, &trie_output, 1e-2, "trie mask specialization");
}
@@ -737,6 +291,7 @@ fn attention_kernel_ring_matches_full_on_wrap() {
&full_values,
KVCacheView::full(CONTEXT as u32),
None,
+ None,
);
let ring = run_attention_with_kernel::<Metal>(
context.as_ref(),
@@ -746,6 +301,7 @@ fn attention_kernel_ring_matches_full_on_wrap() {
&ring_values,
KVCacheView::ring(CAPACITY as u32, ring_offset as u32),
None,
+ None,
);
assert_eq_float::<bf16>(&full, &ring, 1e-2, "wrapped ring attention");
}