Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

remove attention mask (#284)

uuuvn4f1a4a78781e
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs
index 42e39e5ca..befc554b2 100644
--- a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs
@@ -116,13 +116,7 @@ impl StructuredAudioCodecGraph {
local_rope.update_data(&transformer_tree, "local_rope");
}
}
- let scratch_buffers = ScratchBuffers::new(
- context.as_ref(),
- &decoder_config,
- &model_shape,
- max_sequence_length,
- max_sequence_length,
- );
+ let scratch_buffers = ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_sequence_length);
let (layers, output_norm) = Decoder::build_transformer_layers_and_norm(
context.as_ref(),
&decoder_config,
@@ -349,7 +343,6 @@ impl StructuredAudioCodecGraph {
runtime.shared_buffers.clone(),
&token_ids,
&token_positions,
- false,
1,
);
@@ -458,7 +451,6 @@ impl StructuredAudioCodecGraph {
runtime.shared_buffers.clone(),
&token_ids,
&token_positions,
- false,
1,
);
diff --git a/crates/uzu/src/backends/common/gpu_types/attention.rs b/crates/uzu/src/backends/common/gpu_types/attention.rs
index f608e13d1..e575269cb 100644
--- a/crates/uzu/src/backends/common/gpu_types/attention.rs
+++ b/crates/uzu/src/backends/common/gpu_types/attention.rs
@@ -34,16 +34,3 @@ pub struct AttnParams {
/// Key tiling: k_len % BK
pub k_rem: i32,
}
-
-/// Parameters describing the additive attention mask.
-///
-/// Mask is treated as a 2D matrix [qL, kL] with:
-/// - per-batch base offset (`m_strides[0]`)
-/// - per-head base offset (`m_strides[1]`)
-/// - row stride in elements (`m_strides[2]`)
-/// Column stride is assumed to be 1.
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
-pub struct AttnMaskParams {
- pub m_strides: [i64; 3],
-}
diff --git a/crates/uzu/src/backends/common/gpu_types/mod.rs b/crates/uzu/src/backends/common/gpu_types/mod.rs
index 1111d902c..26f84c140 100644
--- a/crates/uzu/src/backends/common/gpu_types/mod.rs
+++ b/crates/uzu/src/backends/common/gpu_types/mod.rs
@@ -9,6 +9,8 @@ pub mod attention;
pub mod kv_cache_update;
pub mod matmul;
pub mod quantization;
+pub mod ring;
+pub mod trie;
pub use activation_type::*;
pub use argmax::*;
diff --git a/crates/uzu/src/backends/common/gpu_types/ring.rs b/crates/uzu/src/backends/common/gpu_types/ring.rs
new file mode 100644
index 000000000..d6739fac6
--- /dev/null
+++ b/crates/uzu/src/backends/common/gpu_types/ring.rs
@@ -0,0 +1,6 @@
+#[derive(Clone, Copy, Debug)]
+#[repr(C)]
+pub struct RingParams {
+ pub ring_offset: u32,
+ pub ring_length: u32,
+}
diff --git a/crates/uzu/src/backends/common/gpu_types/trie.rs b/crates/uzu/src/backends/common/gpu_types/trie.rs
new file mode 100644
index 000000000..fe23a6ec0
--- /dev/null
+++ b/crates/uzu/src/backends/common/gpu_types/trie.rs
@@ -0,0 +1,7 @@
+#[derive(Clone, Copy, Debug)]
+#[repr(C)]
+pub struct TrieNode {
+ pub trie_start: u32,
+ pub trie_end: u32,
+ pub height: u32,
+}
diff --git a/crates/uzu/src/backends/common/kernel/attention.rs b/crates/uzu/src/backends/common/kernel/attention.rs
index 339d104fd..868c04cfd 100644
--- a/crates/uzu/src/backends/common/kernel/attention.rs
+++ b/crates/uzu/src/backends/common/kernel/attention.rs
@@ -7,7 +7,7 @@ use crate::{
DataType,
backends::common::{
Backend, Encoder, Kernels,
- gpu_types::{AttnMaskParams, AttnParams},
+ gpu_types::{AttnParams, ring::RingParams},
kernel::AttentionGemmKernel,
},
};
@@ -15,19 +15,21 @@ use crate::{
const BQ: usize = 32;
pub struct AttentionGemmArguments<'a, B: Backend> {
- pub queries_buffer: &'a B::Buffer, // buffer(0)
- pub keys_buffer: &'a B::Buffer, // buffer(1)
- pub values_buffer: &'a B::Buffer, // buffer(2)
- pub output_buffer: &'a mut B::Buffer, // buffer(3)
- pub mask_buffer: Option<&'a B::Buffer>, // buffer(6)
- pub sinks_buffer: Option<&'a B::Buffer>, // buffer(7)
+ pub queries_buffer: &'a B::Buffer,
+ pub keys_buffer: &'a B::Buffer,
+ pub values_buffer: &'a B::Buffer,
+ pub output_buffer: &'a mut B::Buffer,
+ pub trie_buffer: Option<&'a B::Buffer>,
+ pub sinks_buffer: Option<&'a B::Buffer>,
pub num_heads: usize,
pub num_groups: usize,
pub suffix_length: usize, // qL
pub sequence_length: usize, // kL (prefix + suffix)
pub segment_prefix_length: usize, // qL_off
pub max_sequence_length: usize, // stride for K/V cache
+ pub ring_params: Option<RingParams>,
pub head_dim: usize,
+ pub sliding_window_size: Option<usize>,
pub is_causal: bool,
pub scale: f32,
}
@@ -57,17 +59,23 @@ impl<B: Backend> AttentionGemmBlock<B> {
} else {
16
};
+ let head_dim = args.head_dim;
let align_q = (args.suffix_length % BQ) == 0;
let align_k = (args.sequence_length % bk) == 0;
- let has_mask = args.mask_buffer.is_some();
+ let is_kv_cache_ring = args.ring_params.is_some();
+ let is_causal = args.is_causal;
+ let is_trie = args.trie_buffer.is_some();
+ let is_sliding_window = args.sliding_window_size.is_some();
let has_sinks = args.sinks_buffer.is_some();
let key = KernelKey {
bk,
- head_dim: args.head_dim,
+ head_dim,
align_q,
align_k,
- is_causal: args.is_causal,
- has_mask,
+ is_kv_cache_ring,
+ is_causal,
+ is_trie,
+ is_sliding_window,
has_sinks,
};
@@ -79,11 +87,13 @@ impl<B: Backend> AttentionGemmBlock<B> {
context,
self.data_type,
bk as u32,
- args.head_dim as u32,
+ head_dim as u32,
align_q,
align_k,
- args.is_causal,
- has_mask,
+ is_kv_cache_ring,
+ is_causal,
+ is_trie,
+ is_sliding_window,
has_sinks,
)?;
entry.insert(kernel)
@@ -91,14 +101,14 @@ impl<B: Backend> AttentionGemmBlock<B> {
};
// Params (all strides in elements)
- let q_head_stride = (args.suffix_length * args.head_dim) as i64;
- let q_seq_stride = args.head_dim as i64;
+ let q_head_stride = (args.suffix_length * head_dim) as i64;
+ let q_seq_stride = head_dim as i64;
- let kv_head_stride = (args.max_sequence_length * args.head_dim) as i64;
- let kv_seq_stride = args.head_dim as i64;
+ let kv_head_stride = (args.max_sequence_length * head_dim) as i64;
+ let kv_seq_stride = head_dim as i64;
- let o_head_stride = args.head_dim as i64;
- let o_seq_stride = (args.num_heads * args.head_dim) as i64;
+ let o_head_stride = head_dim as i64;
+ let o_seq_stride = (args.num_heads * head_dim) as i64;
let nk = (args.sequence_length + bk - 1) / bk;
let nq_aligned = args.suffix_length / BQ;
@@ -121,20 +131,15 @@ impl<B: Backend> AttentionGemmBlock<B> {
k_rem: (args.sequence_length - nk_aligned * bk) as i32,
};
- let attn_mask_params = AttnMaskParams {
- // We use a shared bias matrix for all heads/batches.
- m_strides: [0, 0, args.sequence_length as i64],
- };
- let attn_mask_params_opt = args.mask_buffer.map(|_| attn_mask_params);
-
kernel.encode(
args.queries_buffer,
args.keys_buffer,
args.values_buffer,
args.output_buffer,
params,
- attn_mask_params_opt,
- args.mask_buffer,
+ args.ring_params,
+ args.trie_buffer,
+ args.sliding_window_size.map(|s| s as u32),
args.sinks_buffer,
args.num_heads as u32,
args.suffix_length as u32,
@@ -151,7 +156,9 @@ struct KernelKey {
head_dim: usize,
align_q: bool,
align_k: bool,
+ is_kv_cache_ring: bool,
is_causal: bool,
- has_mask: bool,
+ is_trie: bool,
+ is_sliding_window: bool,
has_sinks: bool,
}
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rs
index 15fb2c5d3..d7c096481 100644
--- a/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rs
+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rs
@@ -2,7 +2,10 @@ use dsl::kernel;
use half::{bf16, f16};
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{
+ ArrayElement,
+ backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},
+};
#[kernel(AttentionGemm)]
#[variants(T, f32, f16, bf16)]
@@ -14,21 +17,29 @@ pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(
v: *const T,
o: *mut T,
params: crate::backends::common::gpu_types::attention::AttnParams,
- #[optional(has_mask)] mask_params: Option<crate::backends::common::gpu_types::attention::AttnMaskParams>,
- #[optional(has_mask)] mask: Option<*const T>,
+ #[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,
+ #[optional(is_trie)] trie: Option<*const TrieNode>,
+ #[optional(is_sliding_window)] sliding_window_size: Option<u32>,
#[optional(has_sinks)] sinks: Option<*const f32>,
num_heads: u32,
suffix_length: u32,
#[specialize] align_q: bool,
#[specialize] align_k: bool,
- #[specialize] do_causal: bool,
- #[specialize] has_mask: bool,
+ #[specialize] is_kv_cache_ring: bool,
+ #[specialize] is_causal: bool,
+ #[specialize] is_trie: bool,
+ #[specialize] is_sliding_window: bool,
#[specialize] has_sinks: bool,
) {
let q_len = params.q_len as usize;
let k_len = params.k_len as usize;
let head_dim = BD as usize;
- let q_off = params.q_off as usize;
+ let prefix_length = params.q_off as usize;
+ let suffix_position = if is_kv_cache_ring {
+ ring_params.unwrap().ring_length as usize
+ } else {
+ prefix_length
+ };
for head_idx in 0..num_heads as usize {
let kv_head_idx = head_idx / params.gqa_factor as usize;
@@ -42,6 +53,13 @@ pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(
let q_row = unsafe { q_head.add(qi * params.q_strides[2] as usize) };
let o_row = unsafe { o_head.add(qi * params.o_strides[2] as usize) };
+ let query_position = if is_trie {
+ let trie_node = unsafe { &*trie.unwrap().add(qi) };
+ suffix_position + trie_node.height as usize
+ } else {
+ suffix_position + qi
+ };
+
// Read query row and pre-scale
let mut q_vec = vec![0.0f32; head_dim];
for j in 0..head_dim {
@@ -60,8 +78,17 @@ pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(
// Loop over all key positions
for ki in 0..k_len {
- // Causal mask: query at absolute position q_off + qi can only attend to ki <= q_off + qi
- if do_causal && ki > q_off + qi {
+ if !should_use_key(
+ ring_params,
+ trie,
+ sliding_window_size,
+ qi as u32,
+ prefix_length as u32,
+ suffix_position as u32,
+ query_position as u32,
+ ki as u32,
+ is_causal,
+ ) {
continue;
}
@@ -72,15 +99,6 @@ pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(
score += q_vec[j] * unsafe { *k_row.add(j) }.to_f32().unwrap();
}
- // Add external mask bias
- if has_mask {
- let mask_params = mask_params.unwrap();
- let mask = mask.unwrap();
- let mask_ptr = unsafe { mask.add(head_idx * mask_params.m_strides[1] as usize) };
- let mv = unsafe { *mask_ptr.add(qi * mask_params.m_strides[2] as usize + ki) }.to_f32().unwrap();
- score += f32::max(mv, -1e9);
- }
-
// Online softmax update
let new_max = f32::max(max_score, score);
let factor = (max_score - new_max).exp();
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs
index ffe5b8eac..786a4a744 100644
--- a/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs
+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs
@@ -2,7 +2,10 @@ use dsl::kernel;
use half::{bf16, f16};
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{
+ ArrayElement,
+ backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},
+};
#[kernel(AttentionSinglePass)]
#[variants(T, f32, f16, bf16)]
@@ -18,41 +21,46 @@ pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(
k_seq_stride: u32,
v_head_stride: u32,
v_seq_stride: u32,
+ #[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,
scale: f32,
- #[optional(float_mask)] fmask: Option<*const T>,
- #[optional(has_mask)] mask_kv_seq_stride: Option<u32>,
- #[optional(has_mask)] mask_q_seq_stride: Option<u32>,
- #[optional(has_mask)] mask_head_stride: Option<u32>,
+ #[optional(is_trie)] trie: Option<*const TrieNode>,
+ #[optional(is_sliding_window)] sliding_window_size: Option<u32>,
#[optional(has_sinks)] sinks: Option<*const f32>,
num_heads: u32,
suffix_length: u32,
- #[specialize] float_mask: bool,
- #[specialize] has_mask: bool,
#[specialize] has_sinks: bool,
- #[specialize] do_causal: bool,
+ #[specialize] is_kv_cache_ring: bool,
+ #[specialize] is_causal: bool,
+ #[specialize] is_trie: bool,
+ #[specialize] is_sliding_window: bool,
) {
let value_dim = HEAD_DIM;
+ let prefix_length = sequence_length - suffix_length;
+ let suffix_position = if let Some(ring_params) = ring_params {
+ ring_params.ring_length
+ } else {
+ prefix_length
+ };
+
for head_idx in 0..num_heads {
for q_seq_idx in 0..suffix_length {
let kv_head_idx = head_idx / gqa_factor;
let o_offset = q_seq_idx * num_heads + head_idx;
let q_offset = head_idx * suffix_length + q_seq_idx;
+ let query_position = if is_trie {
+ let trie_node = unsafe { &*trie.unwrap().add(q_seq_idx as usize) };
+ suffix_position + trie_node.height
+ } else {
+ suffix_position + q_seq_idx
+ };
+
let queries: *const T = unsafe { queries.add((q_offset * HEAD_DIM) as usize) };
let keys: *const T = unsafe { keys.add((kv_head_idx * k_head_stride) as usize) };
let values: *const T = unsafe { values.add((kv_head_idx * v_head_stride) as usize) };
let out: *mut T = unsafe { out.add((o_offset * value_dim) as usize) };
- let fmask: Option<*const T> = if let Some(fmask) = fmask {
- unsafe {
- let offset = head_idx * mask_head_stride.unwrap_or(0) + q_seq_idx * mask_q_seq_stride.unwrap_or(0);
- Some(fmask.add(offset as usize))
- }
- } else {
- None
- };
-
// Read the query and 0 the output accumulator
let mut q = vec![0.0f32; HEAD_DIM as usize];
let mut o = vec![0.0f32; HEAD_DIM as usize];
@@ -70,9 +78,17 @@ pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(
// For each key
for i in 0..sequence_length {
- let use_key = !do_causal || i <= (sequence_length - suffix_length + q_seq_idx);
-
- if use_key {
+ if should_use_key(
+ ring_params,
+ trie,
+ sliding_window_size,
+ q_seq_idx,
+ prefix_length,
+ suffix_position,
+ query_position,
+ i,
+ is_causal,
+ ) {
let keys = unsafe { keys.add((i * k_seq_stride) as usize) };
// Compute the i-th score
@@ -80,10 +96,6 @@ pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(
for j in 0..HEAD_DIM as usize {
score += q[j] * unsafe { *keys.add(j) }.to_f32().unwrap();
}
- if float_mask {
- let fmask = unsafe { fmask.unwrap().add((i * mask_kv_seq_stride.unwrap()) as usize) };
- score += f32::max(-1e9, unsafe { *fmask }.to_f32().unwrap());
- }
// Update the accumulators
let new_max = f32::max(max_score, score);
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
index 3be696c10..ba23f1dd1 100644
--- a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs
@@ -2,7 +2,10 @@ use dsl::kernel;
use half::{bf16, f16};
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{
+ ArrayElement,
+ backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},
+};
const TOTAL_BLOCKS_COUNT: u32 = 32;
const SEQUENCE_BLOCK_SIZE_1: u32 = 8;
@@ -23,25 +26,39 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(
k_seq_stride: u32,
v_head_stride: u32,
v_seq_stride: u32,
+ #[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,
scale: f32,
num_heads: u32,
suffix_length: u32,
- #[optional(float_mask)] fmask: Option<*const T>,
- #[optional(has_mask)] mask_kv_seq_stride: Option<u32>,
- #[optional(has_mask)] mask_q_seq_stride: Option<u32>,
- #[optional(has_mask)] mask_head_stride: Option<u32>,
+ #[optional(is_trie)] trie: Option<*const TrieNode>,
+ #[optional(is_sliding_window)] sliding_window_size: Option<u32>,
#[optional(has_sinks)] sinks: Option<*const f32>,
- #[specialize] float_mask: bool,
- #[specialize] has_mask: bool,
#[specialize] has_sinks: bool,
- #[specialize] do_causal: bool,
+ #[specialize] is_kv_cache_ring: bool,
+ #[specialize] is_causal: bool,
+ #[specialize] is_trie: bool,
+ #[specialize] is_sliding_window: bool,
) {
let value_dim = HEAD_DIM;
let mut q = vec![0.0f32; HEAD_DIM as usize];
let mut o = vec![0.0f32; HEAD_DIM as usize];
+ let prefix_length = sequence_length - suffix_length;
+ let suffix_position = if let Some(ring_params) = ring_params {
+ ring_params.ring_length
+ } else {
+ prefix_length
+ };
+
for head_idx in 0..num_heads {
for q_seq_idx in 0..suffix_length {
+ let query_position = if is_trie {
+ let trie_node = unsafe { &*trie.unwrap().add(q_seq_idx as usize) };
+ suffix_position + trie_node.height
+ } else {
+ suffix_position + q_seq_idx
+ };
+
for block_idx in 0..TOTAL_BLOCKS_COUNT {
let o_offset = q_seq_idx * num_heads + head_idx;
let q_offset = head_idx * suffix_length + q_seq_idx;
@@ -53,13 +70,6 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(
let out_base: *mut f32 =
unsafe { out.add((o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim) as usize) };
- let fmask_base: Option<*const T> = if float_mask {
- let offset = head_idx * mask_head_stride.unwrap_or(0) + q_seq_idx * mask_q_seq_stride.unwrap_or(0);
- Some(unsafe { fmask.unwrap().add(offset as usize) })
- } else {
- None
- };
-
// Read the query and scale
for j in 0..HEAD_DIM as usize {
q[j] = scale * unsafe { *queries_base.add(j) }.to_f32().unwrap();
@@ -84,8 +94,17 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(
break;
}
- let use_key = !do_causal || i <= (sequence_length - suffix_length + q_seq_idx);
- if use_key {
+ if should_use_key(
+ ring_params,
+ trie,
+ sliding_window_size,
+ q_seq_idx,
+ prefix_length,
+ suffix_position,
+ query_position,
+ i,
+ is_causal,
+ ) {
let keys_ptr = unsafe { keys_base.add((i * k_seq_stride) as usize) };
// Compute the i-th score
@@ -93,11 +112,6 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(
for j in 0..HEAD_DIM as usize {
score += q[j] * unsafe { *keys_ptr.add(j) }.to_f32().unwrap();
}
- if float_mask {
- let fmask_ptr =
- unsafe { fmask_base.unwrap().add((i * mask_kv_seq_stride.unwrap()) as usize) };
- score += f32::max(-1e9, unsafe { *fmask_ptr }.to_f32().unwrap());
- }
// Update the accumulators
let new_max = f32::max(max_score, score);
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/mask.rs b/crates/uzu/src/backends/cpu/kernel/attention/mask.rs
new file mode 100644
index 000000000..8ebb69313
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/attention/mask.rs
@@ -0,0 +1,62 @@
+use crate::backends::common::gpu_types::{ring::RingParams, trie::TrieNode};
+
+pub fn should_use_key(
+ ring_params: Option<RingParams>,
+ trie: Option<*const TrieNode>,
+ sliding_window_size: Option<u32>,
+ q_seq_idx: u32,
+ prefix_length: u32,
+ suffix_position: u32,
+ query_position: u32,
+ i: u32,
+ is_causal: bool,
+) -> bool {
+ let mut use_key = true;
+ let key_position;
+
+ if i >= prefix_length {
+ // We're in the suffix
+ let key_position_in_suffix = i - prefix_length;
+
+ if let Some(trie) = trie {
+ // Trie, get position from the flat trie buffer
+ let trie_node = unsafe { &*trie.add(key_position_in_suffix as usize) };
+
+ key_position = suffix_position + trie_node.height;
+
+ if is_causal {
+ use_key &= q_seq_idx >= trie_node.trie_start && q_seq_idx <= trie_node.trie_end;
+ }
+ } else {
+ // Non-trie, position equals index in the suffix
+ key_position = suffix_position + key_position_in_suffix;
+
+ if is_causal {
+ use_key &= key_position_in_suffix <= q_seq_idx;
+ }
+ }
+ } else {
+ // We're in the prefix
+ if let Some(ring_params) = ring_params {
+ // Ring, calculate position in the ring
+ key_position = (prefix_length + i - ring_params.ring_offset) % prefix_length;
+ // Ring also skips entries that aren't filled yet
+ use_key &= key_position < ring_params.ring_length;
+ } else {
+ // Non-ring, position equals to the index in the kvcache buffer
+ key_position = i;
+ }
+ }
+
+ if let Some(sliding_window_size) = sliding_window_size {
+ if is_causal {
+ use_key &= key_position <= query_position && (query_position - key_position) < sliding_window_size;
+ } else if key_position <= query_position {
+ use_key &= (query_position - key_position) <= sliding_window_size / 2;
+ } else {
+ use_key &= (key_position - query_position) <= sliding_window_size / 2;
+ }
+ }
+
+ use_key
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu/src/backends/cpu/kernel/attention/mod.rs
index 4a6321bee..c862aeaf0 100644
--- a/crates/uzu/src/backends/cpu/kernel/attention/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/attention/mod.rs
@@ -3,3 +3,5 @@ pub mod attention_single_pass;
pub mod attention_two_pass;
pub mod attention_update_kv_cache;
pub mod sigmoid_gate;
+
+mod mask;
diff --git a/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs b/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs
deleted file mode 100644
index cb7733683..000000000
--- a/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs
+++ /dev/null
@@ -1,20 +0,0 @@
-use dsl::kernel;
-use half::{bf16, f16};
-use num_traits::Float;
-
-use crate::ArrayElement;
-
-#[kernel(MaskUpdate)]
-#[variants(T, f32, f16, bf16)]
-pub fn mask_update<T: ArrayElement + Float>(
- mask: *mut T,
- unmask_col: i32,
- mask_col: i32,
-) {
- if (unmask_col >= 0) {
- unsafe { *mask.add(unmask_col as usize) = T::zero() };
- }
- if (mask_col >= 0) {
- unsafe { *mask.add(mask_col as usize) = T::neg_infinity() };
- }
-}
diff --git a/crates/uzu/src/backends/cpu/kernel/mask_update/mod.rs b/crates/uzu/src/backends/cpu/kernel/mask_update/mod.rs
deleted file mode 100644
index 75a6d52a2..000000000
--- a/crates/uzu/src/backends/cpu/kernel/mask_update/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod mask_update;
diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rs
index 2d9020b1f..f6b5610f5 100644
--- a/crates/uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs
@@ -5,7 +5,7 @@ mod audio;
mod embedding;
mod kv_cache_update;
mod layer_norm;
-mod mask_update;
+
mod matmul;
mod mlp;
mod moe;
diff --git a/crates/uzu/src/backends/metal/generated/attention.h b/crates/uzu/src/backends/metal/generated/attention.h
index e3167fe4c..7e3e3b787 100644
--- a/crates/uzu/src/backends/metal/generated/attention.h
+++ b/crates/uzu/src/backends/metal/generated/attention.h
@@ -46,16 +46,6 @@ typedef struct {
int32_t k_rem;
} AttnParams;
-/**Parameters describing the additive attention mask. */
-/**Mask is treated as a 2D matrix [qL, kL] with: */
-/**- per-batch base offset (`m_strides[0]`) */
-/**- per-head base offset (`m_strides[1]`) */
-/**- row stride in elements (`m_strides[2]`) */
-/**Column stride is assumed to be 1. */
-typedef struct {
- int64_t m_strides[3];
-} AttnMaskParams;
-
#ifdef __METAL_VERSION__
} // namespace attention
} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/generated/ring.h b/crates/uzu/src/backends/metal/generated/ring.h
new file mode 100644
index 000000000..6819327e9
--- /dev/null
+++ b/crates/uzu/src/backends/metal/generated/ring.h
@@ -0,0 +1,27 @@
+// Auto-generated from gpu_types/ring.rs - do not edit manually
+#pragma once
+
+#ifndef UZU_RING_H
+#define UZU_RING_H
+
+#ifdef __METAL_VERSION__
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu {
+namespace ring {
+#else
+#include <stdint.h>
+#endif
+
+typedef struct {
+ uint32_t ring_offset;
+ uint32_t ring_length;
+} RingParams;
+
+#ifdef __METAL_VERSION__
+} // namespace ring
+} // namespace uzu
+#endif
+
+#endif // UZU_RING_H
diff --git a/crates/uzu/src/backends/metal/generated/trie.h b/crates/uzu/src/backends/metal/generated/trie.h
new file mode 100644
index 000000000..2c2ab2497
--- /dev/null
+++ b/crates/uzu/src/backends/metal/generated/trie.h
@@ -0,0 +1,28 @@
+// Auto-generated from gpu_types/trie.rs - do not edit manually
+#pragma once
+
+#ifndef UZU_TRIE_H
+#define UZU_TRIE_H
+
+#ifdef __METAL_VERSION__
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu {
+namespace trie {
+#else
+#include <stdint.h>
+#endif
+
+typedef struct {
+ uint32_t trie_start;
+ uint32_t trie_end;
+ uint32_t height;
+} TrieNode;
+
+#ifdef __METAL_VERSION__
+} // namespace trie
+} // namespace uzu
+#endif
+
+#endif // UZU_TRIE_H
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index 57009d9ae..acf5ff8d6 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -3,11 +3,16 @@
#include "../common/thread_context.h"
#include "../matmul/common/loader.h"
#include "../matmul/common/simdgroup_fragment.h"
-#include "attention.h"
+#include "../generated/ring.h"
+#include "../generated/trie.h"
+#include "../generated/attention.h"
+#include "mask.h"
using namespace metal;
using namespace uzu::matmul;
using namespace uzu::attention;
+using namespace uzu::ring;
+using namespace uzu::trie;
template <typename T>
struct TransformScale {
@@ -46,37 +51,37 @@ PUBLIC KERNEL(AttentionGemm)(
const device T* v,
device T* o,
const constant AttnParams& params,
- const constant AttnMaskParams& mask_params OPTIONAL(has_mask),
- const device T* mask OPTIONAL(has_mask),
+ const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),
+ const device TrieNode* trie OPTIONAL(is_trie),
+ const constant uint& sliding_window_size OPTIONAL(is_sliding_window),
const device float* sinks OPTIONAL(has_sinks),
const constant uint& num_heads,
const constant uint& suffix_length,
const bool align_q SPECIALIZE,
const bool align_k SPECIALIZE,
- const bool do_causal SPECIALIZE,
- const bool has_mask SPECIALIZE,
+ const bool is_kv_cache_ring SPECIALIZE,
+ const bool is_causal SPECIALIZE,
+ const bool is_trie SPECIALIZE,
+ const bool is_sliding_window SPECIALIZE,
const bool has_sinks SPECIALIZE,
threadgroup T q_smem[BLOCK_QUERY_ROWS * (BD + 16 / sizeof(T))],
threadgroup T kv_smem[BK * (BD + 16 / sizeof(T))],
const ThreadContext thread_context,
- const uint tgid_x GROUPS(suffix_length.div_ceil(BLOCK_QUERY_ROWS)),
- const uint tgid_y GROUPS(num_heads),
- const uint tgid_z GROUPS(1),
+ const uint q_tile_idx GROUPS(suffix_length.div_ceil(BLOCK_QUERY_ROWS)),
+ const uint head_idx GROUPS(num_heads),
+ const uint batch_idx GROUPS(1),
const uint lid THREADS(128)
) {
// -------------------------------------------------------------------------
// Pointer setup (all strides are in elements)
- // tgid_x: query tile index (BLOCK_QUERY_ROWS rows)
- // tgid_y: query head index
- // tgid_z: batch index (currently 1 in uzu, but kept for completeness)
- const uint batch_idx = tgid_z;
- const uint head_idx = tgid_y;
- const uint q_tile_idx = tgid_x;
+ // q_tile_idx: query tile index (BLOCK_QUERY_ROWS rows)
+ // head_idx: query head index
+ // batch_idx: batch index (currently 1 in uzu, but kept for completeness)
q += batch_idx * params.q_strides[0] + head_idx * params.q_strides[1] +
q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.q_strides[2];
- const int kv_head_idx = int(tgid_y) / params.gqa_factor;
+ const int kv_head_idx = int(head_idx) / params.gqa_factor;
k += batch_idx * params.k_strides[0] +
int64_t(kv_head_idx) * params.k_strides[1];
v += batch_idx * params.v_strides[0] +
@@ -85,9 +90,8 @@ PUBLIC KERNEL(AttentionGemm)(
o += batch_idx * params.o_strides[0] + head_idx * params.o_strides[1] +
q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.o_strides[2];
- if (has_mask) {
- mask += batch_idx * mask_params.m_strides[0] +
- head_idx * mask_params.m_strides[1];
+ if (is_trie) {
+ trie += batch_idx * suffix_length;
}
// -------------------------------------------------------------------------
@@ -242,7 +246,7 @@ PUBLIC KERNEL(AttentionGemm)(
// Load Q block once (and apply scaling)
threadgroup_barrier(mem_flags::mem_threadgroup);
- if (!align_q && int(tgid_x) == params.nq_aligned) {
+ if (!align_q && int(q_tile_idx) == params.nq_aligned) {
query_loader.load_safe(short2(BD, params.q_rem));
} else {
query_loader.load_unsafe();
@@ -256,21 +260,29 @@ PUBLIC KERNEL(AttentionGemm)(
AccumType sum_score = AccumType(0);
if (has_sinks) {
- max_score = M_LOG2E_F * static_cast<AccumType>(sinks[tgid_y]);
+ max_score = M_LOG2E_F * static_cast<AccumType>(sinks[head_idx]);
sum_score = AccumType(1);
}
// Determine K block loop limit (causal can early-stop)
int kb_lim = params.nk;
- if (do_causal) {
- const int q_max = (int(tgid_x) + 1) * BLOCK_QUERY_ROWS + params.q_off;
+ if (is_causal) {
+ const int q_max = (int(q_tile_idx) + 1) * BLOCK_QUERY_ROWS + params.q_off;
kb_lim = (q_max + BK - 1) / BK;
kb_lim = min(params.nk, kb_lim);
}
- const int q_rel = int(tgid_x) * BLOCK_QUERY_ROWS + int(simdgroup_row_base) +
- int(lane_row); // [0, q_len)
- const int q_abs = q_rel + params.q_off; // [0, k_len)
+ const int q_rel = int(q_tile_idx) * BLOCK_QUERY_ROWS +
+ int(simdgroup_row_base) + int(lane_row); // [0, q_len)
+
+ const int prefix_length = params.q_off;
+
+ const int suffix_position =
+ is_kv_cache_ring ? int(ring_params.ring_length) : prefix_length;
+
+ const int query_position = (is_trie && q_rel < params.q_len)
+ ? suffix_position + int(trie[q_rel].height)
+ : suffix_position + q_rel;
// Loop over KV blocks
for (int kb = 0; kb < kb_lim; kb++) {
@@ -323,31 +335,8 @@ PUBLIC KERNEL(AttentionGemm)(
}
}
- // Causal mask (only needed for the last few blocks near the diagonal)
- if (do_causal) {
- const int tail_blocks = (BLOCK_QUERY_ROWS + BK - 1) / BK + int(!align_k);
- const int tail_start = kb_lim - tail_blocks;
- if (kb >= tail_start) {
- METAL_PRAGMA_UNROLL
- for (short j = 0; j < KEY_GRID_COLS; j++) {
- thread auto& frag = score_fragment.multiply_accumulate_at(0, j);
- const int col_base =
- kb * BK + int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;
- if (q_abs < col_base) {
- frag[0] = neg_inf;
- }
- if (q_abs < (col_base + 1)) {
- frag[1] = neg_inf;
- }
- }
- }
- }
-
- // Add external mask (additive bias in natural-log domain; convert to log2)
- if (has_mask && q_rel < params.q_len) {
- const int64_t row_stride = mask_params.m_strides[2];
- const int64_t row_base = int64_t(q_rel) * row_stride;
-
+ // Unified masking: causal, trie, ring, sliding window
+ if (q_rel < params.q_len) {
METAL_PRAGMA_UNROLL
for (short j = 0; j < KEY_GRID_COLS; j++) {
thread auto& frag = score_fragment.multiply_accumulate_at(0, j);
@@ -357,16 +346,37 @@ PUBLIC KERNEL(AttentionGemm)(
const int k0 = col_base;
const int k1 = col_base + 1;
- if (k0 < params.k_len) {
- AccumType mv = static_cast<AccumType>(mask[row_base + int64_t(k0)]);
- mv = metal::max(mv, static_cast<AccumType>(-1e9f));
- frag[0] += M_LOG2E_F * mv;
- }
- if (k1 < params.k_len) {
- AccumType mv = static_cast<AccumType>(mask[row_base + int64_t(k1)]);
- mv = metal::max(mv, static_cast<AccumType>(-1e9f));
- frag[1] += M_LOG2E_F * mv;
- }
+ if (k0 < params.k_len && !should_use_key(
+ ring_params,
+ trie,
+ sliding_window_size,
+ q_rel,
+ prefix_length,
+ suffix_position,
+ query_position,
+ k0,
+ is_kv_cache_ring,
+ is_causal,
+ is_trie,
+ is_sliding_window
+ ))
+ frag[0] = neg_inf;
+
+ if (k1 < params.k_len && !should_use_key(
+ ring_params,
+ trie,
+ sliding_window_size,
+ q_rel,
+ prefix_length,
+ suffix_position,
+ query_position,
+ k1,
+ is_kv_cache_ring,
+ is_causal,
+ is_trie,
+ is_sliding_window
+ ))
+ frag[1] = neg_inf;
}
}
@@ -466,7 +476,7 @@ PUBLIC KERNEL(AttentionGemm)(
o += int64_t(simdgroup_row_base + lane_row) * params.o_strides[2] +
int64_t(lane_col);
- if (!align_q && int(tgid_x) == params.nq_aligned) {
+ if (!align_q && int(q_tile_idx) == params.nq_aligned) {
const short2 dst_tile_dims =
short2(BD - lane_col, params.q_rem - (simdgroup_row_base + lane_row));
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
index bec72e50c..5f75bbfde 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -2,10 +2,16 @@
#include <metal_simdgroup>
#include "../common/dsl.h"
#include "../common/thread_context.h"
+#include "../generated/ring.h"
+#include "../generated/trie.h"
+#include "mask.h"
#define SEQUENCE_BLOCK_SIZE 32
#define HEAD_BLOCK_SIZE 32
+using namespace uzu::ring;
+using namespace uzu::trie;
+
template <typename T, uint HEAD_DIM>
VARIANTS(T, float, half, bfloat)
VARIANTS(HEAD_DIM, 64, 128, 256)
@@ -20,27 +26,26 @@ PUBLIC KERNEL(AttentionSinglePass)(
const constant uint& k_seq_stride,
const constant uint& v_head_stride,
const constant uint& v_seq_stride,
+ const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),
const constant float& scale,
- const device T* fmask OPTIONAL(float_mask),
- const constant uint& mask_kv_seq_stride OPTIONAL(has_mask),
- const constant uint& mask_q_seq_stride OPTIONAL(has_mask),
- const constant uint& mask_head_stride OPTIONAL(has_mask),
+ const device TrieNode* trie OPTIONAL(is_trie),
+ const constant uint& sliding_window_size OPTIONAL(is_sliding_window),
const device float* sinks OPTIONAL(has_sinks),
const constant uint& num_heads,
const constant uint& suffix_length,
threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],
threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],
threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],
- const bool float_mask SPECIALIZE,
- const bool has_mask SPECIALIZE,
const bool has_sinks SPECIALIZE,
- const bool do_causal SPECIALIZE,
+ const bool is_kv_cache_ring SPECIALIZE,
+ const bool is_causal SPECIALIZE,
+ const bool is_trie SPECIALIZE,
+ const bool is_sliding_window SPECIALIZE,
const ThreadContext thread_context,
const uint head_idx GROUPS(num_heads),
const uint q_seq_idx GROUPS(suffix_length),
const uint tid THREADS(1024)
) {
- const uint3 tpg = {num_heads, suffix_length, 1};
constexpr bool query_transposed = false;
constexpr uint value_dim = HEAD_DIM;
@@ -56,9 +61,17 @@ PUBLIC KERNEL(AttentionSinglePass)(
thread U o[value_elements_per_thread];
const uint kv_head_idx = head_idx / gqa_factor;
- const uint o_offset = q_seq_idx * tpg.x + head_idx;
- const uint q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx
- : head_idx * tpg.y + q_seq_idx;
+ const uint o_offset = q_seq_idx * num_heads + head_idx;
+ const uint q_offset = query_transposed ? num_heads * q_seq_idx + head_idx
+ : head_idx * suffix_length + q_seq_idx;
+
+ const uint prefix_length = sequence_length - suffix_length;
+
+ const uint suffix_position =
+ is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;
+
+ const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height
+ : suffix_position + q_seq_idx;
queries += q_offset * HEAD_DIM +
thread_context.simdgroup_index * qk_elements_per_thread;
@@ -69,12 +82,6 @@ PUBLIC KERNEL(AttentionSinglePass)(
thread_context.threadgroup_index * v_seq_stride +
thread_context.simdgroup_index * value_elements_per_thread;
- if (float_mask) {
- fmask += head_idx * mask_head_stride +
- thread_context.threadgroup_index * mask_kv_seq_stride +
- q_seq_idx * mask_q_seq_stride;
- }
-
out += o_offset * value_dim +
thread_context.threadgroup_index * value_elements_per_thread;
@@ -89,7 +96,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
U max_score = -INFINITY;
U sum_exp_score = 0;
if (has_sinks && thread_context.threadgroup_index == 0) {
- const int num_q_heads = static_cast<int>(tpg.x);
+ const int num_q_heads = static_cast<int>(num_heads);
int q_head_idx = head_idx % num_q_heads;
max_score = static_cast<U>(sinks[q_head_idx]);
sum_exp_score = 1;
@@ -98,12 +105,20 @@ PUBLIC KERNEL(AttentionSinglePass)(
// For each key
for (uint i = thread_context.threadgroup_index; i < sequence_length;
i += SEQUENCE_BLOCK_SIZE) {
- bool use_key = true;
- if (do_causal) {
- use_key = i <= (sequence_length - tpg.y + q_seq_idx);
- }
-
- if (use_key) {
+ if (should_use_key(
+ ring_params,
+ trie,
+ sliding_window_size,
+ q_seq_idx,
+ prefix_length,
+ suffix_position,
+ query_position,
+ i,
+ is_kv_cache_ring,
+ is_causal,
+ is_trie,
+ is_sliding_window
+ )) {
// Read the key
for (uint j = 0; j < qk_elements_per_thread; j++) {
k[j] = keys[j];
@@ -115,9 +130,6 @@ PUBLIC KERNEL(AttentionSinglePass)(
score += q[j] * k[j];
}
score = simd_sum(score);
- if (float_mask) {
- score += max(-1e9f, static_cast<U>(fmask[0]));
- }
// Update the accumulators
U new_max = max(max_score, score);
@@ -136,9 +148,6 @@ PUBLIC KERNEL(AttentionSinglePass)(
// Move the pointers to the next kv
keys += inner_k_stride;
values += inner_v_stride;
- if (float_mask) {
- fmask += SEQUENCE_BLOCK_SIZE * mask_kv_seq_stride;
- }
}
// Each thread has a partial part of the output so we need to combine them.
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
index 51d1b18dd..ab3cf52f8 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -2,12 +2,18 @@
#include <metal_simdgroup>
#include "../common/dsl.h"
#include "../common/thread_context.h"
+#include "../generated/ring.h"
+#include "../generated/trie.h"
+#include "mask.h"
#define HEAD_BLOCK_SIZE 32
#define SEQUENCE_BLOCK_SIZE_1 8
#define SEQUENCE_BLOCK_SIZE_2 32
#define TOTAL_BLOCKS_COUNT 32
+using namespace uzu::ring;
+using namespace uzu::trie;
+
template <typename T, uint HEAD_DIM>
VARIANTS(T, float, half, bfloat)
VARIANTS(HEAD_DIM, 64, 128, 256)
@@ -24,18 +30,18 @@ PUBLIC KERNEL(AttentionTwoPass1)(
const constant uint& k_seq_stride,
const constant uint& v_head_stride,
const constant uint& v_seq_stride,
+ const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),
const constant float& scale,
const constant uint& num_heads,
const constant uint& suffix_length,
- const device T* fmask OPTIONAL(float_mask),
- const constant uint& mask_kv_seq_stride OPTIONAL(has_mask),
- const constant uint& mask_q_seq_stride OPTIONAL(has_mask),
- const constant uint& mask_head_stride OPTIONAL(has_mask),
+ const device TrieNode* trie OPTIONAL(is_trie),
+ const constant uint& sliding_window_size OPTIONAL(is_sliding_window),
const device float* sinks OPTIONAL(has_sinks),
- const bool float_mask SPECIALIZE,
- const bool has_mask SPECIALIZE,
const bool has_sinks SPECIALIZE,
- const bool do_causal SPECIALIZE,
+ const bool is_kv_cache_ring SPECIALIZE,
+ const bool is_causal SPECIALIZE,
+ const bool is_trie SPECIALIZE,
+ const bool is_sliding_window SPECIALIZE,
threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE_1],
threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE_1],
threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_1 * HEAD_BLOCK_SIZE],
@@ -45,8 +51,6 @@ PUBLIC KERNEL(AttentionTwoPass1)(
const uint block_idx GROUPS(TOTAL_BLOCKS_COUNT),
const uint tid THREADS(256)
) {
- const uint3 tpg = {num_heads, suffix_length, TOTAL_BLOCKS_COUNT};
-
constexpr bool query_transposed = false;
constexpr uint value_dim = HEAD_DIM;
constexpr uint qk_elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;
@@ -60,13 +64,21 @@ PUBLIC KERNEL(AttentionTwoPass1)(
thread U k[qk_elements_per_thread];
thread U o[value_elements_per_thread];
- const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
+ const uint o_offset = q_seq_idx * num_heads + head_idx; // Our custom layout
const uint q_offset =
query_transposed
- ? tpg.x * q_seq_idx + head_idx
- : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass
+ ? num_heads * q_seq_idx + head_idx
+ : head_idx * suffix_length + q_seq_idx; // Consistent with single-pass
const uint kv_head_idx = head_idx / gqa_factor;
+ const uint prefix_length = sequence_length - suffix_length;
+
+ const uint suffix_position =
+ is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;
+
+ const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height
+ : suffix_position + q_seq_idx;
+
queries += q_offset * HEAD_DIM +
thread_context.simdgroup_index * qk_elements_per_thread;
keys +=
@@ -81,13 +93,6 @@ PUBLIC KERNEL(AttentionTwoPass1)(
thread_context.simdgroup_index * value_elements_per_thread;
out += o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim +
thread_context.simdgroup_index * value_elements_per_thread;
- if (float_mask) {
- fmask +=
- head_idx * mask_head_stride +
- (block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index) *
- mask_kv_seq_stride +
- q_seq_idx * mask_q_seq_stride;
- }
sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
maxs += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
@@ -102,7 +107,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(
U max_score = -1e9;
U sum_exp_score = 0;
if (has_sinks && block_idx == 0 && thread_context.threadgroup_index == 0) {
- const uint num_q_heads = tpg.x;
+ const uint num_q_heads = num_heads;
int q_head_idx = head_idx % num_q_heads;
max_score = static_cast<U>(sinks[q_head_idx]);
sum_exp_score = 1;
@@ -113,12 +118,20 @@ PUBLIC KERNEL(AttentionTwoPass1)(
block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index;
i < sequence_length;
i += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1) {
- bool use_key = true;
- if (do_causal) {
- use_key = i <= (sequence_length - tpg.y + q_seq_idx);
- }
-
- if (use_key) {
+ if (should_use_key(
+ ring_params,
+ trie,
+ sliding_window_size,
+ q_seq_idx,
+ prefix_length,
+ suffix_position,
+ query_position,
+ i,
+ is_kv_cache_ring,
+ is_causal,
+ is_trie,
+ is_sliding_window
+ )) {
// Read the key
for (uint j = 0; j < qk_elements_per_thread; j++) {
k[j] = keys[j];
@@ -130,9 +143,6 @@ PUBLIC KERNEL(AttentionTwoPass1)(
score += q[j] * k[j];
}
score = simd_sum(score);
- if (float_mask) {
- score += max(-1e9f, static_cast<U>(fmask[0]));
- }
// Update the accumulators
U new_max = max(max_score, score);
@@ -151,9 +161,6 @@ PUBLIC KERNEL(AttentionTwoPass1)(
// Move the pointers to the next kv
keys += TOTAL_BLOCKS_COUNT * inner_k_stride;
values += TOTAL_BLOCKS_COUNT * inner_v_stride;
- if (float_mask) {
- fmask += SEQUENCE_BLOCK_SIZE_1 * TOTAL_BLOCKS_COUNT * mask_kv_seq_stride;
- }
}
// Each thread has a partial part of the output so we need to combine them.
@@ -219,13 +226,12 @@ PUBLIC KERNEL(AttentionTwoPass2)(
const uint q_seq_idx GROUPS(suffix_length),
const uint tid THREADS(1024)
) {
- const uint3 tpg = {num_heads, suffix_length, 1};
constexpr uint elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;
typedef float U;
thread U o[elements_per_thread];
- const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
+ const uint o_offset = q_seq_idx * num_heads + head_idx; // Our custom layout
partials += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +
thread_context.threadgroup_index * HEAD_DIM +
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mask.h b/crates/uzu/src/backends/metal/kernel/attention/mask.h
new file mode 100644
index 000000000..83b696dc1
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/attention/mask.h
@@ -0,0 +1,22 @@
+#pragma once
+
+#include "../generated/ring.h"
+#include "../generated/trie.h"
+
+using namespace uzu::ring;
+using namespace uzu::trie;
+
+bool should_use_key(
+ const constant RingParams& ring_params,
+ const device TrieNode* trie,
+ const constant uint& sliding_window_size,
+ const uint q_seq_idx,
+ const uint prefix_length,
+ const uint suffix_position,
+ const uint query_position,
+ const uint i,
+ const bool is_kv_cache_ring,
+ const bool is_causal,
+ const bool is_trie,
+ const bool is_sliding_window
+);
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mask.metal b/crates/uzu/src/backends/metal/kernel/attention/mask.metal
new file mode 100644
index 000000000..b3260231c
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/attention/mask.metal
@@ -0,0 +1,76 @@
+#include "../generated/ring.h"
+#include "../generated/trie.h"
+
+using namespace uzu::ring;
+using namespace uzu::trie;
+
+bool should_use_key(
+ const constant RingParams& ring_params,
+ const device TrieNode* trie,
+ const constant uint& sliding_window_size,
+ const uint q_seq_idx,
+ const uint prefix_length,
+ const uint suffix_position,
+ const uint query_position,
+ const uint i,
+ const bool is_kv_cache_ring,
+ const bool is_causal,
+ const bool is_trie,
+ const bool is_sliding_window
+) {
+ bool use_key = true;
+
+ uint key_position;
+
+ if (i >= prefix_length) {
+ // We're in the suffix
+ uint key_position_in_suffix = i - prefix_length;
+
+ if (is_trie) {
+ // Trie, get position from the flat trie buffer
+ TrieNode key_trie = trie[key_position_in_suffix];
+
+ key_position = suffix_position + key_trie.height;
+
+ if (is_causal) {
+ use_key &=
+ q_seq_idx >= key_trie.trie_start && q_seq_idx <= key_trie.trie_end;
+ }
+ } else {
+ // Non-trie, position equals index in the suffix
+ key_position = suffix_position + key_position_in_suffix;
+
+ if (is_causal) {
+ use_key &= key_position_in_suffix <= q_seq_idx;
+ }
+ }
+
+ } else {
+ // We're in the prefix
+ if (is_kv_cache_ring) {
+ // Ring, calculate position in the ring
+ key_position =
+ (prefix_length + i - ring_params.ring_offset) % prefix_length;
+ // Ring also skips entries that aren't filled yet
+ use_key &= key_position < ring_params.ring_length;
+ } else {
+ // Non-ring, position equals to the index in the kvcache buffer
+ key_position = i;
+ }
+ }
+
+ if (is_sliding_window) {
+ if (is_causal) {
+ use_key &= key_position <= query_position &&
+ (query_position - key_position) < sliding_window_size;
+ } else {
+ if (key_position <= query_position) {
+ use_key &= (query_position - key_position) <= (sliding_window_size / 2);
+ } else {
+ use_key &= (key_position - query_position) <= (sliding_window_size / 2);
+ }
+ }
+ }
+
+ return use_key;
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
deleted file mode 100644
index 06a9d52f2..000000000
--- a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
+++ /dev/null
@@ -1,17 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(MaskUpdate) (
- device T* mask,
- constant int& unmask_col,
- constant int& mask_col
-) {
- if (unmask_col >= 0) {
- mask[unmask_col] = T(0);
- }
- if (mask_col >= 0) {
- mask[mask_col] = -numeric_limits<T>::infinity();
- }
-}
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index f8c34c890..ddf9a6997 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -104,7 +104,6 @@ impl<B: Backend> Classifier<B> {
self.context.shared_buffers.clone(),
token_ids,
token_positions,
- true,
num_labels,
);
@@ -157,7 +156,6 @@ impl<B: Backend> Classifier<B> {
self.context.shared_buffers.clone(),
token_ids,
token_positions,
- true,
num_labels,
);
diff --git a/crates/uzu/src/classifier/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rs
index 5ef7421b3..2fe163e94 100644
--- a/crates/uzu/src/classifier/classifier_context.rs
+++ b/crates/uzu/src/classifier/classifier_context.rs
@@ -161,8 +161,7 @@ impl<B: Backend> ClassifierContext<B> {
.map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("output norm: {:?}", e))))?;
let context_length = classifier_model_config.model_config.context_length;
- let scratch_buffers =
- ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, context_length, context_length);
+ let scratch_buffers = ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, context_length);
let embedding_norm_tree = root_loader_view
.subtree("embedding_norm")
diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rs
index 87f65dfb0..e14a249d6 100644
--- a/crates/uzu/src/encodable_block/attention.rs
+++ b/crates/uzu/src/encodable_block/attention.rs
@@ -5,10 +5,13 @@ use std::{
ops::{Deref, DerefMut},
};
+use itertools::iproduct;
+
use crate::{
DataType,
backends::common::{
Backend, Encoder, Kernels,
+ gpu_types::ring::RingParams,
kernel::{
AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,
SigmoidGateKernel,
@@ -16,7 +19,10 @@ use crate::{
},
},
encodable_block::EncodingParameters,
- forward_pass::state::{ArrayId, ForwardPassState, HashMapId},
+ forward_pass::{
+ kv_cache_layer::KVCacheLayerState,
+ state::{ArrayId, ForwardPassState},
+ },
};
fn env_gemm_attention_enabled() -> bool {
@@ -65,28 +71,39 @@ impl<B: Backend> Attention<B> {
let mut two_pass_1_kernels = HashMap::new();
let mut two_pass_2_kernels = HashMap::new();
- let supported_head_dims = [64u32, 128u32, 256u32];
- for head_dim in supported_head_dims {
- for has_mask in [false, true] {
- let key = KernelKey {
- head_dim,
- has_mask,
- };
- let float_mask = has_mask;
-
- let sp_kernel = <B::Kernels as Kernels>::AttentionSinglePassKernel::new(
- context, data_type, head_dim, float_mask, has_mask, has_sinks, is_causal,
- )?;
- single_pass_kernels.insert(key, sp_kernel);
-
- let tp1_kernel = <B::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
- context, data_type, head_dim, float_mask, has_mask, has_sinks, is_causal,
- )?;
- two_pass_1_kernels.insert(key, tp1_kernel);
-
- let tp2_kernel = <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, data_type, head_dim)?;
- two_pass_2_kernels.insert(head_dim, tp2_kernel);
- }
+ for (head_dim, is_trie, is_kv_cache_ring) in iproduct!([64u32, 128u32, 256u32], [false, true], [false, true]) {
+ let key = KernelKey {
+ head_dim,
+ is_trie,
+ is_kv_cache_ring,
+ };
+
+ let sp_kernel = <B::Kernels as Kernels>::AttentionSinglePassKernel::new(
+ context,
+ data_type,
+ head_dim,
+ has_sinks,
+ is_kv_cache_ring,
+ is_causal,
+ is_trie,
+ sliding_window_size.is_some(),
+ )?;
+ single_pass_kernels.insert(key, sp_kernel);
+
+ let tp1_kernel = <B::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
+ context,
+ data_type,
+ head_dim,
+ has_sinks,
+ is_kv_cache_ring,
+ is_causal,
+ is_trie,
+ sliding_window_size.is_some(),
+ )?;
+ two_pass_1_kernels.insert(key, tp1_kernel);
+
+ let tp2_kernel = <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, data_type, head_dim)?;
+ two_pass_2_kernels.insert(head_dim, tp2_kernel);
}
let update_kv_cache_kernel =
@@ -122,7 +139,8 @@ impl<B: Backend> Attention<B> {
suffix_length: usize,
head_dim: usize,
sequence_length: usize,
- use_mask: bool,
+ is_trie: bool,
+ is_kv_cache_ring: bool,
) -> KernelVariant {
let use_gemm = gemm_enabled && suffix_length > 8 && matches!(head_dim, 64 | 128 | 256);
if use_gemm {
@@ -131,7 +149,8 @@ impl<B: Backend> Attention<B> {
let kernel_key = KernelKey {
head_dim: head_dim as u32,
- has_mask: use_mask,
+ is_trie,
+ is_kv_cache_ring,
};
if sequence_length > 1024
&& self.two_pass_1_kernels.contains_key(&kernel_key)
@@ -168,18 +187,34 @@ impl<B: Backend> Attention<B> {
suffix_length
};
- let (segment_prefix_length, window_length) = if let Some(cache_layers) = state.cache_layers() {
+ let is_trie = state.token_subtrie_ranges.is_some();
+
+ let (segment_prefix_length, ring_params) = if let Some(cache_layers) = state.cache_layers() {
+ let projection_step = parameters.projection_step.unwrap_or(0);
let cache = cache_layers.borrow();
let layer = cache.data[self.layer_index]
.as_transformer()
.expect("Attention kernel expects transformer layer state");
- (layer.projected_segment_prefix_length(parameters.projection_step.unwrap_or(0)), layer.window_length())
+ let ring_params = match layer.state {
+ KVCacheLayerState::Windowed {
+ ring_offset,
+ ring_length,
+ window_length,
+ } => {
+ let overflow = (ring_length + projection_step).saturating_sub(window_length);
+ Some(RingParams {
+ ring_offset: ((ring_offset + overflow) % window_length) as u32,
+ ring_length: (ring_length + projection_step).min(window_length) as u32,
+ })
+ },
+ _ => None,
+ };
+ (layer.projected_segment_prefix_length(projection_step), ring_params)
} else {
- // For classifiers without KV cache: no prefix, use configured sliding window
- (0, self.sliding_window_size)
+ (0, None)
};
- let use_mask = window_length.is_some();
+ let is_kv_cache_ring = ring_params.is_some();
let sequence_length = segment_prefix_length + suffix_length;
@@ -193,14 +228,11 @@ impl<B: Backend> Attention<B> {
eprintln!("[uzu] Gemm attention disabled via UZU_USE_GEMM_ATTENTION");
});
}
- let variant = self.select_variant(gemm_enabled, suffix_length, head_dim, sequence_length, use_mask);
+ let variant =
+ self.select_variant(gemm_enabled, suffix_length, head_dim, sequence_length, is_trie, is_kv_cache_ring);
- let attention_bias_binding = state.hashmaps(&[HashMapId::AttentionBias]);
let gate_array = self.gate_kernel.as_ref().map(|_| state.array(ArrayId::Gate));
- let mask_kv_seq_stride = 1;
- let mask_q_seq_stride = sequence_length as i32;
- let mask_head_stride = 0;
let sinks_array = self.has_sinks.then(|| state.array(ArrayId::AttentionSinks(self.layer_index)));
let rotated_keys_buf_rc = rotated_keys_array.buffer();
@@ -248,18 +280,9 @@ impl<B: Backend> Attention<B> {
let attention_output_buf_rc = attention_output_array.buffer();
let mut attention_output_buf_borrow = attention_output_buf_rc.borrow_mut();
- let attention_bias_array_borrow = if use_mask {
- Some(
- attention_bias_binding[0]
- .get(&window_length)
- .unwrap_or_else(|| panic!("Attention bias buffer not found for window length {:?}", window_length)),
- )
- } else {
- None
- };
- let attention_bias_buf_rc = attention_bias_array_borrow.as_ref().map(|a| a.buffer());
- let attention_bias_buf_borrow = attention_bias_buf_rc.as_ref().map(|rc| rc.borrow());
- let attention_bias_buffer: Option<&B::Buffer> = attention_bias_buf_borrow.as_ref().map(|b| b.deref());
+ let trie_buf_rc = state.token_subtrie_ranges.as_ref().map(|a| a.buffer());
+ let trie_buf_borrow = trie_buf_rc.as_ref().map(|rc| rc.borrow());
+ let trie_buffer: Option<&B::Buffer> = trie_buf_borrow.as_ref().map(|b| b.deref());
let partials_buf_rc = partials_array.buffer();
let mut partials_buf_borrow = partials_buf_rc.borrow_mut();
@@ -309,18 +332,10 @@ impl<B: Backend> Attention<B> {
let kernel_key = KernelKey {
head_dim: head_dim as u32,
- has_mask: use_mask,
+ is_trie,
+ is_kv_cache_ring,
};
- let mut mask_kv_seq_stride_opt: Option<u32> = None;
- let mut mask_q_seq_stride_opt: Option<u32> = None;
- let mut mask_head_stride_opt: Option<u32> = None;
- if attention_bias_buffer.is_some() {
- mask_kv_seq_stride_opt = Some(mask_kv_seq_stride as u32);
- mask_q_seq_stride_opt = Some(mask_q_seq_stride as u32);
- mask_head_stride_opt = Some(mask_head_stride as u32);
- }
-
match variant {
KernelVariant::Gemm => {
let args = AttentionGemmArguments {
@@ -328,7 +343,7 @@ impl<B: Backend> Attention<B> {
keys_buffer: key_cache_buffer,
values_buffer: value_cache_buffer,
output_buffer: attention_output_buf_borrow.deref_mut(),
- mask_buffer: attention_bias_buffer,
+ trie_buffer,
sinks_buffer,
num_heads,
num_groups,
@@ -336,8 +351,10 @@ impl<B: Backend> Attention<B> {
sequence_length,
segment_prefix_length,
max_sequence_length,
+ ring_params,
head_dim,
is_causal: self.is_causal,
+ sliding_window_size: self.sliding_window_size,
scale,
};
self.gemm_block.encode(state.context(), encoder, args).expect("Failed to encode AttentionGemmBlock");
@@ -358,11 +375,10 @@ impl<B: Backend> Attention<B> {
k_seq_stride as u32,
v_head_stride as u32,
v_seq_stride as u32,
+ ring_params,
scale,
- attention_bias_buffer,
- mask_kv_seq_stride_opt,
- mask_q_seq_stride_opt,
- mask_head_stride_opt,
+ trie_buffer,
+ self.sliding_window_size.map(|s| s as u32),
sinks_buffer,
num_heads as u32,
suffix_length as u32,
@@ -391,13 +407,12 @@ impl<B: Backend> Attention<B> {
k_seq_stride as u32,
v_head_stride as u32,
v_seq_stride as u32,
+ ring_params,
scale,
num_heads as u32,
suffix_length as u32,
- attention_bias_buffer,
- mask_kv_seq_stride_opt,
- mask_q_seq_stride_opt,
- mask_head_stride_opt,
+ trie_buffer,
+ self.sliding_window_size.map(|s| s as u32),
sinks_buffer,
encoder,
);
@@ -439,7 +454,8 @@ enum KernelVariant {
#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)]
struct KernelKey {
pub head_dim: u32,
- pub has_mask: bool,
+ pub is_trie: bool,
+ pub is_kv_cache_ring: bool,
}
#[cfg(test)]
diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rs
index 5b08a20ad..38abcabac 100644
--- a/crates/uzu/src/forward_pass/cache_layers.rs
+++ b/crates/uzu/src/forward_pass/cache_layers.rs
@@ -1,11 +1,11 @@
-use std::{cell::Cell, collections::HashMap};
+use std::cell::Cell;
use crate::{
- array::{Array, ArrayContextExt},
+ array::ArrayContextExt,
backends::common::{Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},
config::DecoderLayerType,
forward_pass::{
- kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState, KVSlice},
+ kv_cache_layer::{KVCacheLayer, KVCacheLayerState, KVSlice},
model_shape::ModelShape,
short_conv_layer::ShortConvLayer,
ssm_layer::SSMLayer,
@@ -133,16 +133,6 @@ impl<B: Backend> CacheLayers<B> {
model_shape.kv_cache_data_type(),
&format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),
),
- prefix_token_positions: match &state {
- KVCacheLayerState::Full {
- ..
- } => Vec::with_capacity(max_prefix_length),
- KVCacheLayerState::Windowed {
- window_length,
- ..
- } => (0..*window_length).map(|_| INVALID_POSITION).collect(),
- },
- max_suffix_length,
})
},
DecoderLayerType::StateSpace {
@@ -214,7 +204,6 @@ impl<B: Backend> CacheLayers<B> {
prefix_len,
} => {
*prefix_len = 0;
- layer.prefix_token_positions.clear();
},
KVCacheLayerState::Windowed {
ring_offset,
@@ -223,7 +212,6 @@ impl<B: Backend> CacheLayers<B> {
} => {
*ring_offset = 0;
*ring_length = 0;
- layer.prefix_token_positions.fill(INVALID_POSITION);
},
},
CacheLayer::StateSpace(layer) => layer.zero(),
@@ -240,38 +228,6 @@ impl<B: Backend> CacheLayers<B> {
self.max_prefix_length
}
- pub fn fill_attention_bias(
- &self,
- dst: &mut HashMap<Option<usize>, Array<B>>,
- suffix_token_positions: &[usize],
- suffix_length: usize,
- external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
- ) {
- for layer in self.data.iter() {
- if let CacheLayer::Transformer(layer) = layer {
- if let Some(array) = dst.get_mut(&layer.window_length()) {
- layer.fill_attention_bias(array, suffix_token_positions, suffix_length, external_bias_fn);
- }
- }
- }
- }
-
- pub fn fill_attention_bias_scratch(
- &self,
- dst: &mut HashMap<Option<usize>, Array<B>>,
- suffix_token_positions: &[usize],
- suffix_length: usize,
- _context: &B::Context,
- ) {
- for layer in self.data.iter() {
- if let CacheLayer::Transformer(layer) = layer {
- if let Some(array) = dst.get_mut(&layer.window_length()) {
- layer.fill_attention_bias(array, suffix_token_positions, suffix_length, None);
- }
- }
- }
- }
-
pub fn update_after_acceptance(
&mut self,
accepted_suffix_indices: &[usize],
@@ -291,28 +247,15 @@ impl<B: Backend> CacheLayers<B> {
pub fn register_accepted_tokens(
&mut self,
- token_positions: &[usize],
+ number_of_accepted_tokens: usize,
) {
for layer in self.data.iter_mut() {
if let Some(layer) = layer.as_transformer_mut() {
- layer.register_accepted_tokens(token_positions);
+ layer.register_accepted_tokens(number_of_accepted_tokens);
}
}
}
- pub fn attention_bias_updates_after_acceptance(
- &self,
- accepted_len: usize,
- ) -> Vec<AttentionBiasUpdate> {
- self.data
- .iter()
- .filter_map(|layer| match layer {
- CacheLayer::Transformer(kv) => kv.attention_bias_update_after_acceptance(accepted_len),
- _ => None,
- })
- .collect()
- }
-
pub fn slice(
&self,
context: &B::Context,
@@ -397,8 +340,6 @@ impl<B: Backend> CacheLayers<B> {
state: layer.state.clone(),
keys: new_keys,
values: new_values,
- prefix_token_positions: layer.prefix_token_positions.clone(),
- max_suffix_length: layer.max_suffix_length,
})
},
CacheLayer::StateSpace(layer) => {
diff --git a/crates/uzu/src/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rs
index b31ca92c3..8901e996b 100644
--- a/crates/uzu/src/forward_pass/kv_cache_layer.rs
+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs
@@ -4,31 +4,20 @@ use crate::{
Backend, Encoder,
kernel::kv_cache_update::{KVCacheUpdate, KVLayerData},
},
- utils::attention::fill_attention_bias,
};
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub struct AttentionBiasUpdate {
- pub key: Option<usize>,
- pub unmask_col: i32,
- pub mask_col: i32,
-}
-
#[derive(Clone)]
pub enum KVSlice<B: Backend> {
Full {
base_prefix_len: usize,
- base_positions_len: usize,
- positions: Vec<usize>,
},
Window {
window_length: usize,
base_ring_offset: usize,
base_ring_length: usize,
slots: Vec<usize>,
- positions: Vec<usize>, // per slot
- keys: Array<B>, // [num_groups, slots.len(), head_dim]
- values: Array<B>, // [num_groups, slots.len(), head_dim]
+ keys: Array<B>, // [num_groups, slots.len(), head_dim]
+ values: Array<B>, // [num_groups, slots.len(), head_dim]
},
}
@@ -43,6 +32,7 @@ pub enum KVCacheLayerState {
ring_offset: usize,
/// Current logical length of the window (<= window_length)
ring_length: usize,
+ /// Maximum length of the window
window_length: usize,
},
}
@@ -56,9 +46,6 @@ pub struct KVCacheLayer<B: Backend> {
pub keys: Array<B>,
/// [num_groups, max_prefix_length + max_suffix_length, head_dim]
pub values: Array<B>,
-
- pub prefix_token_positions: Vec<usize>,
- pub max_suffix_length: usize,
}
impl<B: Backend> KVCacheLayer<B> {
@@ -101,64 +88,6 @@ impl<B: Backend> KVCacheLayer<B> {
}
}
- pub fn fill_attention_bias(
- &self,
- dst: &mut Array<B>,
- suffix_token_positions: &[usize],
- suffix_length: usize,
- external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
- ) {
- let prefix_segment_length = self.prefix_segment_length();
- fill_attention_bias(dst, suffix_length, prefix_segment_length, |row_index, column_index| {
- if let Some(bias_fn) = external_bias_fn {
- bias_fn(row_index, column_index)
- } else {
- self.bias_should_be_neg_inf(row_index, column_index, suffix_token_positions)
- }
- });
- }
-
- pub fn bias_should_be_neg_inf(
- &self,
- row_index: usize,
- column_index: usize,
- suffix_token_positions: &[usize],
- ) -> bool {
- let query_position = suffix_token_positions[row_index];
- if query_position == INVALID_POSITION {
- return true;
- }
-
- let key_position = if column_index >= self.prefix_segment_length() {
- suffix_token_positions[column_index - self.prefix_segment_length()]
- } else {
- match &self.state {
- KVCacheLayerState::Full {
- ..
- } => column_index,
- KVCacheLayerState::Windowed {
- ..
- } => self.prefix_token_positions[column_index],
- }
- };
-
- if key_position == INVALID_POSITION {
- return true;
- }
-
- if query_position < key_position {
- return true;
- }
-
- match &self.state {
- KVCacheLayerState::Windowed {
- window_length,
- ..
- } => query_position >= key_position + window_length,
- _ => false,
- }
- }
-
pub fn update_after_acceptance(
&mut self,
accepted_suffix_indices: &[usize],
@@ -235,64 +164,22 @@ impl<B: Backend> KVCacheLayer<B> {
pub fn register_accepted_tokens(
&mut self,
- token_positions: &[usize],
+ number_of_accepted_tokens: usize,
) {
match &mut self.state {
KVCacheLayerState::Full {
prefix_len,
} => {
- self.prefix_token_positions.extend_from_slice(token_positions);
- *prefix_len = self.prefix_token_positions.len();
+ *prefix_len += number_of_accepted_tokens;
},
KVCacheLayerState::Windowed {
ring_offset,
ring_length,
window_length,
} => {
- for &token_pos in token_positions {
- if *ring_length < *window_length {
- let dst = (*ring_offset + *ring_length) % *window_length;
-
- self.prefix_token_positions[dst] = token_pos;
- *ring_length += 1;
- } else {
- self.prefix_token_positions[*ring_offset] = token_pos;
-
- *ring_offset = (*ring_offset + 1) % *window_length;
- }
- }
- },
- }
- }
-
- pub fn attention_bias_update_after_acceptance(
- &self,
- accepted_len: usize,
- ) -> Option<AttentionBiasUpdate> {
- if accepted_len != 1 {
- return None;
- }
-
- match self.state {
- KVCacheLayerState::Full {
- ..
- } => None,
- KVCacheLayerState::Windowed {
- ring_offset,
- ring_length,
- window_length,
- } => {
- let newest_slot = (ring_length > 0)
- .then_some((ring_offset + ring_length + window_length - 1) % window_length)
- .unwrap_or(0);
- let unmask_col = (ring_length > 0).then_some(newest_slot as i32).unwrap_or(-1);
- let mask_col = (ring_length == window_length).then_some(ring_offset as i32).unwrap_or(-1);
-
- Some(AttentionBiasUpdate {
- key: Some(window_length),
- unmask_col,
- mask_col,
- })
+ let ring_advance = number_of_accepted_tokens.saturating_sub(*window_length - *ring_length);
+ *ring_offset = (*ring_offset + ring_advance) % *window_length;
+ *ring_length = (*ring_length + number_of_accepted_tokens).min(*window_length);
},
}
}
@@ -307,8 +194,6 @@ impl<B: Backend> KVCacheLayer<B> {
prefix_len,
} => Some(KVSlice::Full {
base_prefix_len: prefix_len,
- base_positions_len: self.prefix_token_positions.len(),
- positions: self.prefix_token_positions.clone(),
}),
KVCacheLayerState::Windowed {
ring_offset,
@@ -342,8 +227,6 @@ impl<B: Backend> KVCacheLayer<B> {
})
.collect();
- let positions: Vec<usize> = slots.iter().map(|&s| self.prefix_token_positions[s]).collect();
-
for (i, &slot) in slots.iter().enumerate() {
slice_keys.copy_slice(&self.keys, 1, slot..slot + 1, i);
slice_values.copy_slice(&self.values, 1, slot..slot + 1, i);
@@ -354,7 +237,6 @@ impl<B: Backend> KVCacheLayer<B> {
base_ring_offset: ring_offset,
base_ring_length: ring_length,
slots,
- positions,
keys: slice_keys,
values: slice_values,
})
@@ -371,8 +253,6 @@ impl<B: Backend> KVCacheLayer<B> {
(
KVSlice::Full {
base_prefix_len,
- base_positions_len,
- positions,
},
KVCacheLayerState::Full {
prefix_len,
@@ -380,14 +260,9 @@ impl<B: Backend> KVCacheLayer<B> {
) => match range {
None => {
*prefix_len = *base_prefix_len;
- self.prefix_token_positions.clone_from(positions);
- self.prefix_token_positions.truncate(*base_positions_len);
},
Some(r) => {
- let accepted = r.start;
- *prefix_len = base_prefix_len.saturating_add(accepted);
- let keep_positions = base_positions_len.saturating_add(accepted);
- self.prefix_token_positions.truncate(keep_positions);
+ *prefix_len = base_prefix_len.saturating_add(r.start);
},
},
(
@@ -396,7 +271,6 @@ impl<B: Backend> KVCacheLayer<B> {
base_ring_offset,
base_ring_length,
slots,
- positions,
keys,
values,
},
@@ -412,10 +286,6 @@ impl<B: Backend> KVCacheLayer<B> {
*ring_offset = *base_ring_offset;
*ring_length = *base_ring_length;
- for (i, &slot) in slots.iter().enumerate() {
- self.prefix_token_positions[slot] = positions[i];
- }
-
for (i, &slot) in slots.iter().enumerate() {
self.keys.copy_slice(keys, 1, i..i + 1, slot);
self.values.copy_slice(values, 1, i..i + 1, slot);
@@ -448,7 +318,6 @@ impl<B: Backend> KVCacheLayer<B> {
for (i, &slot) in slots[r.clone()].iter().enumerate() {
let src_i = r.start + i;
- self.prefix_token_positions[slot] = positions[src_i];
self.keys.copy_slice(keys, 1, src_i..src_i + 1, slot);
self.values.copy_slice(values, 1, src_i..src_i + 1, slot);
}
diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rs
index da82e1408..789e1c252 100644
--- a/crates/uzu/src/forward_pass/model_shape.rs
+++ b/crates/uzu/src/forward_pass/model_shape.rs
@@ -169,6 +169,13 @@ impl ModelShape {
[suffix_length, self.model_dim]
}
+ pub fn subtrie_ranges_shape(
+ &self,
+ suffix_length: usize,
+ ) -> [usize; 2] {
+ [suffix_length, 3]
+ }
+
pub fn bitmask_shape(
&self,
suffix_length: usize,
diff --git a/crates/uzu/src/forward_pass/scratch_buffers.rs b/crates/uzu/src/forward_pass/scratch_buffers.rs
index 3f2cd6b25..c36e7a4a2 100644
--- a/crates/uzu/src/forward_pass/scratch_buffers.rs
+++ b/crates/uzu/src/forward_pass/scratch_buffers.rs
@@ -1,5 +1,3 @@
-use std::collections::{HashMap, HashSet};
-
use crate::{
DataType,
array::{Array, ArrayContextExt},
@@ -11,6 +9,7 @@ use crate::{
pub struct ScratchBuffers<B: Backend> {
// 1-D
pub token_ids: Array<B>,
+ pub token_subtrie_ranges: Array<B>,
pub token_positions: Array<B>,
pub token_parents: Array<B>,
pub token_bitmask: Array<B>,
@@ -18,7 +17,6 @@ pub struct ScratchBuffers<B: Backend> {
pub sampling_output: Array<B>,
// 2-D
- pub attention_window_size_to_bias: HashMap<Option<usize>, Array<B>>,
pub logits: Array<B>,
pub main: Array<B>,
pub shortcut: Array<B>,
@@ -72,7 +70,6 @@ impl<B: Backend> ScratchBuffers<B> {
context: &B::Context,
decoder_config: &DecoderConfig,
model_shape: &ModelShape,
- max_prefix_len: usize,
max_suffix_len: usize,
) -> Self {
// Helper closure for allocation
@@ -97,6 +94,11 @@ impl<B: Backend> ScratchBuffers<B> {
Self {
// 1-D
token_ids: alloc(&[max_suffix_len], DataType::U64, "token_ids"),
+ token_subtrie_ranges: alloc(
+ &model_shape.subtrie_ranges_shape(max_suffix_len),
+ DataType::U32,
+ "token_subtrie_ranges",
+ ),
token_positions: alloc(&[max_suffix_len], DataType::I32, "token_positions"),
token_parents: alloc(&[max_suffix_len], DataType::I32, "token_parents"),
token_bitmask: alloc(&model_shape.bitmask_shape(max_suffix_len), DataType::U32, "token_bitmask"),
@@ -104,23 +106,6 @@ impl<B: Backend> ScratchBuffers<B> {
sampling_output: alloc(&[max_suffix_len], DataType::U32, "sampling_output"),
// 2-D
- attention_window_size_to_bias: model_shape
- .sliding_window_length_per_layer
- .iter()
- .copied()
- .collect::<HashSet<_>>()
- .into_iter()
- .map(|window_size| {
- let label = match window_size {
- Some(ws) => {
- format!("attention_bias_for_window_size_{ws}")
- },
- None => "attention_bias_for_window_size_none".to_string(),
- };
- let shape = [max_suffix_len, max_suffix_len + max_prefix_len];
- (window_size, alloc(&shape, act_ty, &label))
- })
- .collect(),
logits: alloc(&model_shape.logits_shape(max_suffix_len), act_ty, "logits"),
main: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "main"),
shortcut: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "shortcut"),
diff --git a/crates/uzu/src/forward_pass/state/hash_map_id.rs b/crates/uzu/src/forward_pass/state/hash_map_id.rs
deleted file mode 100644
index 4d9231e7e..000000000
--- a/crates/uzu/src/forward_pass/state/hash_map_id.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum HashMapId {
- AttentionBias,
-}
diff --git a/crates/uzu/src/forward_pass/state/mod.rs b/crates/uzu/src/forward_pass/state/mod.rs
index d61b747a6..531093517 100644
--- a/crates/uzu/src/forward_pass/state/mod.rs
+++ b/crates/uzu/src/forward_pass/state/mod.rs
@@ -1,5 +1,4 @@
mod array_id;
-mod hash_map_id;
mod rope_type;
mod common_aux_buffers;
@@ -11,7 +10,6 @@ mod state;
pub use array_id::ArrayId;
pub use common_aux_buffers::CommonAuxBuffers;
-pub use hash_map_id::HashMapId;
pub use language_model_generator_aux_buffers::LanguageModelGeneratorAuxBuffers;
pub use rope_buffers::RopeBuffers;
pub use rope_type::RopeType;
diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rs
index 1318e2570..e2cf57288 100644
--- a/crates/uzu/src/forward_pass/state/state.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -1,6 +1,8 @@
#[cfg(feature = "tracing")]
use std::ops::{Deref, DerefMut};
-use std::{cell::RefCell, collections::HashMap, rc::Rc};
+use std::{cell::RefCell, rc::Rc};
+
+use ndarray::ArrayView2;
#[cfg(feature = "tracing")]
use crate::backends::common::Encoder;
@@ -15,10 +17,9 @@ use crate::{
cache_layers::CacheLayers,
model_shape::ModelShape,
scratch_buffers::ScratchBuffers,
- state::{ArrayId, CommonAuxBuffers, HashMapId, LanguageModelGeneratorAuxBuffers, RopeType, SharedBuffers},
+ state::{ArrayId, CommonAuxBuffers, LanguageModelGeneratorAuxBuffers, RopeType, SharedBuffers},
},
session::parameter::SamplingMethod,
- utils::attention::fill_attention_bias,
};
pub enum ForwardPassMode<B: Backend> {
@@ -52,10 +53,10 @@ pub struct ClassifierModeState<B: Backend> {
pub struct ForwardPassState<B: Backend> {
context: Rc<B::Context>,
token_ids: Array<B>,
+ pub token_subtrie_ranges: Option<Array<B>>,
token_positions: Array<B>,
token_parents: Array<B>,
token_bitmask: Option<Array<B>>,
- attention_bias: HashMap<Option<usize>, Array<B>>,
pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
common_aux: CommonAuxBuffers<B>,
llm_aux: Option<LanguageModelGeneratorAuxBuffers<B>>,
@@ -146,6 +147,7 @@ impl<B: Backend> ForwardPassState<B> {
cache_layers: Rc<RefCell<CacheLayers<B>>>,
shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
token_ids: &[u64],
+ token_subtrie_ranges: Option<&[[u32; 3]]>,
token_positions: &[usize],
token_bitmask: Option<&[u32]>,
token_seeds: &[u64],
@@ -153,9 +155,7 @@ impl<B: Backend> ForwardPassState<B> {
sampling_start: usize,
sampling_length: usize,
is_prefilling: bool,
- external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
skip_token_ids_copy: bool,
- should_fill_attention_bias: bool,
async_positions: Option<(Rc<RefCell<B::Buffer>>, usize)>,
async_seeds: Option<(Rc<RefCell<B::Buffer>>, usize)>,
) -> Self {
@@ -170,6 +170,18 @@ impl<B: Backend> ForwardPassState<B> {
Self::init_token_ids(scratch, token_ids)
};
+ // Token subtrie ranges
+ let token_subtrie_ranges_cell = token_subtrie_ranges.map(|subtrie_ranges| {
+ let subtrie_ranges_shape = model_shape.subtrie_ranges_shape(suffix_length);
+ let mut subtrie_ranges_array = scratch.token_subtrie_ranges.view(&subtrie_ranges_shape);
+ subtrie_ranges_array.as_slice_mut::<u32>().fill(0);
+ subtrie_ranges_array.copy_from_view(
+ ArrayView2::from_shape((subtrie_ranges.len(), 3), bytemuck::cast_slice::<_, u32>(subtrie_ranges))
+ .unwrap(),
+ );
+ subtrie_ranges_array
+ });
+
// Token positions - use async buffer if provided
let token_positions_cell = if let Some((async_buf, offset)) = async_positions {
unsafe {
@@ -209,16 +221,6 @@ impl<B: Backend> ForwardPassState<B> {
// Sampling output
let sampling_output = Some(scratch.sampling_output.view(&[suffix_length]));
- // Attention bias (causal + sliding window)
- let attention_bias = Self::init_llm_attention_bias(
- scratch,
- &cache_layers,
- suffix_length,
- token_positions,
- external_bias_fn,
- should_fill_attention_bias,
- );
-
// Common aux buffers
let common_aux = CommonAuxBuffers::new(scratch, model_shape, suffix_length);
@@ -245,10 +247,10 @@ impl<B: Backend> ForwardPassState<B> {
Self {
context,
token_ids: token_ids_cell,
+ token_subtrie_ranges: token_subtrie_ranges_cell,
token_positions: token_positions_cell,
token_parents: token_parents_cell,
token_bitmask: token_bitmask_cell,
- attention_bias,
shared_buffers,
common_aux,
llm_aux,
@@ -256,42 +258,6 @@ impl<B: Backend> ForwardPassState<B> {
}
}
- fn init_llm_attention_bias(
- scratch: &ScratchBuffers<B>,
- cache_layers: &Rc<RefCell<CacheLayers<B>>>,
- suffix_length: usize,
- token_positions: &[usize],
- external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
- should_fill_attention_bias: bool,
- ) -> HashMap<Option<usize>, Array<B>> {
- let cache_ref = cache_layers.borrow();
- let mut attention_bias_map: HashMap<Option<usize>, Array<B>> = scratch
- .attention_window_size_to_bias
- .iter()
- .map(|(window_size, buffer)| {
- let prefix_length = window_size.unwrap_or(cache_ref.max_prefix_length());
- let attention_bias_shape = [suffix_length, suffix_length + prefix_length];
- let array = buffer.view(&attention_bias_shape);
- (*window_size, array)
- })
- .collect();
- drop(cache_ref);
-
- // Use cache_layers' fill_attention_bias which properly handles
- // both causal masking and sliding window constraints
- // Skip fill for async decode passes after the first one (bias already set)
- if should_fill_attention_bias {
- cache_layers.borrow().fill_attention_bias(
- &mut attention_bias_map,
- token_positions,
- suffix_length,
- external_bias_fn,
- );
- }
-
- attention_bias_map
- }
-
// ========================================================================
// Classifier Constructor
// ========================================================================
@@ -304,7 +270,6 @@ impl<B: Backend> ForwardPassState<B> {
shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
token_ids: &[u64],
token_positions: &[usize],
- bidirectional_attention: bool,
num_labels: usize,
) -> Self {
let suffix_length = token_ids.len();
@@ -314,9 +279,6 @@ impl<B: Backend> ForwardPassState<B> {
let token_positions_cell = Self::init_token_positions(scratch, token_positions);
let token_parents_cell = Self::init_token_parents(scratch, token_positions, 0, 0);
- // Attention bias (bidirectional or causal)
- let attention_bias = Self::init_classifier_attention_bias(scratch, suffix_length, bidirectional_attention);
-
// Common aux buffers
let common_aux = CommonAuxBuffers::new(scratch, model_shape, suffix_length);
@@ -330,10 +292,10 @@ impl<B: Backend> ForwardPassState<B> {
Self {
context,
token_ids: token_ids_cell,
+ token_subtrie_ranges: None,
token_positions: token_positions_cell,
token_parents: token_parents_cell,
token_bitmask: None,
- attention_bias,
shared_buffers,
common_aux,
llm_aux: None,
@@ -341,40 +303,6 @@ impl<B: Backend> ForwardPassState<B> {
}
}
- fn init_classifier_attention_bias(
- scratch: &ScratchBuffers<B>,
- suffix_length: usize,
- bidirectional_attention: bool,
- ) -> HashMap<Option<usize>, Array<B>> {
- let mut attention_bias_map: HashMap<Option<usize>, Array<B>> = scratch
- .attention_window_size_to_bias
- .iter()
- .map(|(window_size, buffer)| {
- let attention_bias_shape = [suffix_length, suffix_length];
- let array = buffer.view(&attention_bias_shape);
- (*window_size, array)
- })
- .collect();
-
- for (window, bias_array) in attention_bias_map.iter_mut() {
- if bidirectional_attention {
- if let Some(window_size) = window {
- let half_window = (window_size / 2) as isize;
- fill_attention_bias(bias_array, suffix_length, 0, |row, col| {
- let distance = (row as isize) - (col as isize);
- distance.abs() > half_window
- });
- } else {
- fill_attention_bias(bias_array, suffix_length, 0, |_row, _col| false);
- }
- } else {
- fill_attention_bias(bias_array, suffix_length, 0, |row, col| row < col);
- }
- }
-
- attention_bias_map
- }
-
#[cfg(feature = "tracing")]
fn init_classifier_buffers(
context: &B::Context,
@@ -658,17 +586,6 @@ impl<B: Backend> ForwardPassState<B> {
// Public API Methods (formerly trait methods)
// ========================================================================
- pub fn hashmaps(
- &self,
- ids: &[HashMapId],
- ) -> Box<[HashMap<Option<usize>, Array<B>>]> {
- ids.iter()
- .map(|id| match id {
- HashMapId::AttentionBias => self.attention_bias.clone(),
- })
- .collect()
- }
-
pub fn active_suffix_length(&self) -> usize {
match &self.mode {
ForwardPassMode::LanguageModelGenerator(state) => state.active_suffix_length,
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 5ad622507..8b10b09ae 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -18,13 +18,13 @@ use super::{
use crate::{
backends::common::{
Backend, Buffer, CommandBuffer, Context, Encoder, Executable,
- kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},
+ kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},
},
config::ModelMetadata,
encodable_block::EncodingParameters,
forward_pass::{
cache_layers::{CacheLayer, CacheLayersSlice},
- kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION},
+ kv_cache_layer::INVALID_POSITION,
state::ForwardPassState,
},
session::{
@@ -39,6 +39,7 @@ use crate::{
#[derive(Debug, Clone)]
struct Task<'a> {
token_ids: &'a [u64],
+ token_subtrie_ranges: Option<&'a [[u32; 3]]>,
token_positions: &'a [usize],
token_bitmask: Option<&'a [u32]>,
token_seeds: &'a [u64],
@@ -152,7 +153,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let speculator = &self.decoding_config.speculator_config.speculator;
let suffix_length = if sample_suffix {
- self.decoding_config.generate_suffix_length().saturating_sub(1)
+ self.decoding_config.generate_suffix_length().saturating_sub(1).min(prefill_size - tokens_length)
} else {
prefill_size - tokens_length
};
@@ -171,6 +172,10 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let token_ids =
tokens.iter().copied().take(tokens_length - 1).chain(flat_trie.token_ids()).chunks(prefill_step_size);
+ let token_subtrie_ranges = repeat_n(None, tokens_length - 1)
+ .chain(flat_trie.token_subtrie_ranges().map(Some))
+ .chunks(prefill_step_size);
+
let token_positions = (prefix_offset..prefix_offset + tokens_length - 1)
.chain(flat_trie.token_positions().map(|trie_position| prefix_offset + tokens_length - 1 + trie_position))
.chunks(prefill_step_size);
@@ -184,13 +189,38 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let mut run_times: Vec<f64> = Vec::new();
// Process each prefill step and update the KV cache.
- for (step, (step_token_ids, step_token_positions, step_token_bitmasks, step_token_seeds)) in
- izip!(&token_ids, &token_positions, &token_bitmasks, &token_seeds).enumerate()
+ for (
+ step,
+ (step_token_ids, step_token_subtrie_ranges, step_token_positions, step_token_bitmasks, step_token_seeds),
+ ) in izip!(&token_ids, &token_subtrie_ranges, &token_positions, &token_bitmasks, &token_seeds).enumerate()
{
let tokens_start_index = step * prefill_step_size;
let tokens_end_index = tokens_start_index + prefill_step_size;
let step_token_ids = step_token_ids.collect::<Box<[u64]>>();
+ let step_token_subtrie_ranges = step_token_subtrie_ranges.collect::<Box<[Option<[u32; 3]>]>>();
+ let step_token_subtrie_ranges: Option<Box<[[u32; 3]]>> =
+ if let Some(trie_start) = step_token_subtrie_ranges.iter().position(|e| e.is_some()) {
+ Some(
+ step_token_subtrie_ranges
+ .iter()
+ .enumerate()
+ .map(|(i, me)| {
+ if let Some([subtrie_start, subtrie_end, height]) = me {
+ [
+ trie_start as u32 + subtrie_start,
+ trie_start as u32 + subtrie_end,
+ trie_start as u32 + height,
+ ]
+ } else {
+ [i as u32, step_token_subtrie_ranges.len() as u32 - 1, i as u32]
+ }
+ })
+ .collect(),
+ )
+ } else {
+ None
+ };
let step_token_positions = step_token_positions.collect::<Box<[usize]>>();
let step_token_seeds = step_token_seeds.collect::<Box<[u64]>>();
@@ -241,6 +271,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let task = Task {
token_ids: &step_token_ids,
+ token_subtrie_ranges: step_token_subtrie_ranges.as_deref(),
token_positions: &step_token_positions,
token_bitmask: step_token_bitmask.as_deref(),
token_seeds: &step_token_seeds,
@@ -251,8 +282,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
is_prefilling: !should_sample_after_step,
};
- let (state, run_time) =
- self.run_model(task, self.allow_pre_encode(), sampling_method, self.should_fill_attention_bias())?;
+ let (state, run_time) = self.run_model(task, self.allow_pre_encode(), sampling_method)?;
if should_capture {
self.gpu_capture.stop_capture(&self.context.context, "prefill").map_err(|_| Error::CaptureFailed)?;
@@ -260,28 +290,18 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
// Register the accepted prompt tokens from this step.
let step_end_token_index = std::cmp::min(tokens_end_index, tokens_length);
- let tokens_processed_this_step = step_end_token_index - tokens_start_index;
+ let mut tokens_processed_this_step = step_end_token_index - tokens_start_index;
+
+ if step == prefill_steps - 1 && sample_suffix {
+ tokens_processed_this_step = tokens_processed_this_step.saturating_sub(1);
+ }
if tokens_processed_this_step > 0 {
- let mut positions_for_step: Vec<usize> =
- (tokens_start_index..step_end_token_index).map(|idx| idx + prefix_offset).collect();
- if step == prefill_steps - 1 && sample_suffix {
- // Exclude the last token because it belongs to the suffix for sampling.
- positions_for_step.pop();
- }
-
- if !positions_for_step.is_empty() {
- let accept_indices_for_step: Vec<usize> = (0..positions_for_step.len()).collect();
- if !accept_indices_for_step.is_empty() {
- self.update_cache_layers(&accept_indices_for_step, None, true)?;
- }
+ self.update_cache_layers(&(0..tokens_processed_this_step).collect::<Vec<usize>>(), None, true)?;
- self.context.cache_layers.borrow_mut().register_accepted_tokens(&positions_for_step);
+ self.context.cache_layers.borrow_mut().register_accepted_tokens(tokens_processed_this_step);
- if let Some(&last_idx) = positions_for_step.last() {
- self.registered_prefix_len = last_idx + 1;
- }
- }
+ self.registered_prefix_len = prefix_offset + tokens_start_index + tokens_processed_this_step;
}
last_state = Some(state);
@@ -342,6 +362,11 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let token_ids =
flat_trie.token_ids().chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();
+ let token_subtrie_ranges = flat_trie
+ .token_subtrie_ranges()
+ .chain(repeat_n([u32::MAX, u32::MAX, u32::MAX], suffix_length - active_suffix_length))
+ .collect::<Box<[[u32; 3]]>>();
+
let token_bitmask: Option<Box<[u32]>> = compiled_grammar.is_some().then(|| {
let single_token_bitmask_size = self.context.model_shape.bitmask_shape(1)[1];
flat_trie
@@ -372,6 +397,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let task = Task {
token_ids: &token_ids,
+ token_subtrie_ranges: Some(&token_subtrie_ranges),
token_positions: &token_positions,
token_bitmask: token_bitmask.as_deref(),
token_seeds: &token_seeds,
@@ -382,8 +408,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
is_prefilling: false,
};
- let (mut state, run_time) =
- self.run_model(task, self.allow_pre_encode(), sampling_method, self.should_fill_attention_bias())?;
+ let (mut state, run_time) = self.run_model(task, self.allow_pre_encode(), sampling_method)?;
let sampled_tokens = self.read_sampling_output(&mut state)?;
@@ -412,17 +437,6 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
tokens_to_generate: usize,
) {
let prefill_count = self.tokens.len();
- let first_decode_position = prefill_count.saturating_sub(1);
-
- // Fill attention bias from KV cache state for the first decode position.
- // Windowed buffers get the correct mask; GPU patches maintain it for subsequent passes.
- // Full-attention buffers also get filled (masking beyond-prefix positions).
- self.context.cache_layers.borrow().fill_attention_bias_scratch(
- &mut self.context.scratch_buffers.attention_window_size_to_bias,
- &[first_decode_position],
- 1,
- &self.context.context,
- );
self.context.async_buffers.prepare_positions(prefill_count, tokens_to_generate);
self.context.async_buffers.prepare_seeds(&self.context.seed, prefill_count, tokens_to_generate);
@@ -462,6 +476,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let task = Task {
token_ids: &[last_token],
+ token_subtrie_ranges: None,
token_positions: &[token_position],
token_bitmask: None,
token_seeds: &[0], // Ignored, using async buffer
@@ -475,7 +490,6 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let async_positions = Some((async_positions_buffer.clone(), pass_idx));
let async_seeds = Some((async_seeds_buffer.clone(), pass_idx));
- let should_fill_attention_bias = false; // we fill it once in prepare
let skip_token_ids_copy = pass_idx > 0;
let is_first_decode = !is_continuation;
@@ -492,6 +506,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.context.cache_layers.clone(),
self.context.shared_buffers.clone(),
&task.token_ids,
+ task.token_subtrie_ranges,
&task.token_positions,
task.token_bitmask,
&task.token_seeds,
@@ -499,9 +514,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
/*sampling_start=*/ 0,
/*sampling_length=*/ task.active_suffix_length,
task.is_prefilling,
- None,
skip_token_ids_copy,
- should_fill_attention_bias,
async_positions,
async_seeds,
);
@@ -552,26 +565,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
&mut encoder,
&self.context.kv_cache_update,
);
- self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);
-
- if let Some(mask_update) = &self.context.mask_update {
- let updates: Vec<AttentionBiasUpdate> =
- self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);
- for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {
- if let Some(update) = updates.iter().find(|u| &u.key == window_size) {
- if update.unmask_col >= 0 || update.mask_col >= 0 {
- let mask_buf_rc = mask_buffer.buffer();
- let mut mask_buf_borrow = mask_buf_rc.borrow_mut();
- mask_update.encode(
- mask_buf_borrow.deref_mut(),
- update.unmask_col,
- update.mask_col,
- &mut encoder,
- );
- }
- }
- }
- }
+ self.context.cache_layers.borrow_mut().register_accepted_tokens(1);
// Signal event for next pass
let next_counter = current_counter + 1;
@@ -671,7 +665,6 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
dst.values.copy_slice(&src.values, 1, 0..copy_rows, 0);
}
dst.state = src.state.clone();
- dst.prefix_token_positions = src.prefix_token_positions.clone();
},
(CacheLayer::StateSpace(src), CacheLayer::StateSpace(dst)) => {
dst.conv_state.copy_from_array(&src.conv_state);
@@ -711,7 +704,6 @@ impl<B: Backend> LanguageModelGenerator<B> {
task: Task,
allow_pre_encode: bool,
sampling_method: SamplingMethod,
- should_fill_attention_bias: bool,
) -> Result<(ForwardPassState<B>, f64), Error> {
let run_start = Instant::now();
@@ -723,6 +715,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
self.context.cache_layers.clone(),
self.context.shared_buffers.clone(),
task.token_ids,
+ task.token_subtrie_ranges,
task.token_positions,
task.token_bitmask,
task.token_seeds,
@@ -730,9 +723,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
task.sampling_start,
task.sampling_length,
task.is_prefilling,
- None,
false,
- should_fill_attention_bias,
None,
None,
);
@@ -879,19 +870,9 @@ impl<B: Backend> LanguageModelGenerator<B> {
let desired_prefix_len = self.tokens.len() - 1;
if desired_prefix_len > self.registered_prefix_len {
- let positions: Vec<usize> = (self.registered_prefix_len..desired_prefix_len).collect();
- if !positions.is_empty() {
- self.context.cache_layers.borrow_mut().register_accepted_tokens(&positions);
- }
+ let number_of_accepted_tokens = desired_prefix_len - self.registered_prefix_len;
+ self.context.cache_layers.borrow_mut().register_accepted_tokens(number_of_accepted_tokens);
self.registered_prefix_len = desired_prefix_len;
}
}
-
- fn should_fill_attention_bias(&self) -> bool {
- let sliding_window_sizes = self.context.model_shape.sliding_window_length_per_layer.clone();
- let has_sliding_window = sliding_window_sizes.iter().any(|size| size.is_some());
- let has_speculative_suffix = self.decoding_config.generate_suffix_length() > 1;
-
- has_sliding_window || has_speculative_suffix
- }
}
diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rs
index 1849f66d3..d3d67cc36 100644
--- a/crates/uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/uzu/src/language_model/language_model_generator_context.rs
@@ -9,7 +9,7 @@ use crate::{
DataType,
backends::common::{
Backend, Buffer, Context, Kernels,
- kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate},
+ kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate},
},
config::{DecoderConfig, LanguageModelConfig, ModelMetadata},
encodable_block::{Decoder, Sampling},
@@ -128,8 +128,6 @@ pub struct LanguageModelGeneratorContext<B: Backend> {
/// Kernels for copying sampled tokens in async pipeline
pub token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,
pub token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,
- /// Kernel for updating attention mask between async passes
- pub mask_update: Option<<B::Kernels as Kernels>::MaskUpdateKernel>,
/// Pre-allocated buffers for async generation
pub async_buffers: AsyncBuffers<B>,
}
@@ -164,8 +162,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));
shared_buffers.borrow_mut().update_data(&root_loader_view);
- let scratch_buffers =
- ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_prefix_length, max_suffix_length);
+ let scratch_buffers = ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_suffix_length);
let executables = Decoder::new(context.as_ref(), &decoder_config, &root_loader_view);
@@ -191,16 +188,6 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
let token_copy_results = <B::Kernels as Kernels>::TokenCopyToResultsKernel::new(&context)
.map_err(|e| Error::UnableToCreateContext(e.into()))?;
- // Create mask update kernel if model has attention layers
- let mask_update = if decoder_config.has_attention_layers() {
- Some(
- <B::Kernels as Kernels>::MaskUpdateKernel::new(&context, intermediate_data_type)
- .map_err(|e| Error::UnableToCreateContext(e.into()))?,
- )
- } else {
- None
- };
-
let async_batch_size = decoding_config.async_batch_size.resolve::<B>(model_path, context.as_ref());
let async_buffers = AsyncBuffers::new(context.as_ref(), max_prefix_length, async_batch_size);
@@ -220,7 +207,6 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
seed,
token_copy_sampled,
token_copy_results,
- mask_update,
async_buffers,
};
diff --git a/crates/uzu/src/session/tts_session/decoder_runtime.rs b/crates/uzu/src/session/tts_session/decoder_runtime.rs
index ae287aa1c..9e0d49386 100644
--- a/crates/uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/uzu/src/session/tts_session/decoder_runtime.rs
@@ -19,7 +19,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
transformer_subtree: &str,
embedding_subtree: &str,
readout_subtree: &str,
- max_prefix_length: usize,
max_suffix_length: usize,
) -> Result<Self, Error> {
let weights_path = model_path.join("model.safetensors");
@@ -34,8 +33,7 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
&root_loader_view,
transformer_subtree,
)?;
- let scratch_buffers =
- ScratchBuffers::new(context.as_ref(), decoder_config, model_shape, max_prefix_length, max_suffix_length);
+ let scratch_buffers = ScratchBuffers::new(context.as_ref(), decoder_config, model_shape, max_suffix_length);
let executables = Decoder::new_with_embedding_and_readout_subtrees(
context.as_ref(),
&decoder_config,
@@ -95,12 +93,10 @@ impl<B: Backend> TokenDecoderContext<B> {
embedding_subtree: &str,
readout_subtree: &str,
runtime_config: &TextDecoderRuntimeConfig,
- ) -> Result<(Self, DataType, bool), Error> {
+ ) -> Result<(Self, DataType), Error> {
let model_shape = ModelShape::from_decoder_config(&decoder_config);
let max_prefix_length = decoder_config.context_length;
let max_suffix_length = text_decoder_prefill_step_size(runtime_config, decoder_config.context_length).max(32);
- let should_fill_attention_bias =
- model_shape.sliding_window_length_per_layer.iter().any(|value| value.is_some());
let activation_data_type = model_shape.activation_data_type();
let loaded_model = TokenDecoderLoadedModel::<B>::load(
&context,
@@ -110,7 +106,6 @@ impl<B: Backend> TokenDecoderContext<B> {
transformer_subtree,
embedding_subtree,
readout_subtree,
- max_prefix_length,
max_suffix_length,
)?;
let async_chain_capacity = max_suffix_length.max(1);
@@ -145,7 +140,6 @@ impl<B: Backend> TokenDecoderContext<B> {
async_chain_capacity,
},
activation_data_type,
- should_fill_attention_bias,
))
}
@@ -200,7 +194,6 @@ pub(super) struct TokenDecoderRunner<B: Backend> {
tensor_add_scale: <B::Kernels as Kernels>::TensorAddScaleKernel,
single_token_vocab_masks: HashMap<usize, Box<[u32]>>,
two_token_vocab_masks: HashMap<usize, Box<[u32]>>,
- should_fill_attention_bias: bool,
next_position: usize,
instrumentation: RunnerInstrumentation,
}
@@ -219,7 +212,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
readout_subtree: &str,
runtime_config: &TextDecoderRuntimeConfig,
) -> Result<Self, Error> {
- let (ctx, activation_data_type, should_fill_attention_bias) = TokenDecoderContext::new(
+ let (ctx, activation_data_type) = TokenDecoderContext::new(
context,
model_path,
decoder_config,
@@ -248,7 +241,6 @@ impl<B: Backend> TokenDecoderRunner<B> {
tensor_add_scale,
single_token_vocab_masks: HashMap::new(),
two_token_vocab_masks: HashMap::new(),
- should_fill_attention_bias,
next_position: 0,
instrumentation: RunnerInstrumentation::default(),
})
@@ -281,6 +273,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
self.ctx.cache_layers.clone(),
self.ctx.shared_buffers.clone(),
token_ids,
+ None,
&positions,
None, // no bitmask
&token_seeds,
@@ -288,9 +281,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
0, // sampling_start: irrelevant
0, // sampling_length: no sampling
false,
- None,
false,
- self.should_fill_attention_bias,
None,
None,
);
@@ -507,6 +498,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
cache_layers_rc.clone(),
shared_buffers_rc.clone(),
&token_ids,
+ None,
&[self.next_position + pass],
token_bitmask,
&[0],
@@ -514,9 +506,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
0,
1,
false,
- None,
pass > 0 || results_offset_slots > 0,
- self.should_fill_attention_bias,
Some((positions_rc.clone(), pass)),
Some((seeds_rc.clone(), pass)),
);
@@ -568,7 +558,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
}
self.ctx.cache_layers.borrow_mut().update_after_acceptance(&[0], None, encoder, &self.ctx.kv_cache_update);
- self.ctx.cache_layers.borrow_mut().register_accepted_tokens(&[self.next_position + pass]);
+ self.ctx.cache_layers.borrow_mut().register_accepted_tokens(1);
}
Ok(())
}
@@ -712,6 +702,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
self.ctx.cache_layers.clone(),
self.ctx.shared_buffers.clone(),
token_ids,
+ None,
positions,
token_bitmask,
token_seeds,
@@ -719,9 +710,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
sampling_start,
sampling_length,
false,
- None,
false,
- self.should_fill_attention_bias,
None,
None,
);
@@ -808,12 +797,11 @@ impl<B: Backend> TokenDecoderRunner<B> {
positions_storage = [self.next_position, self.next_position + 1];
&positions_storage[..]
} else {
- let v: Vec<usize> = (self.next_position..self.next_position + token_count).collect();
- self.ctx.cache_layers.borrow_mut().register_accepted_tokens(&v);
+ self.ctx.cache_layers.borrow_mut().register_accepted_tokens(token_count);
self.next_position = self.next_position.saturating_add(token_count);
return;
};
- self.ctx.cache_layers.borrow_mut().register_accepted_tokens(positions);
+ self.ctx.cache_layers.borrow_mut().register_accepted_tokens(positions.len());
self.next_position = self.next_position.saturating_add(token_count);
}
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index 9dd4de189..ce7d4c719 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -256,6 +256,7 @@ impl<B: Backend> TraceValidator<B> {
ctx.cache_layers.clone(),
ctx.shared_buffers.clone(),
&token_ids,
+ None,
&token_positions,
None,
&token_seeds,
@@ -263,9 +264,7 @@ impl<B: Backend> TraceValidator<B> {
/*sampling_start=*/ 0,
/*sampling_length=*/ token_ids.len(),
false,
- None,
false,
- true,
None,
None,
);
@@ -813,13 +812,8 @@ impl<B: Backend> TraceValidator<B> {
}
let decoder_config = &context.decoder_config;
- context.scratch_buffers = ScratchBuffers::new(
- context.context.as_ref(),
- decoder_config,
- &context.model_shape,
- resolved_prefix_length,
- desired_suffix_length,
- );
+ context.scratch_buffers =
+ ScratchBuffers::new(context.context.as_ref(), decoder_config, &context.model_shape, desired_suffix_length);
context.cache_layers = Rc::new(RefCell::new(CacheLayers::new(
context.context.as_ref(),
diff --git a/crates/uzu/src/trie.rs b/crates/uzu/src/trie.rs
index a2d869615..7cdb7bed6 100644
--- a/crates/uzu/src/trie.rs
+++ b/crates/uzu/src/trie.rs
@@ -31,9 +31,16 @@ pub struct TrieNode {
next: Vec<TrieNode>,
}
+#[derive(Debug)]
+struct FlatTrieNode<'a> {
+ node: &'a TrieNode,
+ subtrie_range: (usize, usize),
+ height: usize,
+}
+
#[derive(Debug)]
pub struct FlatTrie<'a> {
- tokens: Box<[(&'a TrieNode, usize)]>,
+ tokens: Box<[FlatTrieNode<'a>]>,
}
impl TrieNode {
@@ -170,20 +177,21 @@ impl TrieNode {
}
pub fn linearize(&self) -> FlatTrie<'_> {
- let mut tokens = vec![(self, 0)];
+ let mut tokens = vec![FlatTrieNode::new(self, (0, 0), 0)];
- let mut stack = vec![(self, 0)];
- while let Some((cur_node, next_child_idx)) = stack.last_mut() {
- let Some(next_node) = cur_node.next.get(*next_child_idx) else {
+ let mut stack = vec![(0, 0)];
+ while let Some((cur_node_idx, next_child_idx)) = stack.last_mut() {
+ let Some(next_node) = tokens[*cur_node_idx].node.next.get(*next_child_idx) else {
+ tokens[*cur_node_idx].subtrie_range.1 = tokens.len() - 1;
stack.pop();
continue;
};
*next_child_idx += 1;
- tokens.push((next_node, stack.len()));
+ tokens.push(FlatTrieNode::new(next_node, (tokens.len(), tokens.len()), stack.len()));
if !next_node.next.is_empty() {
- stack.push((next_node, 0));
+ stack.push((tokens.len() - 1, 0));
}
}
@@ -191,8 +199,22 @@ impl TrieNode {
}
}
+impl<'a> FlatTrieNode<'a> {
+ fn new(
+ node: &'a TrieNode,
+ subtrie_range: (usize, usize),
+ height: usize,
+ ) -> Self {
+ Self {
+ node,
+ subtrie_range,
+ height,
+ }
+ }
+}
+
impl<'a> FlatTrie<'a> {
- pub fn new(tokens: Box<[(&'a TrieNode, usize)]>) -> Self {
+ fn new(tokens: Box<[FlatTrieNode<'a>]>) -> Self {
Self {
tokens,
}
@@ -203,30 +225,38 @@ impl<'a> FlatTrie<'a> {
}
pub fn token_ids(&self) -> impl Iterator<Item = u64> {
- self.tokens.iter().map(|&(n, _p)| n.token)
+ self.tokens.iter().map(|n| n.node.token)
+ }
+
+ pub fn token_subtrie_ranges(&self) -> impl Iterator<Item = [u32; 3]> {
+ self.tokens.iter().map(|n| {
+ let (start, end) = n.subtrie_range;
+
+ [start as u32, end as u32, n.height as u32]
+ })
}
pub fn token_positions(&self) -> impl Iterator<Item = usize> {
- self.tokens.iter().map(|&(_n, p)| p)
+ self.tokens.iter().map(|n| n.height)
}
pub fn token_masks(&self) -> impl Iterator<Item = Option<&[u32]>> {
- self.tokens.iter().map(|&(n, _p)| n.mask.as_deref())
+ self.tokens.iter().map(|n| n.node.mask.as_deref())
}
pub fn token_seeds(&self) -> impl Iterator<Item = u64> {
- self.tokens.iter().map(|&(n, _p)| n.seed)
+ self.tokens.iter().map(|n| n.node.seed)
}
pub fn root(&self) -> Option<&TrieNode> {
- self.tokens.first().map(|&(n, _p)| n)
+ self.tokens.first().map(|n| n.node)
}
pub fn index(
&self,
node: &'a TrieNode,
) -> Option<usize> {
- self.tokens.iter().position(|&(n, _)| std::ptr::eq(n, node))
+ self.tokens.iter().position(|n| std::ptr::eq(n.node, node))
}
pub fn accept(
diff --git a/crates/uzu/src/utils/attention.rs b/crates/uzu/src/utils/attention.rs
deleted file mode 100644
index a50c60a80..000000000
--- a/crates/uzu/src/utils/attention.rs
+++ /dev/null
@@ -1,40 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-
-use crate::{ArrayElement, DataType, array::Array, backends::common::Backend};
-
-pub fn fill_attention_bias(
- dst: &mut Array<impl Backend>,
- suffix_length: usize,
- prefix_segment_length: usize,
- should_be_neg_inf: impl Fn(usize, usize) -> bool,
-) {
- let fill_typed_for_dtype = match dst.data_type() {
- DataType::F16 => fill_typed::<f16>,
- DataType::BF16 => fill_typed::<bf16>,
- DataType::F32 => fill_typed::<f32>,
- DataType::F64 => fill_typed::<f64>,
- _ => panic!("Unsupported data type for attention bias fill"),
- };
-
- fill_typed_for_dtype(dst, suffix_length, prefix_segment_length, should_be_neg_inf);
-}
-
-fn fill_typed<T: ArrayElement + Float>(
- dst: &mut Array<impl Backend>,
- suffix_length: usize,
- prefix_segment_length: usize,
- should_be_neg_inf: impl Fn(usize, usize) -> bool,
-) {
- let cols = suffix_length + prefix_segment_length;
- let slice = dst.as_slice_mut::<T>();
- for row in 0..suffix_length {
- for col in 0..cols {
- slice[row * cols + col] = if should_be_neg_inf(row, col) {
- T::neg_infinity()
- } else {
- T::zero()
- };
- }
- }
-}
diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rs
index 90c8993e4..530a66f61 100644
--- a/crates/uzu/src/utils/mod.rs
+++ b/crates/uzu/src/utils/mod.rs
@@ -1,4 +1,3 @@
-pub mod attention;
pub mod env_utils;
#[cfg(metal_backend)]
pub mod model_size;
diff --git a/crates/uzu/tests/unit/encodable_block/attention_test.rs b/crates/uzu/tests/unit/encodable_block/attention_test.rs
index d60d905be..82bf895b3 100644
--- a/crates/uzu/tests/unit/encodable_block/attention_test.rs
+++ b/crates/uzu/tests/unit/encodable_block/attention_test.rs
@@ -4,7 +4,7 @@ use std::mem::size_of;
use bytemuck;
use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
-use ndarray::{Array3, Array4, s};
+use ndarray::{Array4, s};
use objc2::{rc::Retained, runtime::ProtocolObject};
use test_tag::tag;
@@ -23,12 +23,12 @@ use crate::{
};
fn reference_attention(
- queries: &Array4<f32>, // [batch, num_heads, seq_len, head_dim]
- keys: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]
- values: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]
- mask: Option<&Array3<f32>>, // [batch, seq_len, seq_len] or None
+ queries: &Array4<f32>, // [batch, num_heads, seq_len, head_dim]
+ keys: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]
+ values: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]
sinks: Option<&[f32]>,
scale: f32,
+ is_causal: bool,
) -> Array4<f32> {
let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
let (_, num_kv_heads, kv_seq_len, _) = keys.dim();
@@ -51,9 +51,12 @@ fn reference_attention(
// Efficient matrix multiplication: scores = Q @ K^T
let mut scores = q.dot(&k.t());
- if let Some(mask_data) = mask {
- let mask_slice = mask_data.slice(s![b, .., ..kv_seq_len.min(seq_len)]);
- scores.slice_mut(s![.., ..kv_seq_len.min(seq_len)]).zip_mut_with(&mask_slice, |s, &m| *s += m);
+ if is_causal {
+ for i in 0..seq_len {
+ for j in (i + 1)..kv_seq_len {
+ scores[[i, j]] = f32::NEG_INFINITY;
+ }
+ }
}
// Softmax with optional sink logit
@@ -84,9 +87,12 @@ fn reference_attention(
// Efficient matrix multiplication: scores = Q @ K^T
let mut scores = q.dot(&k.t());
- if let Some(mask_data) = mask {
- let mask_slice = mask_data.slice(s![b, .., ..kv_seq_len.min(seq_len)]);
- scores.slice_mut(s![.., ..kv_seq_len.min(seq_len)]).zip_mut_with(&mask_slice, |s, &m| *s += m);
+ if is_causal {
+ for i in 0..seq_len {
+ for j in (i + 1)..kv_seq_len {
+ scores[[i, j]] = f32::NEG_INFINITY;
+ }
+ }
}
// Softmax with optional sink logit
@@ -120,7 +126,7 @@ fn create_test_data(
seq_len: usize,
head_dim: usize,
seed: u64,
-) -> (Array4<f32>, Array4<f32>, Array4<f32>, Array3<f32>) {
+) -> (Array4<f32>, Array4<f32>, Array4<f32>) {
use rand::{RngExt, SeedableRng, rngs::StdRng};
let mut rng = StdRng::seed_from_u64(seed);
@@ -131,15 +137,7 @@ fn create_test_data(
let values = Array4::from_shape_fn((batch_size, num_kv_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));
- let mask = Array3::from_shape_fn((batch_size, seq_len, seq_len), |(_, i, j)| {
- if j <= i {
- 0.0
- } else {
- -1e9
- }
- });
-
- (queries, keys, values, mask)
+ (queries, keys, values)
}
/// Convert ndarray to Metal buffer layout expected by our kernel
@@ -217,49 +215,6 @@ fn create_value_cache_buffer(
.expect("Failed to create buffer")
}
-fn create_mask_buffer(
- mask: &Array3<f32>,
- num_heads: usize,
- context: &<Metal as Backend>::Context,
-) -> Retained<ProtocolObject<dyn MTLBuffer>> {
- let (_batch_size, seq_len, _) = mask.dim();
-
- // Create mask buffer - the kernel expects mask layout: [num_heads, seq_len, seq_len]
- let mut mask_data = vec![0.0f32; num_heads * seq_len * seq_len];
- for h in 0..num_heads {
- for i in 0..seq_len {
- for j in 0..seq_len {
- let mask_idx = h * seq_len * seq_len + i * seq_len + j;
- mask_data[mask_idx] = mask[[0, i, j]];
- }
- }
- }
-
- context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&mask_data), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer")
-}
-
-fn create_mask_2d_buffer(
- mask: &Array3<f32>,
- context: &<Metal as Backend>::Context,
-) -> Retained<ProtocolObject<dyn MTLBuffer>> {
- let (_batch_size, seq_len, _) = mask.dim();
-
- let mut mask_data = vec![0.0f32; seq_len * seq_len];
- for i in 0..seq_len {
- for j in 0..seq_len {
- mask_data[i * seq_len + j] = mask[[0, i, j]];
- }
- }
-
- context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&mask_data), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer")
-}
-
fn create_sinks_buffer(
sinks: &[f32],
context: &<Metal as Backend>::Context,
@@ -297,7 +252,6 @@ fn run_single_pass_attention(
queries: &Array4<f32>,
keys: &Array4<f32>,
values: &Array4<f32>,
- mask: Option<&Array3<f32>>,
sinks: Option<&[f32]>,
scale: f32,
) -> Result<Array4<f32>, Box<dyn std::error::Error>> {
@@ -308,7 +262,6 @@ fn run_single_pass_attention(
let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);
let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);
- let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));
let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
let mut output_buffer = context
@@ -318,15 +271,6 @@ fn run_single_pass_attention(
let mut encoder = Encoder::new(context).expect("Failed to create encoder");
- let mut mask_kv_seq_stride: Option<u32> = None;
- let mut mask_q_seq_stride: Option<u32> = None;
- let mut mask_head_stride: Option<u32> = None;
- if mask_buffer.is_some() {
- mask_kv_seq_stride = Some(1);
- mask_q_seq_stride = Some(seq_len as u32);
- mask_head_stride = Some(0);
- }
-
kernel.encode(
&query_buffer,
&key_cache_buffer,
@@ -338,11 +282,10 @@ fn run_single_pass_attention(
head_dim as u32,
(seq_len * head_dim) as u32,
head_dim as u32,
+ None,
scale,
- mask_buffer.as_ref().map(|b| b),
- mask_kv_seq_stride,
- mask_q_seq_stride,
- mask_head_stride,
+ None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,
+ None,
sinks_buffer.as_ref().map(|b| b),
num_heads as u32,
seq_len as u32,
@@ -364,7 +307,6 @@ fn run_single_pass_attention_with_is_causal(
queries: &Array4<f32>,
keys: &Array4<f32>,
values: &Array4<f32>,
- mask: Option<&Array3<f32>>,
sinks: Option<&[f32]>,
scale: f32,
_is_causal: bool,
@@ -376,8 +318,6 @@ fn run_single_pass_attention_with_is_causal(
let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);
let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);
- let has_mask = mask.is_some();
- let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));
let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
let mut output_buffer = context
@@ -387,14 +327,6 @@ fn run_single_pass_attention_with_is_causal(
let mut encoder = Encoder::new(context).expect("Failed to create encoder");
- let mut mask_kv_seq_stride: Option<u32> = None;
- let mut mask_q_seq_stride: Option<u32> = None;
- let mut mask_head_stride: Option<u32> = None;
- if has_mask {
- mask_kv_seq_stride = Some(1);
- mask_q_seq_stride = Some(seq_len as u32);
- mask_head_stride = Some(0);
- }
kernel.encode(
&query_buffer,
&key_cache_buffer,
@@ -406,11 +338,10 @@ fn run_single_pass_attention_with_is_causal(
head_dim as u32,
(seq_len * head_dim) as u32,
head_dim as u32,
+ None,
scale,
- mask_buffer.as_ref().map(|b| b),
- mask_kv_seq_stride,
- mask_q_seq_stride,
- mask_head_stride,
+ None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,
+ None,
sinks_buffer.as_ref().map(|b| b),
num_heads as u32,
seq_len as u32,
@@ -432,7 +363,6 @@ fn run_gemm_attention(
queries: &Array4<f32>,
keys: &Array4<f32>,
values: &Array4<f32>,
- mask: Option<&Array3<f32>>,
sinks: Option<&[f32]>,
scale: f32,
is_causal: bool,
@@ -444,7 +374,6 @@ fn run_gemm_attention(
let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);
let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);
- let mask_buffer = mask.map(|m| create_mask_2d_buffer(m, context));
let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
let mut output_buffer = context
@@ -459,7 +388,7 @@ fn run_gemm_attention(
keys_buffer: &key_cache_buffer,
values_buffer: &value_cache_buffer,
output_buffer: &mut output_buffer,
- mask_buffer: mask_buffer.as_ref(),
+ trie_buffer: None,
sinks_buffer: sinks_buffer.as_ref(),
num_heads,
num_groups: num_kv_heads,
@@ -467,7 +396,9 @@ fn run_gemm_attention(
sequence_length: seq_len,
segment_prefix_length: 0,
max_sequence_length: seq_len,
+ ring_params: None,
head_dim,
+ sliding_window_size: None,
is_causal,
scale,
};
@@ -542,14 +473,14 @@ fn test_single_pass_attention_basic() {
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- println!("Testing reference implementation without mask...");
- let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
+ println!("Testing reference implementation...");
+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
let ref_max = reference_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));
let ref_min = reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));
- println!("Reference output range (no mask): [{}, {}]", ref_min, ref_max);
+ println!("Reference output range: [{}, {}]", ref_min, ref_max);
println!("Reference sample values: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());
let is_causal = false; // Non-causal attention for this test
@@ -559,13 +490,13 @@ fn test_single_pass_attention_basic() {
head_dim as u32,
false,
false,
- false,
is_causal,
+ false,
+ false,
)
.expect("Failed to create attention single pass metal");
- let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale)
- {
+ let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale) {
Ok(output) => output,
Err(e) => {
panic!("Failed to run single-pass attention: {:?}", e);
@@ -589,14 +520,14 @@ fn test_gemm_attention_basic() {
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
let kernel = AttentionGemmBlock::new(DataType::F32);
- let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
let kernel_output =
- run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, None, scale, /*is_causal=*/ false)
+ run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, scale, /*is_causal=*/ false)
.expect("run gemm attention");
let tolerance = 1e-2;
@@ -616,14 +547,14 @@ fn test_gemm_attention_f32_head_dim_128() {
let head_dim = 128;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);
+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);
let kernel = AttentionGemmBlock::new(DataType::F32);
- let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
let kernel_output =
- run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, None, scale, /*is_causal=*/ false)
+ run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, scale, /*is_causal=*/ false)
.expect("run gemm attention f32 head_dim=128");
let tolerance = 1e-2;
@@ -644,40 +575,31 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
let scale = 1.0 / (head_dim as f32).sqrt();
let is_causal = true;
- let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);
+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);
- // CPU reference (uses the same additive mask)
- let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);
+ // CPU reference
+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, is_causal);
// vector attention path (single-pass)
let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
&context,
DataType::F32,
head_dim as u32,
- true,
- true,
+ false,
false,
is_causal,
+ false,
+ false,
)
.expect("Failed to create attention single pass metal");
- let vector_output = run_single_pass_attention_with_is_causal(
- &kernel,
- &context,
- &queries,
- &keys,
- &values,
- Some(&mask),
- None,
- scale,
- is_causal,
- )
- .expect("run vector attention");
+ let vector_output =
+ run_single_pass_attention_with_is_causal(&kernel, &context, &queries, &keys, &values, None, scale, is_causal)
+ .expect("run vector attention");
// matrix attention path (gemm)
let gemm_kernel = AttentionGemmBlock::new(DataType::F32);
- let matrix_output =
- run_gemm_attention(&gemm_kernel, &context, &queries, &keys, &values, Some(&mask), None, scale, is_causal)
- .expect("run matrix attention");
+ let matrix_output = run_gemm_attention(&gemm_kernel, &context, &queries, &keys, &values, None, scale, is_causal)
+ .expect("run matrix attention");
// Compare both to CPU
let tol_cpu = 5e-2;
@@ -696,53 +618,6 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
);
}
-#[test]
-fn test_single_pass_attention_with_mask() {
- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
-
- let batch_size = 1;
- let num_heads = 4;
- let num_kv_heads = 4;
- let seq_len = 8;
- let head_dim = 64;
- let scale = 1.0 / (head_dim as f32).sqrt();
-
- let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
-
- let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
- &context,
- DataType::F32,
- head_dim as u32,
- true,
- true,
- false,
- false,
- )
- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");
- let kernel_output =
- match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&mask), None, scale) {
- Ok(output) => output,
- Err(e) => {
- panic!("Failed to run single-pass attention with mask: {:?}", e);
- },
- };
-
- let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);
-
- println!("Mask values:");
- for i in 0..seq_len.min(4) {
- for j in 0..seq_len.min(4) {
- print!("{:8.1} ", mask[[0, i, j]]);
- }
- println!();
- }
-
- let tolerance = 1e-2;
- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention with mask") {
- panic!("{}", e);
- }
-}
-
#[test]
fn test_single_pass_attention_with_sinks() {
let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");
@@ -754,7 +629,7 @@ fn test_single_pass_attention_with_sinks() {
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 444);
+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 444);
let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();
println!("Using sinks: {:?}", sinks);
@@ -763,17 +638,18 @@ fn test_single_pass_attention_with_sinks() {
&context,
DataType::F32,
head_dim as u32,
+ true,
+ false,
false,
false,
- true,
false,
)
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");
- let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);
+ let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);
let kernel_output =
- match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, Some(&sinks), scale) {
+ match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale) {
Ok(output) => output,
Err(e) => {
panic!("Failed to run single-pass attention with sinks: {:?}", e);
@@ -797,7 +673,7 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 777);
+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 777);
let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();
println!("Long sequence sinks: {:?}", sinks);
@@ -806,17 +682,18 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
&context,
DataType::F32,
head_dim as u32,
+ true,
+ false,
false,
false,
- true,
false,
)
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");
- let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);
+ let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);
let kernel_output =
- match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, Some(&sinks), scale) {
+ match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale) {
Ok(output) => output,
Err(e) => {
panic!("Failed to run single-pass attention long sequence with sinks: {:?}", e);
@@ -845,7 +722,7 @@ fn test_single_pass_attention_gqa() {
let head_dim = 64;
let scale = 1.0 / (head_dim as f32).sqrt();
- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
let is_causal = false; // Non-causal attention for this test
let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
@@ -854,20 +731,20 @@ fn test_single_pass_attention_gqa() {
head_dim as u32,
false,
false,
- false,
is_causal,
+ false,
+ false,
)
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");
- let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale)
- {
+ let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale) {
Ok(output) => output,
Err(e) => {
panic!("Failed to run single-pass attention GQA: {:?}", e);
},
};
- let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
let tolerance = 1e-2;
if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention GQA") {
@@ -882,7 +759,6 @@ fn run_two_pass_attention(
queries: &Array4<f32>,
keys: &Array4<f32>,
values: &Array4<f32>,
- mask: Option<&Array3<f32>>,
sinks: Option<&[f32]>,
scale: f32,
) -> Result<Array4<f32>, Box<dyn std::error::Error>> {
@@ -892,7 +768,6 @@ fn run_two_pass_attention(
let queries_buffer = create_query_buffer(queries, context);
let keys_buffer = create_key_cache_buffer(keys, seq_len, context);
let values_buffer = create_value_cache_buffer(values, seq_len, context);
- let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));
let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
let total_blocks_count = 32;
@@ -924,14 +799,6 @@ fn run_two_pass_attention(
let mut encoder = Encoder::new(context).expect("Failed to create encoder");
- let mut mask_kv_seq_stride: Option<u32> = None;
- let mut mask_q_seq_stride: Option<u32> = None;
- let mut mask_head_stride: Option<u32> = None;
- if mask_buffer.is_some() {
- mask_kv_seq_stride = Some(1);
- mask_q_seq_stride = Some(seq_len as u32);
- mask_head_stride = Some(0);
- }
kernel_pass1.encode(
&queries_buffer,
&keys_buffer,
@@ -945,13 +812,12 @@ fn run_two_pass_attention(
head_dim as u32,
(seq_len * head_dim) as u32,
head_dim as u32,
+ None,
scale,
num_heads as u32,
seq_len as u32,
- mask_buffer.as_ref().map(|b| b),
- mask_kv_seq_stride,
- mask_q_seq_stride,
- mask_head_stride,
+ None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,
+ None,
sinks_buffer.as_ref().map(|b| b),
&mut encoder,
);
@@ -986,9 +852,9 @@ fn test_two_pass_attention() {
let scale = 1.0 / (head_dim as f32).sqrt();
let is_causal = false; // Non-causal attention for this test
- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
&context,
@@ -996,8 +862,9 @@ fn test_two_pass_attention() {
head_dim as u32,
false,
false,
- false,
is_causal,
+ false,
+ false,
)
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");
let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(
@@ -1006,22 +873,13 @@ fn test_two_pass_attention() {
head_dim as u32,
)
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");
- let kernel_output = match run_two_pass_attention(
- &kernel_pass1,
- &kernel_pass2,
- &context,
- &queries,
- &keys,
- &values,
- None,
- None,
- scale,
- ) {
- Ok(output) => output,
- Err(e) => {
- panic!("Failed to run two-pass attention: {:?}", e);
- },
- };
+ let kernel_output =
+ match run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale) {
+ Ok(output) => output,
+ Err(e) => {
+ panic!("Failed to run two-pass attention: {:?}", e);
+ },
+ };
let tolerance = 1e-2;
if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Two-pass attention") {
@@ -1041,17 +899,18 @@ fn test_two_pass_attention_gqa() {
let scale = 1.0 / (head_dim as f32).sqrt();
let is_causal = false; // Non-causal attention for this test
- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
&context,
DataType::F32,
head_dim as u32,
- is_causal,
false,
false,
+ is_causal,
+ false,
false,
)
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");
@@ -1061,22 +920,13 @@ fn test_two_pass_attention_gqa() {
head_dim as u32,
)
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");
- let kernel_output = match run_two_pass_attention(
- &kernel_pass1,
- &kernel_pass2,
- &context,
- &queries,
- &keys,
- &values,
- None,
- None,
- scale,
- ) {
- Ok(output) => output,
- Err(e) => {
- panic!("Failed to run two-pass attention GQA: {:?}", e);
- },
- };
+ let kernel_output =
+ match run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale) {
+ Ok(output) => output,
+ Err(e) => {
+ panic!("Failed to run two-pass attention GQA: {:?}", e);
+ },
+ };
let tolerance = 1e-2;
if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Two-pass attention GQA") {
@@ -1106,15 +956,16 @@ fn perf_two_pass_attention() {
seq_len - suffix_length,
suffix_length
);
- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
&context,
DataType::F32,
head_dim as u32,
- is_causal,
false,
false,
+ is_causal,
+ false,
false,
)
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");
@@ -1159,7 +1010,6 @@ fn perf_two_pass_attention() {
// ---- Launch and time ----
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- let mask_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;
let sinks_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;
kernel_pass1.encode(
&queries_buffer,
@@ -1174,12 +1024,11 @@ fn perf_two_pass_attention() {
head_dim as u32,
(seq_len * head_dim) as u32,
head_dim as u32,
+ None,
scale,
num_heads as u32,
suffix_length as u32,
- mask_buffer.as_ref().map(|b| b),
- None,
- None,
+ None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,
None,
sinks_buffer.as_ref().map(|b| b),
&mut encoder,
diff --git a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs
index eb4f77a06..ce78d6f79 100644
--- a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs
+++ b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs
@@ -6,7 +6,7 @@ use crate::{
common::{Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},
metal::Metal,
},
- forward_pass::kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},
+ forward_pass::kv_cache_layer::{KVCacheLayer, KVCacheLayerState},
};
#[derive(Debug)]
@@ -15,13 +15,9 @@ struct Scenario {
state: KVCacheLayerState,
prefix_capacity: usize,
suffix_capacity: usize,
- initial_prefix_positions: Vec<usize>,
accepted_suffix_indices: Vec<usize>,
- accepted_token_positions: Vec<usize>,
- suffix_token_positions: Vec<usize>,
+ number_of_accepted_tokens: usize,
suffix_start: Option<usize>,
- expected_prefix_positions: Vec<usize>,
- expected_mask: Vec<Vec<bool>>,
expected_ring_offset: Option<usize>,
expected_ring_length: Option<usize>,
expected_prefix_len: Option<usize>,
@@ -48,22 +44,10 @@ fn make_test_layer(
let keys = context.create_array_zeros(&shape, DataType::F32, "kv_cache_keys");
let values = context.create_array_zeros(&shape, DataType::F32, "kv_cache_values");
- let prefix_token_positions = match &state {
- KVCacheLayerState::Full {
- ..
- } => Vec::with_capacity(prefix_capacity),
- KVCacheLayerState::Windowed {
- window_length,
- ..
- } => vec![INVALID_POSITION; *window_length],
- };
-
KVCacheLayer {
state,
keys,
values,
- prefix_token_positions,
- max_suffix_length: suffix_capacity,
}
}
@@ -127,22 +111,6 @@ fn expected_after_update(
expected
}
-fn collect_mask(
- layer: &KVCacheLayer<Metal>,
- suffix_positions: &[usize],
-) -> Vec<Vec<bool>> {
- let suffix_length = suffix_positions.len();
- let prefix_segment_length = layer.prefix_segment_length();
-
- (0..suffix_length)
- .map(|row| {
- (0..prefix_segment_length + suffix_length)
- .map(|col| layer.bias_should_be_neg_inf(row, col, suffix_positions))
- .collect()
- })
- .collect()
-}
-
fn run_scenario(
context: &<Metal as Backend>::Context,
scenario: &Scenario,
@@ -150,8 +118,6 @@ fn run_scenario(
let mut layer =
make_test_layer(context, scenario.state.clone(), scenario.prefix_capacity, scenario.suffix_capacity);
- layer.prefix_token_positions = scenario.initial_prefix_positions.clone();
-
let state_before_update = layer.state.clone();
let (initial_keys, initial_values) = fill_arrays(&mut layer);
@@ -170,9 +136,6 @@ fn run_scenario(
} => window_length + scenario.suffix_capacity,
};
- let mask = collect_mask(&layer, &scenario.suffix_token_positions);
- assert_eq!(mask, scenario.expected_mask, "{}: bias mask mismatch", scenario.name);
-
let kv_cache_update = match KVCacheUpdate::new(context, DataType::F32, total_sequence_length) {
Ok(update) => update,
Err(e) => {
@@ -190,7 +153,7 @@ fn run_scenario(
encoder.end_encoding().submit().wait_until_completed().unwrap();
- layer.register_accepted_tokens(&scenario.accepted_token_positions);
+ layer.register_accepted_tokens(scenario.number_of_accepted_tokens);
let actual_keys = layer.keys.as_slice::<f32>().to_vec();
assert_eq!(actual_keys, expected_keys, "{}: key buffer mismatch", scenario.name);
@@ -198,12 +161,6 @@ fn run_scenario(
let actual_values = layer.values.as_slice::<f32>().to_vec();
assert_eq!(actual_values, expected_values, "{}: value buffer mismatch", scenario.name);
- assert_eq!(
- layer.prefix_token_positions, scenario.expected_prefix_positions,
- "{}: prefix positions mismatch",
- scenario.name
- );
-
match &layer.state {
KVCacheLayerState::Full {
prefix_len,
@@ -235,7 +192,7 @@ fn run_scenario(
}
#[test]
-fn kv_cache_state_and_mask_scenarios() {
+fn kv_cache_state_scenarios() {
let Some(context) = <Metal as Backend>::Context::new().ok() else {
return;
};
@@ -250,17 +207,9 @@ fn kv_cache_state_and_mask_scenarios() {
},
prefix_capacity: 6,
suffix_capacity: 3,
- initial_prefix_positions: vec![6, 1, 2, 3, 4, 5],
accepted_suffix_indices: vec![0, 1, 2],
- accepted_token_positions: vec![7, 8, 9],
- suffix_token_positions: vec![7, 8, 9],
+ number_of_accepted_tokens: 3,
suffix_start: None,
- expected_prefix_positions: vec![6, 7, 8, 9, 4, 5],
- expected_mask: vec![
- vec![false, true, false, false, false, false, false, true, true],
- vec![false, true, true, false, false, false, false, false, true],
- vec![false, true, true, true, false, false, false, false, false],
- ],
expected_ring_offset: Some(4),
expected_ring_length: Some(6),
expected_prefix_len: None,
@@ -275,17 +224,9 @@ fn kv_cache_state_and_mask_scenarios() {
},
prefix_capacity: 6,
suffix_capacity: 3,
- initial_prefix_positions: vec![0, 1, 2, 3, INVALID_POSITION, INVALID_POSITION],
accepted_suffix_indices: vec![0, 1, 2],
- accepted_token_positions: vec![4, 5, 6],
- suffix_token_positions: vec![4, 5, 6],
+ number_of_accepted_tokens: 3,
suffix_start: None,
- expected_prefix_positions: vec![6, 1, 2, 3, 4, 5],
- expected_mask: vec![
- vec![false, false, false, false, true, true, false, true, true],
- vec![false, false, false, false, true, true, false, false, true],
- vec![true, false, false, false, true, true, false, false, false],
- ],
expected_ring_offset: Some(1),
expected_ring_length: Some(6),
expected_prefix_len: None,
@@ -300,13 +241,9 @@ fn kv_cache_state_and_mask_scenarios() {
},
prefix_capacity: 6,
suffix_capacity: 1,
- initial_prefix_positions: vec![11, 12, 13, 14, 15, 10],
accepted_suffix_indices: vec![0],
- accepted_token_positions: vec![16],
- suffix_token_positions: vec![16],
+ number_of_accepted_tokens: 1,
suffix_start: None,
- expected_prefix_positions: vec![11, 12, 13, 14, 15, 16],
- expected_mask: vec![vec![false, false, false, false, false, true, false]],
expected_ring_offset: Some(0),
expected_ring_length: Some(6),
expected_prefix_len: None,
@@ -319,17 +256,9 @@ fn kv_cache_state_and_mask_scenarios() {
},
prefix_capacity: 6,
suffix_capacity: 3,
- initial_prefix_positions: vec![0, 1, 2, 3],
accepted_suffix_indices: vec![0, 1, 2],
- accepted_token_positions: vec![4, 5, 6],
- suffix_token_positions: vec![4, 5, 6],
+ number_of_accepted_tokens: 3,
suffix_start: None,
- expected_prefix_positions: vec![0, 1, 2, 3, 4, 5, 6],
- expected_mask: vec![
- vec![false, false, false, false, false, true, true],
- vec![false, false, false, false, false, false, true],
- vec![false, false, false, false, false, false, false],
- ],
expected_ring_offset: None,
expected_ring_length: None,
expected_prefix_len: Some(7),
@@ -358,14 +287,11 @@ fn kv_cache_slice_apply_contiguous_window() {
4,
0,
);
- layer.prefix_token_positions = vec![10, 11, 12, INVALID_POSITION];
let (initial_keys, initial_values) = fill_arrays(&mut layer);
let slice = layer.slice(&context, 0..2).expect("slice should exist");
// Mutate the captured slots.
{
- layer.prefix_token_positions[0] = 999;
- layer.prefix_token_positions[1] = 998;
layer.keys.as_slice_mut::<f32>()[0] = -1.0;
layer.keys.as_slice_mut::<f32>()[1] = -2.0;
layer.values.as_slice_mut::<f32>()[0] = -3.0;
@@ -374,12 +300,6 @@ fn kv_cache_slice_apply_contiguous_window() {
layer.apply_slice(&slice, None);
- assert_eq!(
- layer.prefix_token_positions,
- vec![10, 11, 12, INVALID_POSITION],
- "positions restored for contiguous slice"
- );
-
let keys_after = layer.keys.as_slice::<f32>().to_vec();
let values_after = layer.values.as_slice::<f32>().to_vec();
assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for contiguous slice");
@@ -402,14 +322,11 @@ fn kv_cache_slice_apply_wrap_window() {
4,
0,
);
- layer.prefix_token_positions = vec![30, 31, 32, 33];
let (initial_keys, initial_values) = fill_arrays(&mut layer);
let slice = layer.slice(&context, 2..4).expect("slice should exist");
// Captured slots are expected to wrap; mutate them.
{
- layer.prefix_token_positions[2] = 777;
- layer.prefix_token_positions[3] = 778;
layer.keys.as_slice_mut::<f32>()[2] = -11.0;
layer.keys.as_slice_mut::<f32>()[3] = -12.0;
layer.values.as_slice_mut::<f32>()[2] = -13.0;
@@ -418,8 +335,6 @@ fn kv_cache_slice_apply_wrap_window() {
layer.apply_slice(&slice, None);
- assert_eq!(layer.prefix_token_positions, vec![30, 31, 32, 33], "positions restored for wrapped slice");
-
let keys_after = layer.keys.as_slice::<f32>().to_vec();
let values_after = layer.values.as_slice::<f32>().to_vec();
assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for wrapped slice");
@@ -440,7 +355,6 @@ fn kv_cache_slice_apply_full_restores_metadata() {
6,
0,
);
- layer.prefix_token_positions = vec![1, 2, 3];
let slice = layer.slice(&context, 0..2).expect("full slice exists");
@@ -451,11 +365,9 @@ fn kv_cache_slice_apply_full_restores_metadata() {
{
*prefix_len = 1;
}
- layer.prefix_token_positions = vec![9, 9, 9, 9];
layer.apply_slice(&slice, None);
- assert_eq!(layer.prefix_token_positions, vec![1, 2, 3], "full slice restores positions");
if let KVCacheLayerState::Full {
prefix_len,
} = &layer.state
diff --git a/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs
index e3216cd1b..946684835 100644
--- a/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs
+++ b/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs
@@ -19,7 +19,6 @@ struct Input<T: ArrayElement + Float> {
queries: Box<[T]>,
keys: Box<[T]>,
values: Box<[T]>,
- mask: Option<Box<[T]>>,
num_heads: usize,
num_kv_heads: usize,
sequence_length: usize,
@@ -36,7 +35,6 @@ fn get_test_data<T: ArrayElement + Float>(
suffix_length: usize,
head_dim: usize,
do_causal: bool,
- with_mask: bool,
) -> (Input<T>, Vec<T>) {
// queries: [num_heads, suffix_length, head_dim] (contiguous, row-major)
let q_size = num_heads * suffix_length * head_dim;
@@ -59,31 +57,12 @@ fn get_test_data<T: ArrayElement + Float>(
values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();
}
- // mask: [suffix_length, sequence_length] (broadcast over heads)
- let mask = if with_mask {
- let m_size = suffix_length * sequence_length;
- let mut m = vec![T::zero(); m_size];
- for q in 0..suffix_length {
- for k in 0..sequence_length {
- let idx = q * sequence_length + k;
- let diff = (k as i32) - (sequence_length as i32 - suffix_length as i32 + q as i32);
- if diff.abs() > 3 {
- m[idx] = T::from(-1e9f32).unwrap();
- }
- }
- }
- Some(m.into_boxed_slice())
- } else {
- None
- };
-
let scale = 1.0 / (head_dim as f32).sqrt();
let input = Input {
queries: queries.into_boxed_slice(),
keys: keys.into_boxed_slice(),
values: values.into_boxed_slice(),
- mask,
num_heads,
num_kv_heads,
sequence_length,
@@ -109,8 +88,6 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let output_size = input.suffix_length * input.num_heads * input.head_dim;
let output_array = context.create_array_uninitialized(&[output_size], T::data_type(), "");
- let mask_array = input.mask.as_ref().map(|m| context.create_array_from(&[m.len()], m, ""));
-
let segment_prefix_length = input.sequence_length - input.suffix_length;
{
@@ -123,16 +100,12 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let o_buf_rc = output_array.buffer();
let mut o_buf = o_buf_rc.borrow_mut();
- let mask_buf_rc = mask_array.as_ref().map(|a| a.buffer());
- let mask_buf_borrow = mask_buf_rc.as_ref().map(|rc| rc.borrow());
- let mask_buffer: Option<&B::Buffer> = mask_buf_borrow.as_ref().map(|b| b.deref());
-
let args = AttentionGemmArguments::<B> {
queries_buffer: q_buf.deref(),
keys_buffer: k_buf.deref(),
values_buffer: v_buf.deref(),
output_buffer: o_buf.deref_mut(),
- mask_buffer,
+ trie_buffer: None,
sinks_buffer: None,
num_heads: input.num_heads,
num_groups: input.num_kv_heads,
@@ -140,7 +113,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
sequence_length: input.sequence_length,
segment_prefix_length,
max_sequence_length: input.sequence_length,
+ ring_params: None,
head_dim: input.head_dim,
+ sliding_window_size: None,
is_causal: input.do_causal,
scale: input.scale,
};
@@ -166,7 +141,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
for_each_non_cpu_backend!(|B| {
let output = get_output::<T, B>(input);
let msg = format!(
- "AttentionGemm failed (backend={}, heads={}, kv_heads={}, seq={}, suffix={}, head_dim={}, causal={}, mask={})",
+ "AttentionGemm failed (backend={}, heads={}, kv_heads={}, seq={}, suffix={}, head_dim={}, causal={})",
std::any::type_name::<B>(),
input.num_heads,
input.num_kv_heads,
@@ -174,63 +149,53 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
input.suffix_length,
input.head_dim,
input.do_causal,
- input.mask.is_some(),
);
assert_eq_float::<T>(expected, &output, eps, &msg);
});
}
fn test_basic<T: ArrayElement + Float + Debug + Display>() {
- // Non-causal, no mask, single token
- let (input, expected) = get_test_data::<T>(4, 4, 8, 1, 64, false, false);
+ // Non-causal, single token
+ let (input, expected) = get_test_data::<T>(4, 4, 8, 1, 64, false);
test_internal(&input, &expected);
- // Non-causal, no mask, multiple tokens
- let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, false, false);
+ // Non-causal, multiple tokens
+ let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, false);
test_internal(&input, &expected);
}
fn test_causal<T: ArrayElement + Float + Debug + Display>() {
// Causal, single token decode
- let (input, expected) = get_test_data::<T>(4, 4, 16, 1, 64, true, false);
+ let (input, expected) = get_test_data::<T>(4, 4, 16, 1, 64, true);
test_internal(&input, &expected);
// Causal, multi-token prefill
- let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, true, false);
+ let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, true);
test_internal(&input, &expected);
}
fn test_gqa<T: ArrayElement + Float + Debug + Display>() {
// GQA: 8 query heads, 2 kv heads
- let (input, expected) = get_test_data::<T>(8, 2, 8, 1, 64, false, false);
+ let (input, expected) = get_test_data::<T>(8, 2, 8, 1, 64, false);
test_internal(&input, &expected);
// GQA causal
- let (input, expected) = get_test_data::<T>(8, 2, 8, 4, 64, true, false);
- test_internal(&input, &expected);
-}
-
-fn test_mask<T: ArrayElement + Float + Debug + Display>() {
- let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, false, true);
- test_internal(&input, &expected);
-
- // Causal + mask
- let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, true, true);
+ let (input, expected) = get_test_data::<T>(8, 2, 8, 4, 64, true);
test_internal(&input, &expected);
}
fn test_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: usize) {
- let (input, expected) = get_test_data::<T>(4, 4, 8, 2, head_dim, true, false);
+ let (input, expected) = get_test_data::<T>(4, 4, 8, 2, head_dim, true);
test_internal(&input, &expected);
}
fn test_unaligned<T: ArrayElement + Float + Debug + Display>() {
// suffix_length not aligned to BQ=32
- let (input, expected) = get_test_data::<T>(4, 4, 40, 7, 64, true, false);
+ let (input, expected) = get_test_data::<T>(4, 4, 40, 7, 64, true);
test_internal(&input, &expected);
// sequence_length not aligned to BK
- let (input, expected) = get_test_data::<T>(4, 4, 13, 4, 64, false, false);
+ let (input, expected) = get_test_data::<T>(4, 4, 13, 4, 64, false);
test_internal(&input, &expected);
}
@@ -282,22 +247,6 @@ fn test_gqa_bf16() {
test_gqa::<bf16>();
}
-// Mask tests
-#[test]
-fn test_mask_f32() {
- test_mask::<f32>();
-}
-
-#[test]
-fn test_mask_f16() {
- test_mask::<f16>();
-}
-
-#[test]
-fn test_mask_bf16() {
- test_mask::<bf16>();
-}
-
// Head dim 128
#[test]
fn test_head_dim_128_f32() {
diff --git a/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs
index 8fc02f929..ffc99f37b 100644
--- a/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs
+++ b/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs
@@ -19,7 +19,6 @@ struct Input<T: ArrayElement + Float> {
queries: Box<[T]>,
keys: Box<[T]>,
values: Box<[T]>,
- mask: Option<Box<[T]>>,
num_heads: u32,
gqa_factor: u32,
sequence_length: u32,
@@ -27,7 +26,6 @@ struct Input<T: ArrayElement + Float> {
head_dim: u32,
scale: f32,
do_causal: bool,
- has_mask: bool,
}
fn get_input<T: ArrayElement + Float>(
@@ -37,7 +35,6 @@ fn get_input<T: ArrayElement + Float>(
suffix_length: u32,
head_dim: u32,
do_causal: bool,
- with_mask: bool,
) -> Input<T> {
let gqa_factor = num_heads / num_kv_heads;
@@ -62,32 +59,12 @@ fn get_input<T: ArrayElement + Float>(
values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();
}
- // mask: [sequence_length * suffix_length] (broadcast over heads with stride 0)
- let mask = if with_mask {
- let m_size = (suffix_length * sequence_length) as usize;
- let mut m = vec![T::zero(); m_size];
- for q in 0..suffix_length {
- for k in 0..sequence_length {
- let idx = (q * sequence_length + k) as usize;
- // Create a sliding window mask: allow only nearby keys
- let diff = (k as i32) - (sequence_length as i32 - suffix_length as i32 + q as i32);
- if diff.abs() > 3 {
- m[idx] = T::from(-1e9f32).unwrap();
- }
- }
- }
- Some(m.into_boxed_slice())
- } else {
- None
- };
-
let scale = 1.0 / (head_dim as f32).sqrt();
Input {
queries: queries.into_boxed_slice(),
keys: keys.into_boxed_slice(),
values: values.into_boxed_slice(),
- mask,
num_heads,
gqa_factor,
sequence_length,
@@ -95,22 +72,21 @@ fn get_input<T: ArrayElement + Float>(
head_dim,
scale,
do_causal,
- has_mask: with_mask,
}
}
fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
- let float_mask = input.has_mask;
let kernel = <<B as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
&context,
T::data_type(),
input.head_dim,
- float_mask,
- input.has_mask,
+ false,
false,
input.do_causal,
+ false,
+ false,
)
.expect("Failed to create AttentionSinglePassKernel");
@@ -121,15 +97,6 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let output_size = (input.suffix_length * input.num_heads * input.head_dim) as usize;
let output_array = context.create_array_uninitialized(&[output_size], T::data_type(), "");
- let mask_array = input.mask.as_ref().map(|m| context.create_array_from(&[m.len()], m, ""));
- let mask_buf_rc = mask_array.as_ref().map(|a| a.buffer());
- let mask_buf_borrow = mask_buf_rc.as_ref().map(|rc| rc.borrow());
- let mask_buffer: Option<&B::Buffer> = mask_buf_borrow.as_ref().map(|b| b.deref());
-
- let mask_kv_seq_stride: Option<u32> = input.has_mask.then_some(1);
- let mask_q_seq_stride: Option<u32> = input.has_mask.then_some(input.sequence_length);
- let mask_head_stride: Option<u32> = input.has_mask.then_some(0);
-
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
queries_array.buffer().borrow().deref(),
@@ -142,11 +109,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
input.head_dim,
input.sequence_length * input.head_dim,
input.head_dim,
+ None,
input.scale,
- mask_buffer,
- mask_kv_seq_stride,
- mask_q_seq_stride,
- mask_head_stride,
+ None::<&B::Buffer>,
+ None,
None::<&B::Buffer>,
input.num_heads,
input.suffix_length,
@@ -170,68 +136,56 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
for_each_non_cpu_backend!(|B| {
let output = get_output::<T, B>(input);
let msg = format!(
- "AttentionSinglePass failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={}, mask={})",
+ "AttentionSinglePass failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={})",
std::any::type_name::<B>(),
input.num_heads,
input.sequence_length,
input.suffix_length,
input.head_dim,
input.do_causal,
- input.has_mask,
);
assert_eq_float::<T>(expected, &output, eps, &msg);
});
}
fn test_basic<T: ArrayElement + Float + Debug + Display>() {
- // Non-causal, no mask, single token
- let input = get_input::<T>(4, 4, 8, 1, 64, false, false);
+ // Non-causal, single token
+ let input = get_input::<T>(4, 4, 8, 1, 64, false);
let expected = get_output::<T, Cpu>(&input);
test_internal(&input, &expected);
- // Non-causal, no mask, multiple tokens
- let input = get_input::<T>(4, 4, 8, 4, 64, false, false);
+ // Non-causal, multiple tokens
+ let input = get_input::<T>(4, 4, 8, 4, 64, false);
let expected = get_output::<T, Cpu>(&input);
test_internal(&input, &expected);
}
fn test_causal<T: ArrayElement + Float + Debug + Display>() {
// Causal, single token decode
- let input = get_input::<T>(4, 4, 16, 1, 64, true, false);
+ let input = get_input::<T>(4, 4, 16, 1, 64, true);
let expected = get_output::<T, Cpu>(&input);
test_internal(&input, &expected);
// Causal, multi-token prefill
- let input = get_input::<T>(4, 4, 8, 4, 64, true, false);
+ let input = get_input::<T>(4, 4, 8, 4, 64, true);
let expected = get_output::<T, Cpu>(&input);
test_internal(&input, &expected);
}
fn test_gqa<T: ArrayElement + Float + Debug + Display>() {
// GQA: 8 query heads, 2 kv heads
- let input = get_input::<T>(8, 2, 8, 1, 64, false, false);
+ let input = get_input::<T>(8, 2, 8, 1, 64, false);
let expected = get_output::<T, Cpu>(&input);
test_internal(&input, &expected);
// GQA causal
- let input = get_input::<T>(8, 2, 8, 4, 64, true, false);
- let expected = get_output::<T, Cpu>(&input);
- test_internal(&input, &expected);
-}
-
-fn test_mask<T: ArrayElement + Float + Debug + Display>() {
- let input = get_input::<T>(4, 4, 8, 4, 64, false, true);
- let expected = get_output::<T, Cpu>(&input);
- test_internal(&input, &expected);
-
- // Causal + mask
- let input = get_input::<T>(4, 4, 8, 4, 64, true, true);
+ let input = get_input::<T>(8, 2, 8, 4, 64, true);
let expected = get_output::<T, Cpu>(&input);
test_internal(&input, &expected);
}
fn test_head_dim_128<T: ArrayElement + Float + Debug + Display>() {
- let input = get_input::<T>(4, 4, 8, 2, 128, true, false);
+ let input = get_input::<T>(4, 4, 8, 2, 128, true);
let expected = get_output::<T, Cpu>(&input);
test_internal(&input, &expected);
}
@@ -284,22 +238,6 @@ fn test_gqa_bf16() {
test_gqa::<bf16>();
}
-// Mask tests
-#[test]
-fn test_mask_f32() {
- test_mask::<f32>();
-}
-
-#[test]
-fn test_mask_f16() {
- test_mask::<f16>();
-}
-
-#[test]
-fn test_mask_bf16() {
- test_mask::<bf16>();
-}
-
// Head dim 128
#[test]
fn test_head_dim_128_f32() {
diff --git a/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs
index f9b2936fa..b6cec168c 100644
--- a/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs
+++ b/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs
@@ -26,7 +26,6 @@ struct FirstPassInput<T: ArrayElement + Float> {
queries: Box<[T]>,
keys: Box<[T]>,
values: Box<[T]>,
- mask: Option<Box<[T]>>,
num_heads: u32,
gqa_factor: u32,
sequence_length: u32,
@@ -34,7 +33,6 @@ struct FirstPassInput<T: ArrayElement + Float> {
head_dim: u32,
scale: f32,
do_causal: bool,
- has_mask: bool,
}
struct FirstPassOutput {
@@ -50,7 +48,6 @@ fn get_first_pass_input<T: ArrayElement + Float>(
suffix_length: u32,
head_dim: u32,
do_causal: bool,
- with_mask: bool,
) -> FirstPassInput<T> {
let gqa_factor = num_heads / num_kv_heads;
@@ -72,30 +69,12 @@ fn get_first_pass_input<T: ArrayElement + Float>(
values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();
}
- let mask = if with_mask {
- let m_size = (suffix_length * sequence_length) as usize;
- let mut m = vec![T::zero(); m_size];
- for q in 0..suffix_length {
- for k in 0..sequence_length {
- let idx = (q * sequence_length + k) as usize;
- let diff = (k as i32) - (sequence_length as i32 - suffix_length as i32 + q as i32);
- if diff.abs() > 3 {
- m[idx] = T::from(-1e9f32).unwrap();
- }
- }
- }
- Some(m.into_boxed_slice())
- } else {
- None
- };
-
let scale = 1.0 / (head_dim as f32).sqrt();
FirstPassInput {
queries: queries.into_boxed_slice(),
keys: keys.into_boxed_slice(),
values: values.into_boxed_slice(),
- mask,
num_heads,
gqa_factor,
sequence_length,
@@ -103,22 +82,21 @@ fn get_first_pass_input<T: ArrayElement + Float>(
head_dim,
scale,
do_causal,
- has_mask: with_mask,
}
}
fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassInput<T>) -> FirstPassOutput {
let context = B::Context::new().expect("Failed to create Context");
- let float_mask = input.has_mask;
let kernel = <<B as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
&context,
T::data_type(),
input.head_dim,
- float_mask,
- input.has_mask,
+ false,
false,
input.do_causal,
+ false,
+ false,
)
.expect("Failed to create AttentionTwoPass1Kernel");
@@ -135,15 +113,6 @@ fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassI
let sums_array = context.create_array_uninitialized(&[sums_size], DataType::F32, "");
let maxs_array = context.create_array_uninitialized(&[maxs_size], DataType::F32, "");
- let mask_array = input.mask.as_ref().map(|m| context.create_array_from(&[m.len()], m, ""));
- let mask_buf_rc = mask_array.as_ref().map(|a| a.buffer());
- let mask_buf_borrow = mask_buf_rc.as_ref().map(|rc| rc.borrow());
- let mask_buffer: Option<&B::Buffer> = mask_buf_borrow.as_ref().map(|b| b.deref());
-
- let mask_kv_seq_stride: Option<u32> = input.has_mask.then_some(1);
- let mask_q_seq_stride: Option<u32> = input.has_mask.then_some(input.sequence_length);
- let mask_head_stride: Option<u32> = input.has_mask.then_some(0);
-
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
queries_array.buffer().borrow().deref(),
@@ -158,13 +127,12 @@ fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassI
input.head_dim,
input.sequence_length * input.head_dim,
input.head_dim,
+ None,
input.scale,
input.num_heads,
input.suffix_length,
- mask_buffer,
- mask_kv_seq_stride,
- mask_q_seq_stride,
- mask_head_stride,
+ None::<&B::Buffer>,
+ None,
None::<&B::Buffer>,
&mut encoder,
);
@@ -268,14 +236,13 @@ fn test_first_pass_internal<T: ArrayElement + Float + Debug + Display>(
for_each_non_cpu_backend!(|B| {
let output = get_first_pass_output::<T, B>(input);
let msg = format!(
- "AttentionTwoPass1 failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={}, mask={})",
+ "AttentionTwoPass1 failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={})",
std::any::type_name::<B>(),
input.num_heads,
input.sequence_length,
input.suffix_length,
input.head_dim,
input.do_causal,
- input.has_mask,
);
assert_eq_float::<f32>(&expected.partials, &output.partials, eps, &format!("{msg} [partials]"));
assert_eq_float::<f32>(&expected.sums, &output.sums, eps, &format!("{msg} [sums]"));
@@ -284,47 +251,37 @@ fn test_first_pass_internal<T: ArrayElement + Float + Debug + Display>(
}
fn test_first_pass_basic<T: ArrayElement + Float + Debug + Display>() {
- let input = get_first_pass_input::<T>(4, 4, 8, 1, 64, false, false);
+ let input = get_first_pass_input::<T>(4, 4, 8, 1, 64, false);
let expected = get_first_pass_output::<T, Cpu>(&input);
test_first_pass_internal(&input, &expected);
- let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, false, false);
+ let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, false);
let expected = get_first_pass_output::<T, Cpu>(&input);
test_first_pass_internal(&input, &expected);
}
fn test_first_pass_causal<T: ArrayElement + Float + Debug + Display>() {
- let input = get_first_pass_input::<T>(4, 4, 16, 1, 64, true, false);
+ let input = get_first_pass_input::<T>(4, 4, 16, 1, 64, true);
let expected = get_first_pass_output::<T, Cpu>(&input);
test_first_pass_internal(&input, &expected);
- let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, true, false);
+ let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, true);
let expected = get_first_pass_output::<T, Cpu>(&input);
test_first_pass_internal(&input, &expected);
}
fn test_first_pass_gqa<T: ArrayElement + Float + Debug + Display>() {
- let input = get_first_pass_input::<T>(8, 2, 8, 1, 64, false, false);
+ let input = get_first_pass_input::<T>(8, 2, 8, 1, 64, false);
let expected = get_first_pass_output::<T, Cpu>(&input);
test_first_pass_internal(&input, &expected);
- let input = get_first_pass_input::<T>(8, 2, 8, 4, 64, true, false);
- let expected = get_first_pass_output::<T, Cpu>(&input);
- test_first_pass_internal(&input, &expected);
-}
-
-fn test_first_pass_mask<T: ArrayElement + Float + Debug + Display>() {
- let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, false, true);
- let expected = get_first_pass_output::<T, Cpu>(&input);
- test_first_pass_internal(&input, &expected);
-
- let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, true, true);
+ let input = get_first_pass_input::<T>(8, 2, 8, 4, 64, true);
let expected = get_first_pass_output::<T, Cpu>(&input);
test_first_pass_internal(&input, &expected);
}
fn test_first_pass_head_dim_128<T: ArrayElement + Float + Debug + Display>() {
- let input = get_first_pass_input::<T>(4, 4, 8, 2, 128, true, false);
+ let input = get_first_pass_input::<T>(4, 4, 8, 2, 128, true);
let expected = get_first_pass_output::<T, Cpu>(&input);
test_first_pass_internal(&input, &expected);
}
@@ -417,21 +374,6 @@ fn test_first_pass_gqa_bf16() {
test_first_pass_gqa::<bf16>();
}
-#[test]
-fn test_first_pass_mask_f32() {
- test_first_pass_mask::<f32>();
-}
-
-#[test]
-fn test_first_pass_mask_f16() {
- test_first_pass_mask::<f16>();
-}
-
-#[test]
-fn test_first_pass_mask_bf16() {
- test_first_pass_mask::<bf16>();
-}
-
#[test]
fn test_first_pass_head_dim_128_f32() {
test_first_pass_head_dim_128::<f32>();
diff --git a/crates/uzu/tests/unit/kernel/mask_update_test.rs b/crates/uzu/tests/unit/kernel/mask_update_test.rs
deleted file mode 100644
index f4113d5ab..000000000
--- a/crates/uzu/tests/unit/kernel/mask_update_test.rs
+++ /dev/null
@@ -1,240 +0,0 @@
-use std::{
- fmt::{Debug, Display},
- ops::DerefMut,
-};
-
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu::{
- ArrayContextExt, ArrayElement,
- backends::{
- common::{Backend, Context, Encoder, Kernels, kernel::MaskUpdateKernel},
- cpu::Cpu,
- },
-};
-
-use crate::common::assert::assert_eq_float;
-
-struct Input<T: ArrayElement + Float> {
- mask: Box<[T]>,
- unmask_col: i32,
- mask_col: i32,
-}
-
-fn make_mask<T: ArrayElement + Float>(size: usize) -> Vec<T> {
- let mut mask = Vec::with_capacity(size);
- for i in 0..size {
- mask.push(T::from(i as f32 * 0.5).unwrap());
- }
- mask
-}
-
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
- let context = B::Context::new().expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::MaskUpdateKernel::new(&context, T::data_type())
- .expect("Failed to create MaskUpdateKernel");
-
- let len = input.mask.len();
- let mask_array = context.create_array_from(&[len], &input.mask, "");
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");
- kernel.encode(mask_array.buffer().borrow_mut().deref_mut(), input.unmask_col, input.mask_col, &mut encoder);
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- mask_array.as_slice().to_vec()
-}
-
-/// Unmask one column and mask another.
-fn get_test_data_unmask_and_mask<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
- let size = 8;
- let mask = make_mask::<T>(size);
-
- let input = Input {
- mask: mask.into_boxed_slice(),
- unmask_col: 2,
- mask_col: 5,
- };
-
- let expected = get_output::<T, Cpu>(&input);
- (input, expected)
-}
-
-/// Only unmask (mask_col negative).
-fn get_test_data_unmask_only<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
- let size = 8;
- let mask = make_mask::<T>(size);
-
- let input = Input {
- mask: mask.into_boxed_slice(),
- unmask_col: 3,
- mask_col: -1,
- };
-
- let expected = get_output::<T, Cpu>(&input);
- (input, expected)
-}
-
-/// Only mask (unmask_col negative).
-fn get_test_data_mask_only<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
- let size = 8;
- let mask = make_mask::<T>(size);
-
- let input = Input {
- mask: mask.into_boxed_slice(),
- unmask_col: -1,
- mask_col: 4,
- };
-
- let expected = get_output::<T, Cpu>(&input);
- (input, expected)
-}
-
-/// Both columns negative — data should remain unchanged.
-fn get_test_data_no_op<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
- let size = 8;
- let mask = make_mask::<T>(size);
-
- let expected = mask.clone();
-
- let input = Input {
- mask: mask.into_boxed_slice(),
- unmask_col: -1,
- mask_col: -1,
- };
-
- (input, expected)
-}
-
-/// Larger mask to exercise more threads.
-fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
- let size = 1024;
- let mask = make_mask::<T>(size);
-
- let input = Input {
- mask: mask.into_boxed_slice(),
- unmask_col: 100,
- mask_col: 900,
- };
-
- let expected = get_output::<T, Cpu>(&input);
- (input, expected)
-}
-
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
- input: &Input<T>,
- expected: &[T],
- test_name: &str,
-) {
- let eps = 1e-5;
- let expected_norm = expected.to_vec();
- for_each_non_cpu_backend!(|B| {
- let actual = get_output::<T, B>(input);
- let msg = format!("MaskUpdate {} failed with backend={}", test_name, std::any::type_name::<B>());
- assert_eq_float::<T>(&expected_norm, &actual, eps, &msg);
- });
-}
-
-fn test_unmask_and_mask_internal<T: ArrayElement + Float + Debug + Display>() {
- let (input, expected) = get_test_data_unmask_and_mask::<T>();
- test_internal::<T>(&input, &expected, "unmask_and_mask");
-}
-
-fn test_unmask_only_internal<T: ArrayElement + Float + Debug + Display>() {
- let (input, expected) = get_test_data_unmask_only::<T>();
- test_internal::<T>(&input, &expected, "unmask_only");
-}
-
-fn test_mask_only_internal<T: ArrayElement + Float + Debug + Display>() {
- let (input, expected) = get_test_data_mask_only::<T>();
- test_internal::<T>(&input, &expected, "mask_only");
-}
-
-fn test_no_op_internal<T: ArrayElement + Float + Debug + Display>() {
- let (input, expected) = get_test_data_no_op::<T>();
- test_internal::<T>(&input, &expected, "no_op");
-}
-
-fn test_large_internal<T: ArrayElement + Float + Debug + Display>() {
- let (input, expected) = get_test_data_large::<T>();
- test_internal::<T>(&input, &expected, "large");
-}
-
-// Unmask and mask tests
-#[test]
-fn test_unmask_and_mask_f32() {
- test_unmask_and_mask_internal::<f32>();
-}
-
-#[test]
-fn test_unmask_and_mask_f16() {
- test_unmask_and_mask_internal::<f16>();
-}
-
-#[test]
-fn test_unmask_and_mask_bf16() {
- test_unmask_and_mask_internal::<bf16>();
-}
-
-// Unmask only tests
-#[test]
-fn test_unmask_only_f32() {
- test_unmask_only_internal::<f32>();
-}
-
-#[test]
-fn test_unmask_only_f16() {
- test_unmask_only_internal::<f16>();
-}
-
-#[test]
-fn test_unmask_only_bf16() {
- test_unmask_only_internal::<bf16>();
-}
-
-// Mask only tests
-#[test]
-fn test_mask_only_f32() {
- test_mask_only_internal::<f32>();
-}
-
-#[test]
-fn test_mask_only_f16() {
- test_mask_only_internal::<f16>();
-}
-
-#[test]
-fn test_mask_only_bf16() {
- test_mask_only_internal::<bf16>();
-}
-
-// No-op tests
-#[test]
-fn test_no_op_f32() {
- test_no_op_internal::<f32>();
-}
-
-#[test]
-fn test_no_op_f16() {
- test_no_op_internal::<f16>();
-}
-
-#[test]
-fn test_no_op_bf16() {
- test_no_op_internal::<bf16>();
-}
-
-// Large dimension tests
-#[test]
-fn test_large_f32() {
- test_large_internal::<f32>();
-}
-
-#[test]
-fn test_large_f16() {
- test_large_internal::<f16>();
-}
-
-#[test]
-fn test_large_bf16() {
- test_large_internal::<bf16>();
-}
diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rs
index 34e21cfb0..178642bf1 100644
--- a/crates/uzu/tests/unit/kernel/mod.rs
+++ b/crates/uzu/tests/unit/kernel/mod.rs
@@ -9,7 +9,6 @@ mod audio;
mod embedding;
mod kv_cache_update_test;
mod layer_norm_test;
-mod mask_update_test;
mod matmul;
mod moe;
mod pooling;