A high-performance inference engine for AI models
remove attention mask (#284)
uuuvn4f1a4a78781e
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rsindex 42e39e5ca..befc554b2 100644--- a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs@@ -116,13 +116,7 @@ impl StructuredAudioCodecGraph {local_rope.update_data(&transformer_tree, "local_rope");}}- let scratch_buffers = ScratchBuffers::new(- context.as_ref(),- &decoder_config,- &model_shape,- max_sequence_length,- max_sequence_length,- );+ let scratch_buffers = ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_sequence_length);let (layers, output_norm) = Decoder::build_transformer_layers_and_norm(context.as_ref(),&decoder_config,@@ -349,7 +343,6 @@ impl StructuredAudioCodecGraph {runtime.shared_buffers.clone(),&token_ids,&token_positions,- false,1,);@@ -458,7 +451,6 @@ impl StructuredAudioCodecGraph {runtime.shared_buffers.clone(),&token_ids,&token_positions,- false,1,);diff --git a/crates/uzu/src/backends/common/gpu_types/attention.rs b/crates/uzu/src/backends/common/gpu_types/attention.rsindex f608e13d1..e575269cb 100644--- a/crates/uzu/src/backends/common/gpu_types/attention.rs+++ b/crates/uzu/src/backends/common/gpu_types/attention.rs@@ -34,16 +34,3 @@ pub struct AttnParams {/// Key tiling: k_len % BKpub k_rem: i32,}--/// Parameters describing the additive attention mask.-///-/// Mask is treated as a 2D matrix [qL, kL] with:-/// - per-batch base offset (`m_strides[0]`)-/// - per-head base offset (`m_strides[1]`)-/// - row stride in elements (`m_strides[2]`)-/// Column stride is assumed to be 1.-#[repr(C)]-#[derive(Debug, Default, Copy, Clone)]-pub struct AttnMaskParams {- pub m_strides: [i64; 3],-}diff --git a/crates/uzu/src/backends/common/gpu_types/mod.rs b/crates/uzu/src/backends/common/gpu_types/mod.rsindex 1111d902c..26f84c140 100644--- a/crates/uzu/src/backends/common/gpu_types/mod.rs+++ b/crates/uzu/src/backends/common/gpu_types/mod.rs@@ -9,6 +9,8 @@ pub mod attention;pub mod kv_cache_update;pub mod matmul;pub mod quantization;+pub mod ring;+pub mod trie;pub use activation_type::*;pub use argmax::*;diff --git a/crates/uzu/src/backends/common/gpu_types/ring.rs b/crates/uzu/src/backends/common/gpu_types/ring.rsnew file mode 100644index 000000000..d6739fac6--- /dev/null+++ b/crates/uzu/src/backends/common/gpu_types/ring.rs@@ -0,0 +1,6 @@+#[derive(Clone, Copy, Debug)]+#[repr(C)]+pub struct RingParams {+ pub ring_offset: u32,+ pub ring_length: u32,+}diff --git a/crates/uzu/src/backends/common/gpu_types/trie.rs b/crates/uzu/src/backends/common/gpu_types/trie.rsnew file mode 100644index 000000000..fe23a6ec0--- /dev/null+++ b/crates/uzu/src/backends/common/gpu_types/trie.rs@@ -0,0 +1,7 @@+#[derive(Clone, Copy, Debug)]+#[repr(C)]+pub struct TrieNode {+ pub trie_start: u32,+ pub trie_end: u32,+ pub height: u32,+}diff --git a/crates/uzu/src/backends/common/kernel/attention.rs b/crates/uzu/src/backends/common/kernel/attention.rsindex 339d104fd..868c04cfd 100644--- a/crates/uzu/src/backends/common/kernel/attention.rs+++ b/crates/uzu/src/backends/common/kernel/attention.rs@@ -7,7 +7,7 @@ use crate::{DataType,backends::common::{Backend, Encoder, Kernels,- gpu_types::{AttnMaskParams, AttnParams},+ gpu_types::{AttnParams, ring::RingParams},kernel::AttentionGemmKernel,},};@@ -15,19 +15,21 @@ use crate::{const BQ: usize = 32;pub struct AttentionGemmArguments<'a, B: Backend> {- pub queries_buffer: &'a B::Buffer, // buffer(0)- pub keys_buffer: &'a B::Buffer, // buffer(1)- pub values_buffer: &'a B::Buffer, // buffer(2)- pub output_buffer: &'a mut B::Buffer, // buffer(3)- pub mask_buffer: Option<&'a B::Buffer>, // buffer(6)- pub sinks_buffer: Option<&'a B::Buffer>, // buffer(7)+ pub queries_buffer: &'a B::Buffer,+ pub keys_buffer: &'a B::Buffer,+ pub values_buffer: &'a B::Buffer,+ pub output_buffer: &'a mut B::Buffer,+ pub trie_buffer: Option<&'a B::Buffer>,+ pub sinks_buffer: Option<&'a B::Buffer>,pub num_heads: usize,pub num_groups: usize,pub suffix_length: usize, // qLpub sequence_length: usize, // kL (prefix + suffix)pub segment_prefix_length: usize, // qL_offpub max_sequence_length: usize, // stride for K/V cache+ pub ring_params: Option<RingParams>,pub head_dim: usize,+ pub sliding_window_size: Option<usize>,pub is_causal: bool,pub scale: f32,}@@ -57,17 +59,23 @@ impl<B: Backend> AttentionGemmBlock<B> {} else {16};+ let head_dim = args.head_dim;let align_q = (args.suffix_length % BQ) == 0;let align_k = (args.sequence_length % bk) == 0;- let has_mask = args.mask_buffer.is_some();+ let is_kv_cache_ring = args.ring_params.is_some();+ let is_causal = args.is_causal;+ let is_trie = args.trie_buffer.is_some();+ let is_sliding_window = args.sliding_window_size.is_some();let has_sinks = args.sinks_buffer.is_some();let key = KernelKey {bk,- head_dim: args.head_dim,+ head_dim,align_q,align_k,- is_causal: args.is_causal,- has_mask,+ is_kv_cache_ring,+ is_causal,+ is_trie,+ is_sliding_window,has_sinks,};@@ -79,11 +87,13 @@ impl<B: Backend> AttentionGemmBlock<B> {context,self.data_type,bk as u32,- args.head_dim as u32,+ head_dim as u32,align_q,align_k,- args.is_causal,- has_mask,+ is_kv_cache_ring,+ is_causal,+ is_trie,+ is_sliding_window,has_sinks,)?;entry.insert(kernel)@@ -91,14 +101,14 @@ impl<B: Backend> AttentionGemmBlock<B> {};// Params (all strides in elements)- let q_head_stride = (args.suffix_length * args.head_dim) as i64;- let q_seq_stride = args.head_dim as i64;+ let q_head_stride = (args.suffix_length * head_dim) as i64;+ let q_seq_stride = head_dim as i64;- let kv_head_stride = (args.max_sequence_length * args.head_dim) as i64;- let kv_seq_stride = args.head_dim as i64;+ let kv_head_stride = (args.max_sequence_length * head_dim) as i64;+ let kv_seq_stride = head_dim as i64;- let o_head_stride = args.head_dim as i64;- let o_seq_stride = (args.num_heads * args.head_dim) as i64;+ let o_head_stride = head_dim as i64;+ let o_seq_stride = (args.num_heads * head_dim) as i64;let nk = (args.sequence_length + bk - 1) / bk;let nq_aligned = args.suffix_length / BQ;@@ -121,20 +131,15 @@ impl<B: Backend> AttentionGemmBlock<B> {k_rem: (args.sequence_length - nk_aligned * bk) as i32,};- let attn_mask_params = AttnMaskParams {- // We use a shared bias matrix for all heads/batches.- m_strides: [0, 0, args.sequence_length as i64],- };- let attn_mask_params_opt = args.mask_buffer.map(|_| attn_mask_params);-kernel.encode(args.queries_buffer,args.keys_buffer,args.values_buffer,args.output_buffer,params,- attn_mask_params_opt,- args.mask_buffer,+ args.ring_params,+ args.trie_buffer,+ args.sliding_window_size.map(|s| s as u32),args.sinks_buffer,args.num_heads as u32,args.suffix_length as u32,@@ -151,7 +156,9 @@ struct KernelKey {head_dim: usize,align_q: bool,align_k: bool,+ is_kv_cache_ring: bool,is_causal: bool,- has_mask: bool,+ is_trie: bool,+ is_sliding_window: bool,has_sinks: bool,}diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rsindex 15fb2c5d3..d7c096481 100644--- a/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rs+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_gemm.rs@@ -2,7 +2,10 @@ use dsl::kernel;use half::{bf16, f16};use num_traits::Float;-use crate::ArrayElement;+use crate::{+ ArrayElement,+ backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},+};#[kernel(AttentionGemm)]#[variants(T, f32, f16, bf16)]@@ -14,21 +17,29 @@ pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(v: *const T,o: *mut T,params: crate::backends::common::gpu_types::attention::AttnParams,- #[optional(has_mask)] mask_params: Option<crate::backends::common::gpu_types::attention::AttnMaskParams>,- #[optional(has_mask)] mask: Option<*const T>,+ #[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,+ #[optional(is_trie)] trie: Option<*const TrieNode>,+ #[optional(is_sliding_window)] sliding_window_size: Option<u32>,#[optional(has_sinks)] sinks: Option<*const f32>,num_heads: u32,suffix_length: u32,#[specialize] align_q: bool,#[specialize] align_k: bool,- #[specialize] do_causal: bool,- #[specialize] has_mask: bool,+ #[specialize] is_kv_cache_ring: bool,+ #[specialize] is_causal: bool,+ #[specialize] is_trie: bool,+ #[specialize] is_sliding_window: bool,#[specialize] has_sinks: bool,) {let q_len = params.q_len as usize;let k_len = params.k_len as usize;let head_dim = BD as usize;- let q_off = params.q_off as usize;+ let prefix_length = params.q_off as usize;+ let suffix_position = if is_kv_cache_ring {+ ring_params.unwrap().ring_length as usize+ } else {+ prefix_length+ };for head_idx in 0..num_heads as usize {let kv_head_idx = head_idx / params.gqa_factor as usize;@@ -42,6 +53,13 @@ pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(let q_row = unsafe { q_head.add(qi * params.q_strides[2] as usize) };let o_row = unsafe { o_head.add(qi * params.o_strides[2] as usize) };+ let query_position = if is_trie {+ let trie_node = unsafe { &*trie.unwrap().add(qi) };+ suffix_position + trie_node.height as usize+ } else {+ suffix_position + qi+ };+// Read query row and pre-scalelet mut q_vec = vec![0.0f32; head_dim];for j in 0..head_dim {@@ -60,8 +78,17 @@ pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(// Loop over all key positionsfor ki in 0..k_len {- // Causal mask: query at absolute position q_off + qi can only attend to ki <= q_off + qi- if do_causal && ki > q_off + qi {+ if !should_use_key(+ ring_params,+ trie,+ sliding_window_size,+ qi as u32,+ prefix_length as u32,+ suffix_position as u32,+ query_position as u32,+ ki as u32,+ is_causal,+ ) {continue;}@@ -72,15 +99,6 @@ pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(score += q_vec[j] * unsafe { *k_row.add(j) }.to_f32().unwrap();}- // Add external mask bias- if has_mask {- let mask_params = mask_params.unwrap();- let mask = mask.unwrap();- let mask_ptr = unsafe { mask.add(head_idx * mask_params.m_strides[1] as usize) };- let mv = unsafe { *mask_ptr.add(qi * mask_params.m_strides[2] as usize + ki) }.to_f32().unwrap();- score += f32::max(mv, -1e9);- }-// Online softmax updatelet new_max = f32::max(max_score, score);let factor = (max_score - new_max).exp();diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rsindex ffe5b8eac..786a4a744 100644--- a/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_single_pass.rs@@ -2,7 +2,10 @@ use dsl::kernel;use half::{bf16, f16};use num_traits::Float;-use crate::ArrayElement;+use crate::{+ ArrayElement,+ backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},+};#[kernel(AttentionSinglePass)]#[variants(T, f32, f16, bf16)]@@ -18,41 +21,46 @@ pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(k_seq_stride: u32,v_head_stride: u32,v_seq_stride: u32,+ #[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,scale: f32,- #[optional(float_mask)] fmask: Option<*const T>,- #[optional(has_mask)] mask_kv_seq_stride: Option<u32>,- #[optional(has_mask)] mask_q_seq_stride: Option<u32>,- #[optional(has_mask)] mask_head_stride: Option<u32>,+ #[optional(is_trie)] trie: Option<*const TrieNode>,+ #[optional(is_sliding_window)] sliding_window_size: Option<u32>,#[optional(has_sinks)] sinks: Option<*const f32>,num_heads: u32,suffix_length: u32,- #[specialize] float_mask: bool,- #[specialize] has_mask: bool,#[specialize] has_sinks: bool,- #[specialize] do_causal: bool,+ #[specialize] is_kv_cache_ring: bool,+ #[specialize] is_causal: bool,+ #[specialize] is_trie: bool,+ #[specialize] is_sliding_window: bool,) {let value_dim = HEAD_DIM;+ let prefix_length = sequence_length - suffix_length;+ let suffix_position = if let Some(ring_params) = ring_params {+ ring_params.ring_length+ } else {+ prefix_length+ };+for head_idx in 0..num_heads {for q_seq_idx in 0..suffix_length {let kv_head_idx = head_idx / gqa_factor;let o_offset = q_seq_idx * num_heads + head_idx;let q_offset = head_idx * suffix_length + q_seq_idx;+ let query_position = if is_trie {+ let trie_node = unsafe { &*trie.unwrap().add(q_seq_idx as usize) };+ suffix_position + trie_node.height+ } else {+ suffix_position + q_seq_idx+ };+let queries: *const T = unsafe { queries.add((q_offset * HEAD_DIM) as usize) };let keys: *const T = unsafe { keys.add((kv_head_idx * k_head_stride) as usize) };let values: *const T = unsafe { values.add((kv_head_idx * v_head_stride) as usize) };let out: *mut T = unsafe { out.add((o_offset * value_dim) as usize) };- let fmask: Option<*const T> = if let Some(fmask) = fmask {- unsafe {- let offset = head_idx * mask_head_stride.unwrap_or(0) + q_seq_idx * mask_q_seq_stride.unwrap_or(0);- Some(fmask.add(offset as usize))- }- } else {- None- };-// Read the query and 0 the output accumulatorlet mut q = vec![0.0f32; HEAD_DIM as usize];let mut o = vec![0.0f32; HEAD_DIM as usize];@@ -70,9 +78,17 @@ pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(// For each keyfor i in 0..sequence_length {- let use_key = !do_causal || i <= (sequence_length - suffix_length + q_seq_idx);-- if use_key {+ if should_use_key(+ ring_params,+ trie,+ sliding_window_size,+ q_seq_idx,+ prefix_length,+ suffix_position,+ query_position,+ i,+ is_causal,+ ) {let keys = unsafe { keys.add((i * k_seq_stride) as usize) };// Compute the i-th score@@ -80,10 +96,6 @@ pub fn attention_single_pass<T: ArrayElement + Float, const HEAD_DIM: u32>(for j in 0..HEAD_DIM as usize {score += q[j] * unsafe { *keys.add(j) }.to_f32().unwrap();}- if float_mask {- let fmask = unsafe { fmask.unwrap().add((i * mask_kv_seq_stride.unwrap()) as usize) };- score += f32::max(-1e9, unsafe { *fmask }.to_f32().unwrap());- }// Update the accumulatorslet new_max = f32::max(max_score, score);diff --git a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rsindex 3be696c10..ba23f1dd1 100644--- a/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs+++ b/crates/uzu/src/backends/cpu/kernel/attention/attention_two_pass.rs@@ -2,7 +2,10 @@ use dsl::kernel;use half::{bf16, f16};use num_traits::Float;-use crate::ArrayElement;+use crate::{+ ArrayElement,+ backends::{common::gpu_types::trie::TrieNode, cpu::kernel::attention::mask::should_use_key},+};const TOTAL_BLOCKS_COUNT: u32 = 32;const SEQUENCE_BLOCK_SIZE_1: u32 = 8;@@ -23,25 +26,39 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(k_seq_stride: u32,v_head_stride: u32,v_seq_stride: u32,+ #[optional(is_kv_cache_ring)] ring_params: Option<crate::backends::common::gpu_types::ring::RingParams>,scale: f32,num_heads: u32,suffix_length: u32,- #[optional(float_mask)] fmask: Option<*const T>,- #[optional(has_mask)] mask_kv_seq_stride: Option<u32>,- #[optional(has_mask)] mask_q_seq_stride: Option<u32>,- #[optional(has_mask)] mask_head_stride: Option<u32>,+ #[optional(is_trie)] trie: Option<*const TrieNode>,+ #[optional(is_sliding_window)] sliding_window_size: Option<u32>,#[optional(has_sinks)] sinks: Option<*const f32>,- #[specialize] float_mask: bool,- #[specialize] has_mask: bool,#[specialize] has_sinks: bool,- #[specialize] do_causal: bool,+ #[specialize] is_kv_cache_ring: bool,+ #[specialize] is_causal: bool,+ #[specialize] is_trie: bool,+ #[specialize] is_sliding_window: bool,) {let value_dim = HEAD_DIM;let mut q = vec![0.0f32; HEAD_DIM as usize];let mut o = vec![0.0f32; HEAD_DIM as usize];+ let prefix_length = sequence_length - suffix_length;+ let suffix_position = if let Some(ring_params) = ring_params {+ ring_params.ring_length+ } else {+ prefix_length+ };+for head_idx in 0..num_heads {for q_seq_idx in 0..suffix_length {+ let query_position = if is_trie {+ let trie_node = unsafe { &*trie.unwrap().add(q_seq_idx as usize) };+ suffix_position + trie_node.height+ } else {+ suffix_position + q_seq_idx+ };+for block_idx in 0..TOTAL_BLOCKS_COUNT {let o_offset = q_seq_idx * num_heads + head_idx;let q_offset = head_idx * suffix_length + q_seq_idx;@@ -53,13 +70,6 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(let out_base: *mut f32 =unsafe { out.add((o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim) as usize) };- let fmask_base: Option<*const T> = if float_mask {- let offset = head_idx * mask_head_stride.unwrap_or(0) + q_seq_idx * mask_q_seq_stride.unwrap_or(0);- Some(unsafe { fmask.unwrap().add(offset as usize) })- } else {- None- };-// Read the query and scalefor j in 0..HEAD_DIM as usize {q[j] = scale * unsafe { *queries_base.add(j) }.to_f32().unwrap();@@ -84,8 +94,17 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(break;}- let use_key = !do_causal || i <= (sequence_length - suffix_length + q_seq_idx);- if use_key {+ if should_use_key(+ ring_params,+ trie,+ sliding_window_size,+ q_seq_idx,+ prefix_length,+ suffix_position,+ query_position,+ i,+ is_causal,+ ) {let keys_ptr = unsafe { keys_base.add((i * k_seq_stride) as usize) };// Compute the i-th score@@ -93,11 +112,6 @@ pub fn attention_two_pass1<T: ArrayElement + Float, const HEAD_DIM: u32>(for j in 0..HEAD_DIM as usize {score += q[j] * unsafe { *keys_ptr.add(j) }.to_f32().unwrap();}- if float_mask {- let fmask_ptr =- unsafe { fmask_base.unwrap().add((i * mask_kv_seq_stride.unwrap()) as usize) };- score += f32::max(-1e9, unsafe { *fmask_ptr }.to_f32().unwrap());- }// Update the accumulatorslet new_max = f32::max(max_score, score);diff --git a/crates/uzu/src/backends/cpu/kernel/attention/mask.rs b/crates/uzu/src/backends/cpu/kernel/attention/mask.rsnew file mode 100644index 000000000..8ebb69313--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/attention/mask.rs@@ -0,0 +1,62 @@+use crate::backends::common::gpu_types::{ring::RingParams, trie::TrieNode};++pub fn should_use_key(+ ring_params: Option<RingParams>,+ trie: Option<*const TrieNode>,+ sliding_window_size: Option<u32>,+ q_seq_idx: u32,+ prefix_length: u32,+ suffix_position: u32,+ query_position: u32,+ i: u32,+ is_causal: bool,+) -> bool {+ let mut use_key = true;+ let key_position;++ if i >= prefix_length {+ // We're in the suffix+ let key_position_in_suffix = i - prefix_length;++ if let Some(trie) = trie {+ // Trie, get position from the flat trie buffer+ let trie_node = unsafe { &*trie.add(key_position_in_suffix as usize) };++ key_position = suffix_position + trie_node.height;++ if is_causal {+ use_key &= q_seq_idx >= trie_node.trie_start && q_seq_idx <= trie_node.trie_end;+ }+ } else {+ // Non-trie, position equals index in the suffix+ key_position = suffix_position + key_position_in_suffix;++ if is_causal {+ use_key &= key_position_in_suffix <= q_seq_idx;+ }+ }+ } else {+ // We're in the prefix+ if let Some(ring_params) = ring_params {+ // Ring, calculate position in the ring+ key_position = (prefix_length + i - ring_params.ring_offset) % prefix_length;+ // Ring also skips entries that aren't filled yet+ use_key &= key_position < ring_params.ring_length;+ } else {+ // Non-ring, position equals to the index in the kvcache buffer+ key_position = i;+ }+ }++ if let Some(sliding_window_size) = sliding_window_size {+ if is_causal {+ use_key &= key_position <= query_position && (query_position - key_position) < sliding_window_size;+ } else if key_position <= query_position {+ use_key &= (query_position - key_position) <= sliding_window_size / 2;+ } else {+ use_key &= (key_position - query_position) <= sliding_window_size / 2;+ }+ }++ use_key+}diff --git a/crates/uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/uzu/src/backends/cpu/kernel/attention/mod.rsindex 4a6321bee..c862aeaf0 100644--- a/crates/uzu/src/backends/cpu/kernel/attention/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/attention/mod.rs@@ -3,3 +3,5 @@ pub mod attention_single_pass;pub mod attention_two_pass;pub mod attention_update_kv_cache;pub mod sigmoid_gate;++mod mask;diff --git a/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs b/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rsdeleted file mode 100644index cb7733683..000000000--- a/crates/uzu/src/backends/cpu/kernel/mask_update/mask_update.rs+++ /dev/null@@ -1,20 +0,0 @@-use dsl::kernel;-use half::{bf16, f16};-use num_traits::Float;--use crate::ArrayElement;--#[kernel(MaskUpdate)]-#[variants(T, f32, f16, bf16)]-pub fn mask_update<T: ArrayElement + Float>(- mask: *mut T,- unmask_col: i32,- mask_col: i32,-) {- if (unmask_col >= 0) {- unsafe { *mask.add(unmask_col as usize) = T::zero() };- }- if (mask_col >= 0) {- unsafe { *mask.add(mask_col as usize) = T::neg_infinity() };- }-}diff --git a/crates/uzu/src/backends/cpu/kernel/mask_update/mod.rs b/crates/uzu/src/backends/cpu/kernel/mask_update/mod.rsdeleted file mode 100644index 75a6d52a2..000000000--- a/crates/uzu/src/backends/cpu/kernel/mask_update/mod.rs+++ /dev/null@@ -1 +0,0 @@-pub mod mask_update;diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rsindex 2d9020b1f..f6b5610f5 100644--- a/crates/uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs@@ -5,7 +5,7 @@ mod audio;mod embedding;mod kv_cache_update;mod layer_norm;-mod mask_update;+mod matmul;mod mlp;mod moe;diff --git a/crates/uzu/src/backends/metal/generated/attention.h b/crates/uzu/src/backends/metal/generated/attention.hindex e3167fe4c..7e3e3b787 100644--- a/crates/uzu/src/backends/metal/generated/attention.h+++ b/crates/uzu/src/backends/metal/generated/attention.h@@ -46,16 +46,6 @@ typedef struct {int32_t k_rem;} AttnParams;-/**Parameters describing the additive attention mask. */-/**Mask is treated as a 2D matrix [qL, kL] with: */-/**- per-batch base offset (`m_strides[0]`) */-/**- per-head base offset (`m_strides[1]`) */-/**- row stride in elements (`m_strides[2]`) */-/**Column stride is assumed to be 1. */-typedef struct {- int64_t m_strides[3];-} AttnMaskParams;-#ifdef __METAL_VERSION__} // namespace attention} // namespace uzudiff --git a/crates/uzu/src/backends/metal/generated/ring.h b/crates/uzu/src/backends/metal/generated/ring.hnew file mode 100644index 000000000..6819327e9--- /dev/null+++ b/crates/uzu/src/backends/metal/generated/ring.h@@ -0,0 +1,27 @@+// Auto-generated from gpu_types/ring.rs - do not edit manually+#pragma once++#ifndef UZU_RING_H+#define UZU_RING_H++#ifdef __METAL_VERSION__+#include <metal_stdlib>+using namespace metal;++namespace uzu {+namespace ring {+#else+#include <stdint.h>+#endif++typedef struct {+ uint32_t ring_offset;+ uint32_t ring_length;+} RingParams;++#ifdef __METAL_VERSION__+} // namespace ring+} // namespace uzu+#endif++#endif // UZU_RING_Hdiff --git a/crates/uzu/src/backends/metal/generated/trie.h b/crates/uzu/src/backends/metal/generated/trie.hnew file mode 100644index 000000000..2c2ab2497--- /dev/null+++ b/crates/uzu/src/backends/metal/generated/trie.h@@ -0,0 +1,28 @@+// Auto-generated from gpu_types/trie.rs - do not edit manually+#pragma once++#ifndef UZU_TRIE_H+#define UZU_TRIE_H++#ifdef __METAL_VERSION__+#include <metal_stdlib>+using namespace metal;++namespace uzu {+namespace trie {+#else+#include <stdint.h>+#endif++typedef struct {+ uint32_t trie_start;+ uint32_t trie_end;+ uint32_t height;+} TrieNode;++#ifdef __METAL_VERSION__+} // namespace trie+} // namespace uzu+#endif++#endif // UZU_TRIE_Hdiff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex 57009d9ae..acf5ff8d6 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -3,11 +3,16 @@#include "../common/thread_context.h"#include "../matmul/common/loader.h"#include "../matmul/common/simdgroup_fragment.h"-#include "attention.h"+#include "../generated/ring.h"+#include "../generated/trie.h"+#include "../generated/attention.h"+#include "mask.h"using namespace metal;using namespace uzu::matmul;using namespace uzu::attention;+using namespace uzu::ring;+using namespace uzu::trie;template <typename T>struct TransformScale {@@ -46,37 +51,37 @@ PUBLIC KERNEL(AttentionGemm)(const device T* v,device T* o,const constant AttnParams& params,- const constant AttnMaskParams& mask_params OPTIONAL(has_mask),- const device T* mask OPTIONAL(has_mask),+ const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),+ const device TrieNode* trie OPTIONAL(is_trie),+ const constant uint& sliding_window_size OPTIONAL(is_sliding_window),const device float* sinks OPTIONAL(has_sinks),const constant uint& num_heads,const constant uint& suffix_length,const bool align_q SPECIALIZE,const bool align_k SPECIALIZE,- const bool do_causal SPECIALIZE,- const bool has_mask SPECIALIZE,+ const bool is_kv_cache_ring SPECIALIZE,+ const bool is_causal SPECIALIZE,+ const bool is_trie SPECIALIZE,+ const bool is_sliding_window SPECIALIZE,const bool has_sinks SPECIALIZE,threadgroup T q_smem[BLOCK_QUERY_ROWS * (BD + 16 / sizeof(T))],threadgroup T kv_smem[BK * (BD + 16 / sizeof(T))],const ThreadContext thread_context,- const uint tgid_x GROUPS(suffix_length.div_ceil(BLOCK_QUERY_ROWS)),- const uint tgid_y GROUPS(num_heads),- const uint tgid_z GROUPS(1),+ const uint q_tile_idx GROUPS(suffix_length.div_ceil(BLOCK_QUERY_ROWS)),+ const uint head_idx GROUPS(num_heads),+ const uint batch_idx GROUPS(1),const uint lid THREADS(128)) {// -------------------------------------------------------------------------// Pointer setup (all strides are in elements)- // tgid_x: query tile index (BLOCK_QUERY_ROWS rows)- // tgid_y: query head index- // tgid_z: batch index (currently 1 in uzu, but kept for completeness)- const uint batch_idx = tgid_z;- const uint head_idx = tgid_y;- const uint q_tile_idx = tgid_x;+ // q_tile_idx: query tile index (BLOCK_QUERY_ROWS rows)+ // head_idx: query head index+ // batch_idx: batch index (currently 1 in uzu, but kept for completeness)q += batch_idx * params.q_strides[0] + head_idx * params.q_strides[1] +q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.q_strides[2];- const int kv_head_idx = int(tgid_y) / params.gqa_factor;+ const int kv_head_idx = int(head_idx) / params.gqa_factor;k += batch_idx * params.k_strides[0] +int64_t(kv_head_idx) * params.k_strides[1];v += batch_idx * params.v_strides[0] +@@ -85,9 +90,8 @@ PUBLIC KERNEL(AttentionGemm)(o += batch_idx * params.o_strides[0] + head_idx * params.o_strides[1] +q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.o_strides[2];- if (has_mask) {- mask += batch_idx * mask_params.m_strides[0] +- head_idx * mask_params.m_strides[1];+ if (is_trie) {+ trie += batch_idx * suffix_length;}// -------------------------------------------------------------------------@@ -242,7 +246,7 @@ PUBLIC KERNEL(AttentionGemm)(// Load Q block once (and apply scaling)threadgroup_barrier(mem_flags::mem_threadgroup);- if (!align_q && int(tgid_x) == params.nq_aligned) {+ if (!align_q && int(q_tile_idx) == params.nq_aligned) {query_loader.load_safe(short2(BD, params.q_rem));} else {query_loader.load_unsafe();@@ -256,21 +260,29 @@ PUBLIC KERNEL(AttentionGemm)(AccumType sum_score = AccumType(0);if (has_sinks) {- max_score = M_LOG2E_F * static_cast<AccumType>(sinks[tgid_y]);+ max_score = M_LOG2E_F * static_cast<AccumType>(sinks[head_idx]);sum_score = AccumType(1);}// Determine K block loop limit (causal can early-stop)int kb_lim = params.nk;- if (do_causal) {- const int q_max = (int(tgid_x) + 1) * BLOCK_QUERY_ROWS + params.q_off;+ if (is_causal) {+ const int q_max = (int(q_tile_idx) + 1) * BLOCK_QUERY_ROWS + params.q_off;kb_lim = (q_max + BK - 1) / BK;kb_lim = min(params.nk, kb_lim);}- const int q_rel = int(tgid_x) * BLOCK_QUERY_ROWS + int(simdgroup_row_base) +- int(lane_row); // [0, q_len)- const int q_abs = q_rel + params.q_off; // [0, k_len)+ const int q_rel = int(q_tile_idx) * BLOCK_QUERY_ROWS ++ int(simdgroup_row_base) + int(lane_row); // [0, q_len)++ const int prefix_length = params.q_off;++ const int suffix_position =+ is_kv_cache_ring ? int(ring_params.ring_length) : prefix_length;++ const int query_position = (is_trie && q_rel < params.q_len)+ ? suffix_position + int(trie[q_rel].height)+ : suffix_position + q_rel;// Loop over KV blocksfor (int kb = 0; kb < kb_lim; kb++) {@@ -323,31 +335,8 @@ PUBLIC KERNEL(AttentionGemm)(}}- // Causal mask (only needed for the last few blocks near the diagonal)- if (do_causal) {- const int tail_blocks = (BLOCK_QUERY_ROWS + BK - 1) / BK + int(!align_k);- const int tail_start = kb_lim - tail_blocks;- if (kb >= tail_start) {- METAL_PRAGMA_UNROLL- for (short j = 0; j < KEY_GRID_COLS; j++) {- thread auto& frag = score_fragment.multiply_accumulate_at(0, j);- const int col_base =- kb * BK + int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;- if (q_abs < col_base) {- frag[0] = neg_inf;- }- if (q_abs < (col_base + 1)) {- frag[1] = neg_inf;- }- }- }- }-- // Add external mask (additive bias in natural-log domain; convert to log2)- if (has_mask && q_rel < params.q_len) {- const int64_t row_stride = mask_params.m_strides[2];- const int64_t row_base = int64_t(q_rel) * row_stride;-+ // Unified masking: causal, trie, ring, sliding window+ if (q_rel < params.q_len) {METAL_PRAGMA_UNROLLfor (short j = 0; j < KEY_GRID_COLS; j++) {thread auto& frag = score_fragment.multiply_accumulate_at(0, j);@@ -357,16 +346,37 @@ PUBLIC KERNEL(AttentionGemm)(const int k0 = col_base;const int k1 = col_base + 1;- if (k0 < params.k_len) {- AccumType mv = static_cast<AccumType>(mask[row_base + int64_t(k0)]);- mv = metal::max(mv, static_cast<AccumType>(-1e9f));- frag[0] += M_LOG2E_F * mv;- }- if (k1 < params.k_len) {- AccumType mv = static_cast<AccumType>(mask[row_base + int64_t(k1)]);- mv = metal::max(mv, static_cast<AccumType>(-1e9f));- frag[1] += M_LOG2E_F * mv;- }+ if (k0 < params.k_len && !should_use_key(+ ring_params,+ trie,+ sliding_window_size,+ q_rel,+ prefix_length,+ suffix_position,+ query_position,+ k0,+ is_kv_cache_ring,+ is_causal,+ is_trie,+ is_sliding_window+ ))+ frag[0] = neg_inf;++ if (k1 < params.k_len && !should_use_key(+ ring_params,+ trie,+ sliding_window_size,+ q_rel,+ prefix_length,+ suffix_position,+ query_position,+ k1,+ is_kv_cache_ring,+ is_causal,+ is_trie,+ is_sliding_window+ ))+ frag[1] = neg_inf;}}@@ -466,7 +476,7 @@ PUBLIC KERNEL(AttentionGemm)(o += int64_t(simdgroup_row_base + lane_row) * params.o_strides[2] +int64_t(lane_col);- if (!align_q && int(tgid_x) == params.nq_aligned) {+ if (!align_q && int(q_tile_idx) == params.nq_aligned) {const short2 dst_tile_dims =short2(BD - lane_col, params.q_rem - (simdgroup_row_base + lane_row));diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metalindex bec72e50c..5f75bbfde 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -2,10 +2,16 @@#include <metal_simdgroup>#include "../common/dsl.h"#include "../common/thread_context.h"+#include "../generated/ring.h"+#include "../generated/trie.h"+#include "mask.h"#define SEQUENCE_BLOCK_SIZE 32#define HEAD_BLOCK_SIZE 32+using namespace uzu::ring;+using namespace uzu::trie;+template <typename T, uint HEAD_DIM>VARIANTS(T, float, half, bfloat)VARIANTS(HEAD_DIM, 64, 128, 256)@@ -20,27 +26,26 @@ PUBLIC KERNEL(AttentionSinglePass)(const constant uint& k_seq_stride,const constant uint& v_head_stride,const constant uint& v_seq_stride,+ const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),const constant float& scale,- const device T* fmask OPTIONAL(float_mask),- const constant uint& mask_kv_seq_stride OPTIONAL(has_mask),- const constant uint& mask_q_seq_stride OPTIONAL(has_mask),- const constant uint& mask_head_stride OPTIONAL(has_mask),+ const device TrieNode* trie OPTIONAL(is_trie),+ const constant uint& sliding_window_size OPTIONAL(is_sliding_window),const device float* sinks OPTIONAL(has_sinks),const constant uint& num_heads,const constant uint& suffix_length,threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE * HEAD_BLOCK_SIZE],- const bool float_mask SPECIALIZE,- const bool has_mask SPECIALIZE,const bool has_sinks SPECIALIZE,- const bool do_causal SPECIALIZE,+ const bool is_kv_cache_ring SPECIALIZE,+ const bool is_causal SPECIALIZE,+ const bool is_trie SPECIALIZE,+ const bool is_sliding_window SPECIALIZE,const ThreadContext thread_context,const uint head_idx GROUPS(num_heads),const uint q_seq_idx GROUPS(suffix_length),const uint tid THREADS(1024)) {- const uint3 tpg = {num_heads, suffix_length, 1};constexpr bool query_transposed = false;constexpr uint value_dim = HEAD_DIM;@@ -56,9 +61,17 @@ PUBLIC KERNEL(AttentionSinglePass)(thread U o[value_elements_per_thread];const uint kv_head_idx = head_idx / gqa_factor;- const uint o_offset = q_seq_idx * tpg.x + head_idx;- const uint q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx- : head_idx * tpg.y + q_seq_idx;+ const uint o_offset = q_seq_idx * num_heads + head_idx;+ const uint q_offset = query_transposed ? num_heads * q_seq_idx + head_idx+ : head_idx * suffix_length + q_seq_idx;++ const uint prefix_length = sequence_length - suffix_length;++ const uint suffix_position =+ is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;++ const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height+ : suffix_position + q_seq_idx;queries += q_offset * HEAD_DIM +thread_context.simdgroup_index * qk_elements_per_thread;@@ -69,12 +82,6 @@ PUBLIC KERNEL(AttentionSinglePass)(thread_context.threadgroup_index * v_seq_stride +thread_context.simdgroup_index * value_elements_per_thread;- if (float_mask) {- fmask += head_idx * mask_head_stride +- thread_context.threadgroup_index * mask_kv_seq_stride +- q_seq_idx * mask_q_seq_stride;- }-out += o_offset * value_dim +thread_context.threadgroup_index * value_elements_per_thread;@@ -89,7 +96,7 @@ PUBLIC KERNEL(AttentionSinglePass)(U max_score = -INFINITY;U sum_exp_score = 0;if (has_sinks && thread_context.threadgroup_index == 0) {- const int num_q_heads = static_cast<int>(tpg.x);+ const int num_q_heads = static_cast<int>(num_heads);int q_head_idx = head_idx % num_q_heads;max_score = static_cast<U>(sinks[q_head_idx]);sum_exp_score = 1;@@ -98,12 +105,20 @@ PUBLIC KERNEL(AttentionSinglePass)(// For each keyfor (uint i = thread_context.threadgroup_index; i < sequence_length;i += SEQUENCE_BLOCK_SIZE) {- bool use_key = true;- if (do_causal) {- use_key = i <= (sequence_length - tpg.y + q_seq_idx);- }-- if (use_key) {+ if (should_use_key(+ ring_params,+ trie,+ sliding_window_size,+ q_seq_idx,+ prefix_length,+ suffix_position,+ query_position,+ i,+ is_kv_cache_ring,+ is_causal,+ is_trie,+ is_sliding_window+ )) {// Read the keyfor (uint j = 0; j < qk_elements_per_thread; j++) {k[j] = keys[j];@@ -115,9 +130,6 @@ PUBLIC KERNEL(AttentionSinglePass)(score += q[j] * k[j];}score = simd_sum(score);- if (float_mask) {- score += max(-1e9f, static_cast<U>(fmask[0]));- }// Update the accumulatorsU new_max = max(max_score, score);@@ -136,9 +148,6 @@ PUBLIC KERNEL(AttentionSinglePass)(// Move the pointers to the next kvkeys += inner_k_stride;values += inner_v_stride;- if (float_mask) {- fmask += SEQUENCE_BLOCK_SIZE * mask_kv_seq_stride;- }}// Each thread has a partial part of the output so we need to combine them.diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metalindex 51d1b18dd..ab3cf52f8 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -2,12 +2,18 @@#include <metal_simdgroup>#include "../common/dsl.h"#include "../common/thread_context.h"+#include "../generated/ring.h"+#include "../generated/trie.h"+#include "mask.h"#define HEAD_BLOCK_SIZE 32#define SEQUENCE_BLOCK_SIZE_1 8#define SEQUENCE_BLOCK_SIZE_2 32#define TOTAL_BLOCKS_COUNT 32+using namespace uzu::ring;+using namespace uzu::trie;+template <typename T, uint HEAD_DIM>VARIANTS(T, float, half, bfloat)VARIANTS(HEAD_DIM, 64, 128, 256)@@ -24,18 +30,18 @@ PUBLIC KERNEL(AttentionTwoPass1)(const constant uint& k_seq_stride,const constant uint& v_head_stride,const constant uint& v_seq_stride,+ const constant RingParams& ring_params OPTIONAL(is_kv_cache_ring),const constant float& scale,const constant uint& num_heads,const constant uint& suffix_length,- const device T* fmask OPTIONAL(float_mask),- const constant uint& mask_kv_seq_stride OPTIONAL(has_mask),- const constant uint& mask_q_seq_stride OPTIONAL(has_mask),- const constant uint& mask_head_stride OPTIONAL(has_mask),+ const device TrieNode* trie OPTIONAL(is_trie),+ const constant uint& sliding_window_size OPTIONAL(is_sliding_window),const device float* sinks OPTIONAL(has_sinks),- const bool float_mask SPECIALIZE,- const bool has_mask SPECIALIZE,const bool has_sinks SPECIALIZE,- const bool do_causal SPECIALIZE,+ const bool is_kv_cache_ring SPECIALIZE,+ const bool is_causal SPECIALIZE,+ const bool is_trie SPECIALIZE,+ const bool is_sliding_window SPECIALIZE,threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE_1],threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE_1],threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_1 * HEAD_BLOCK_SIZE],@@ -45,8 +51,6 @@ PUBLIC KERNEL(AttentionTwoPass1)(const uint block_idx GROUPS(TOTAL_BLOCKS_COUNT),const uint tid THREADS(256)) {- const uint3 tpg = {num_heads, suffix_length, TOTAL_BLOCKS_COUNT};-constexpr bool query_transposed = false;constexpr uint value_dim = HEAD_DIM;constexpr uint qk_elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;@@ -60,13 +64,21 @@ PUBLIC KERNEL(AttentionTwoPass1)(thread U k[qk_elements_per_thread];thread U o[value_elements_per_thread];- const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout+ const uint o_offset = q_seq_idx * num_heads + head_idx; // Our custom layoutconst uint q_offset =query_transposed- ? tpg.x * q_seq_idx + head_idx- : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass+ ? num_heads * q_seq_idx + head_idx+ : head_idx * suffix_length + q_seq_idx; // Consistent with single-passconst uint kv_head_idx = head_idx / gqa_factor;+ const uint prefix_length = sequence_length - suffix_length;++ const uint suffix_position =+ is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;++ const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height+ : suffix_position + q_seq_idx;+queries += q_offset * HEAD_DIM +thread_context.simdgroup_index * qk_elements_per_thread;keys +=@@ -81,13 +93,6 @@ PUBLIC KERNEL(AttentionTwoPass1)(thread_context.simdgroup_index * value_elements_per_thread;out += o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim +thread_context.simdgroup_index * value_elements_per_thread;- if (float_mask) {- fmask +=- head_idx * mask_head_stride +- (block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index) *- mask_kv_seq_stride +- q_seq_idx * mask_q_seq_stride;- }sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;maxs += o_offset * TOTAL_BLOCKS_COUNT + block_idx;@@ -102,7 +107,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(U max_score = -1e9;U sum_exp_score = 0;if (has_sinks && block_idx == 0 && thread_context.threadgroup_index == 0) {- const uint num_q_heads = tpg.x;+ const uint num_q_heads = num_heads;int q_head_idx = head_idx % num_q_heads;max_score = static_cast<U>(sinks[q_head_idx]);sum_exp_score = 1;@@ -113,12 +118,20 @@ PUBLIC KERNEL(AttentionTwoPass1)(block_idx * SEQUENCE_BLOCK_SIZE_1 + thread_context.threadgroup_index;i < sequence_length;i += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1) {- bool use_key = true;- if (do_causal) {- use_key = i <= (sequence_length - tpg.y + q_seq_idx);- }-- if (use_key) {+ if (should_use_key(+ ring_params,+ trie,+ sliding_window_size,+ q_seq_idx,+ prefix_length,+ suffix_position,+ query_position,+ i,+ is_kv_cache_ring,+ is_causal,+ is_trie,+ is_sliding_window+ )) {// Read the keyfor (uint j = 0; j < qk_elements_per_thread; j++) {k[j] = keys[j];@@ -130,9 +143,6 @@ PUBLIC KERNEL(AttentionTwoPass1)(score += q[j] * k[j];}score = simd_sum(score);- if (float_mask) {- score += max(-1e9f, static_cast<U>(fmask[0]));- }// Update the accumulatorsU new_max = max(max_score, score);@@ -151,9 +161,6 @@ PUBLIC KERNEL(AttentionTwoPass1)(// Move the pointers to the next kvkeys += TOTAL_BLOCKS_COUNT * inner_k_stride;values += TOTAL_BLOCKS_COUNT * inner_v_stride;- if (float_mask) {- fmask += SEQUENCE_BLOCK_SIZE_1 * TOTAL_BLOCKS_COUNT * mask_kv_seq_stride;- }}// Each thread has a partial part of the output so we need to combine them.@@ -219,13 +226,12 @@ PUBLIC KERNEL(AttentionTwoPass2)(const uint q_seq_idx GROUPS(suffix_length),const uint tid THREADS(1024)) {- const uint3 tpg = {num_heads, suffix_length, 1};constexpr uint elements_per_thread = HEAD_DIM / HEAD_BLOCK_SIZE;typedef float U;thread U o[elements_per_thread];- const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout+ const uint o_offset = q_seq_idx * num_heads + head_idx; // Our custom layoutpartials += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +thread_context.threadgroup_index * HEAD_DIM +diff --git a/crates/uzu/src/backends/metal/kernel/attention/mask.h b/crates/uzu/src/backends/metal/kernel/attention/mask.hnew file mode 100644index 000000000..83b696dc1--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/attention/mask.h@@ -0,0 +1,22 @@+#pragma once++#include "../generated/ring.h"+#include "../generated/trie.h"++using namespace uzu::ring;+using namespace uzu::trie;++bool should_use_key(+ const constant RingParams& ring_params,+ const device TrieNode* trie,+ const constant uint& sliding_window_size,+ const uint q_seq_idx,+ const uint prefix_length,+ const uint suffix_position,+ const uint query_position,+ const uint i,+ const bool is_kv_cache_ring,+ const bool is_causal,+ const bool is_trie,+ const bool is_sliding_window+);diff --git a/crates/uzu/src/backends/metal/kernel/attention/mask.metal b/crates/uzu/src/backends/metal/kernel/attention/mask.metalnew file mode 100644index 000000000..b3260231c--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/attention/mask.metal@@ -0,0 +1,76 @@+#include "../generated/ring.h"+#include "../generated/trie.h"++using namespace uzu::ring;+using namespace uzu::trie;++bool should_use_key(+ const constant RingParams& ring_params,+ const device TrieNode* trie,+ const constant uint& sliding_window_size,+ const uint q_seq_idx,+ const uint prefix_length,+ const uint suffix_position,+ const uint query_position,+ const uint i,+ const bool is_kv_cache_ring,+ const bool is_causal,+ const bool is_trie,+ const bool is_sliding_window+) {+ bool use_key = true;++ uint key_position;++ if (i >= prefix_length) {+ // We're in the suffix+ uint key_position_in_suffix = i - prefix_length;++ if (is_trie) {+ // Trie, get position from the flat trie buffer+ TrieNode key_trie = trie[key_position_in_suffix];++ key_position = suffix_position + key_trie.height;++ if (is_causal) {+ use_key &=+ q_seq_idx >= key_trie.trie_start && q_seq_idx <= key_trie.trie_end;+ }+ } else {+ // Non-trie, position equals index in the suffix+ key_position = suffix_position + key_position_in_suffix;++ if (is_causal) {+ use_key &= key_position_in_suffix <= q_seq_idx;+ }+ }++ } else {+ // We're in the prefix+ if (is_kv_cache_ring) {+ // Ring, calculate position in the ring+ key_position =+ (prefix_length + i - ring_params.ring_offset) % prefix_length;+ // Ring also skips entries that aren't filled yet+ use_key &= key_position < ring_params.ring_length;+ } else {+ // Non-ring, position equals to the index in the kvcache buffer+ key_position = i;+ }+ }++ if (is_sliding_window) {+ if (is_causal) {+ use_key &= key_position <= query_position &&+ (query_position - key_position) < sliding_window_size;+ } else {+ if (key_position <= query_position) {+ use_key &= (query_position - key_position) <= (sliding_window_size / 2);+ } else {+ use_key &= (key_position - query_position) <= (sliding_window_size / 2);+ }+ }+ }++ return use_key;+}diff --git a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metaldeleted file mode 100644index 06a9d52f2..000000000--- a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal+++ /dev/null@@ -1,17 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(MaskUpdate) (- device T* mask,- constant int& unmask_col,- constant int& mask_col-) {- if (unmask_col >= 0) {- mask[unmask_col] = T(0);- }- if (mask_col >= 0) {- mask[mask_col] = -numeric_limits<T>::infinity();- }-}diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex f8c34c890..ddf9a6997 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -104,7 +104,6 @@ impl<B: Backend> Classifier<B> {self.context.shared_buffers.clone(),token_ids,token_positions,- true,num_labels,);@@ -157,7 +156,6 @@ impl<B: Backend> Classifier<B> {self.context.shared_buffers.clone(),token_ids,token_positions,- true,num_labels,);diff --git a/crates/uzu/src/classifier/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rsindex 5ef7421b3..2fe163e94 100644--- a/crates/uzu/src/classifier/classifier_context.rs+++ b/crates/uzu/src/classifier/classifier_context.rs@@ -161,8 +161,7 @@ impl<B: Backend> ClassifierContext<B> {.map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("output norm: {:?}", e))))?;let context_length = classifier_model_config.model_config.context_length;- let scratch_buffers =- ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, context_length, context_length);+ let scratch_buffers = ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, context_length);let embedding_norm_tree = root_loader_view.subtree("embedding_norm")diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rsindex 87f65dfb0..e14a249d6 100644--- a/crates/uzu/src/encodable_block/attention.rs+++ b/crates/uzu/src/encodable_block/attention.rs@@ -5,10 +5,13 @@ use std::{ops::{Deref, DerefMut},};+use itertools::iproduct;+use crate::{DataType,backends::common::{Backend, Encoder, Kernels,+ gpu_types::ring::RingParams,kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,SigmoidGateKernel,@@ -16,7 +19,10 @@ use crate::{},},encodable_block::EncodingParameters,- forward_pass::state::{ArrayId, ForwardPassState, HashMapId},+ forward_pass::{+ kv_cache_layer::KVCacheLayerState,+ state::{ArrayId, ForwardPassState},+ },};fn env_gemm_attention_enabled() -> bool {@@ -65,28 +71,39 @@ impl<B: Backend> Attention<B> {let mut two_pass_1_kernels = HashMap::new();let mut two_pass_2_kernels = HashMap::new();- let supported_head_dims = [64u32, 128u32, 256u32];- for head_dim in supported_head_dims {- for has_mask in [false, true] {- let key = KernelKey {- head_dim,- has_mask,- };- let float_mask = has_mask;-- let sp_kernel = <B::Kernels as Kernels>::AttentionSinglePassKernel::new(- context, data_type, head_dim, float_mask, has_mask, has_sinks, is_causal,- )?;- single_pass_kernels.insert(key, sp_kernel);-- let tp1_kernel = <B::Kernels as Kernels>::AttentionTwoPass1Kernel::new(- context, data_type, head_dim, float_mask, has_mask, has_sinks, is_causal,- )?;- two_pass_1_kernels.insert(key, tp1_kernel);-- let tp2_kernel = <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, data_type, head_dim)?;- two_pass_2_kernels.insert(head_dim, tp2_kernel);- }+ for (head_dim, is_trie, is_kv_cache_ring) in iproduct!([64u32, 128u32, 256u32], [false, true], [false, true]) {+ let key = KernelKey {+ head_dim,+ is_trie,+ is_kv_cache_ring,+ };++ let sp_kernel = <B::Kernels as Kernels>::AttentionSinglePassKernel::new(+ context,+ data_type,+ head_dim,+ has_sinks,+ is_kv_cache_ring,+ is_causal,+ is_trie,+ sliding_window_size.is_some(),+ )?;+ single_pass_kernels.insert(key, sp_kernel);++ let tp1_kernel = <B::Kernels as Kernels>::AttentionTwoPass1Kernel::new(+ context,+ data_type,+ head_dim,+ has_sinks,+ is_kv_cache_ring,+ is_causal,+ is_trie,+ sliding_window_size.is_some(),+ )?;+ two_pass_1_kernels.insert(key, tp1_kernel);++ let tp2_kernel = <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, data_type, head_dim)?;+ two_pass_2_kernels.insert(head_dim, tp2_kernel);}let update_kv_cache_kernel =@@ -122,7 +139,8 @@ impl<B: Backend> Attention<B> {suffix_length: usize,head_dim: usize,sequence_length: usize,- use_mask: bool,+ is_trie: bool,+ is_kv_cache_ring: bool,) -> KernelVariant {let use_gemm = gemm_enabled && suffix_length > 8 && matches!(head_dim, 64 | 128 | 256);if use_gemm {@@ -131,7 +149,8 @@ impl<B: Backend> Attention<B> {let kernel_key = KernelKey {head_dim: head_dim as u32,- has_mask: use_mask,+ is_trie,+ is_kv_cache_ring,};if sequence_length > 1024&& self.two_pass_1_kernels.contains_key(&kernel_key)@@ -168,18 +187,34 @@ impl<B: Backend> Attention<B> {suffix_length};- let (segment_prefix_length, window_length) = if let Some(cache_layers) = state.cache_layers() {+ let is_trie = state.token_subtrie_ranges.is_some();++ let (segment_prefix_length, ring_params) = if let Some(cache_layers) = state.cache_layers() {+ let projection_step = parameters.projection_step.unwrap_or(0);let cache = cache_layers.borrow();let layer = cache.data[self.layer_index].as_transformer().expect("Attention kernel expects transformer layer state");- (layer.projected_segment_prefix_length(parameters.projection_step.unwrap_or(0)), layer.window_length())+ let ring_params = match layer.state {+ KVCacheLayerState::Windowed {+ ring_offset,+ ring_length,+ window_length,+ } => {+ let overflow = (ring_length + projection_step).saturating_sub(window_length);+ Some(RingParams {+ ring_offset: ((ring_offset + overflow) % window_length) as u32,+ ring_length: (ring_length + projection_step).min(window_length) as u32,+ })+ },+ _ => None,+ };+ (layer.projected_segment_prefix_length(projection_step), ring_params)} else {- // For classifiers without KV cache: no prefix, use configured sliding window- (0, self.sliding_window_size)+ (0, None)};- let use_mask = window_length.is_some();+ let is_kv_cache_ring = ring_params.is_some();let sequence_length = segment_prefix_length + suffix_length;@@ -193,14 +228,11 @@ impl<B: Backend> Attention<B> {eprintln!("[uzu] Gemm attention disabled via UZU_USE_GEMM_ATTENTION");});}- let variant = self.select_variant(gemm_enabled, suffix_length, head_dim, sequence_length, use_mask);+ let variant =+ self.select_variant(gemm_enabled, suffix_length, head_dim, sequence_length, is_trie, is_kv_cache_ring);- let attention_bias_binding = state.hashmaps(&[HashMapId::AttentionBias]);let gate_array = self.gate_kernel.as_ref().map(|_| state.array(ArrayId::Gate));- let mask_kv_seq_stride = 1;- let mask_q_seq_stride = sequence_length as i32;- let mask_head_stride = 0;let sinks_array = self.has_sinks.then(|| state.array(ArrayId::AttentionSinks(self.layer_index)));let rotated_keys_buf_rc = rotated_keys_array.buffer();@@ -248,18 +280,9 @@ impl<B: Backend> Attention<B> {let attention_output_buf_rc = attention_output_array.buffer();let mut attention_output_buf_borrow = attention_output_buf_rc.borrow_mut();- let attention_bias_array_borrow = if use_mask {- Some(- attention_bias_binding[0]- .get(&window_length)- .unwrap_or_else(|| panic!("Attention bias buffer not found for window length {:?}", window_length)),- )- } else {- None- };- let attention_bias_buf_rc = attention_bias_array_borrow.as_ref().map(|a| a.buffer());- let attention_bias_buf_borrow = attention_bias_buf_rc.as_ref().map(|rc| rc.borrow());- let attention_bias_buffer: Option<&B::Buffer> = attention_bias_buf_borrow.as_ref().map(|b| b.deref());+ let trie_buf_rc = state.token_subtrie_ranges.as_ref().map(|a| a.buffer());+ let trie_buf_borrow = trie_buf_rc.as_ref().map(|rc| rc.borrow());+ let trie_buffer: Option<&B::Buffer> = trie_buf_borrow.as_ref().map(|b| b.deref());let partials_buf_rc = partials_array.buffer();let mut partials_buf_borrow = partials_buf_rc.borrow_mut();@@ -309,18 +332,10 @@ impl<B: Backend> Attention<B> {let kernel_key = KernelKey {head_dim: head_dim as u32,- has_mask: use_mask,+ is_trie,+ is_kv_cache_ring,};- let mut mask_kv_seq_stride_opt: Option<u32> = None;- let mut mask_q_seq_stride_opt: Option<u32> = None;- let mut mask_head_stride_opt: Option<u32> = None;- if attention_bias_buffer.is_some() {- mask_kv_seq_stride_opt = Some(mask_kv_seq_stride as u32);- mask_q_seq_stride_opt = Some(mask_q_seq_stride as u32);- mask_head_stride_opt = Some(mask_head_stride as u32);- }-match variant {KernelVariant::Gemm => {let args = AttentionGemmArguments {@@ -328,7 +343,7 @@ impl<B: Backend> Attention<B> {keys_buffer: key_cache_buffer,values_buffer: value_cache_buffer,output_buffer: attention_output_buf_borrow.deref_mut(),- mask_buffer: attention_bias_buffer,+ trie_buffer,sinks_buffer,num_heads,num_groups,@@ -336,8 +351,10 @@ impl<B: Backend> Attention<B> {sequence_length,segment_prefix_length,max_sequence_length,+ ring_params,head_dim,is_causal: self.is_causal,+ sliding_window_size: self.sliding_window_size,scale,};self.gemm_block.encode(state.context(), encoder, args).expect("Failed to encode AttentionGemmBlock");@@ -358,11 +375,10 @@ impl<B: Backend> Attention<B> {k_seq_stride as u32,v_head_stride as u32,v_seq_stride as u32,+ ring_params,scale,- attention_bias_buffer,- mask_kv_seq_stride_opt,- mask_q_seq_stride_opt,- mask_head_stride_opt,+ trie_buffer,+ self.sliding_window_size.map(|s| s as u32),sinks_buffer,num_heads as u32,suffix_length as u32,@@ -391,13 +407,12 @@ impl<B: Backend> Attention<B> {k_seq_stride as u32,v_head_stride as u32,v_seq_stride as u32,+ ring_params,scale,num_heads as u32,suffix_length as u32,- attention_bias_buffer,- mask_kv_seq_stride_opt,- mask_q_seq_stride_opt,- mask_head_stride_opt,+ trie_buffer,+ self.sliding_window_size.map(|s| s as u32),sinks_buffer,encoder,);@@ -439,7 +454,8 @@ enum KernelVariant {#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)]struct KernelKey {pub head_dim: u32,- pub has_mask: bool,+ pub is_trie: bool,+ pub is_kv_cache_ring: bool,}#[cfg(test)]diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rsindex 5b08a20ad..38abcabac 100644--- a/crates/uzu/src/forward_pass/cache_layers.rs+++ b/crates/uzu/src/forward_pass/cache_layers.rs@@ -1,11 +1,11 @@-use std::{cell::Cell, collections::HashMap};+use std::cell::Cell;use crate::{- array::{Array, ArrayContextExt},+ array::ArrayContextExt,backends::common::{Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},config::DecoderLayerType,forward_pass::{- kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState, KVSlice},+ kv_cache_layer::{KVCacheLayer, KVCacheLayerState, KVSlice},model_shape::ModelShape,short_conv_layer::ShortConvLayer,ssm_layer::SSMLayer,@@ -133,16 +133,6 @@ impl<B: Backend> CacheLayers<B> {model_shape.kv_cache_data_type(),&format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),),- prefix_token_positions: match &state {- KVCacheLayerState::Full {- ..- } => Vec::with_capacity(max_prefix_length),- KVCacheLayerState::Windowed {- window_length,- ..- } => (0..*window_length).map(|_| INVALID_POSITION).collect(),- },- max_suffix_length,})},DecoderLayerType::StateSpace {@@ -214,7 +204,6 @@ impl<B: Backend> CacheLayers<B> {prefix_len,} => {*prefix_len = 0;- layer.prefix_token_positions.clear();},KVCacheLayerState::Windowed {ring_offset,@@ -223,7 +212,6 @@ impl<B: Backend> CacheLayers<B> {} => {*ring_offset = 0;*ring_length = 0;- layer.prefix_token_positions.fill(INVALID_POSITION);},},CacheLayer::StateSpace(layer) => layer.zero(),@@ -240,38 +228,6 @@ impl<B: Backend> CacheLayers<B> {self.max_prefix_length}- pub fn fill_attention_bias(- &self,- dst: &mut HashMap<Option<usize>, Array<B>>,- suffix_token_positions: &[usize],- suffix_length: usize,- external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,- ) {- for layer in self.data.iter() {- if let CacheLayer::Transformer(layer) = layer {- if let Some(array) = dst.get_mut(&layer.window_length()) {- layer.fill_attention_bias(array, suffix_token_positions, suffix_length, external_bias_fn);- }- }- }- }-- pub fn fill_attention_bias_scratch(- &self,- dst: &mut HashMap<Option<usize>, Array<B>>,- suffix_token_positions: &[usize],- suffix_length: usize,- _context: &B::Context,- ) {- for layer in self.data.iter() {- if let CacheLayer::Transformer(layer) = layer {- if let Some(array) = dst.get_mut(&layer.window_length()) {- layer.fill_attention_bias(array, suffix_token_positions, suffix_length, None);- }- }- }- }-pub fn update_after_acceptance(&mut self,accepted_suffix_indices: &[usize],@@ -291,28 +247,15 @@ impl<B: Backend> CacheLayers<B> {pub fn register_accepted_tokens(&mut self,- token_positions: &[usize],+ number_of_accepted_tokens: usize,) {for layer in self.data.iter_mut() {if let Some(layer) = layer.as_transformer_mut() {- layer.register_accepted_tokens(token_positions);+ layer.register_accepted_tokens(number_of_accepted_tokens);}}}- pub fn attention_bias_updates_after_acceptance(- &self,- accepted_len: usize,- ) -> Vec<AttentionBiasUpdate> {- self.data- .iter()- .filter_map(|layer| match layer {- CacheLayer::Transformer(kv) => kv.attention_bias_update_after_acceptance(accepted_len),- _ => None,- })- .collect()- }-pub fn slice(&self,context: &B::Context,@@ -397,8 +340,6 @@ impl<B: Backend> CacheLayers<B> {state: layer.state.clone(),keys: new_keys,values: new_values,- prefix_token_positions: layer.prefix_token_positions.clone(),- max_suffix_length: layer.max_suffix_length,})},CacheLayer::StateSpace(layer) => {diff --git a/crates/uzu/src/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rsindex b31ca92c3..8901e996b 100644--- a/crates/uzu/src/forward_pass/kv_cache_layer.rs+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs@@ -4,31 +4,20 @@ use crate::{Backend, Encoder,kernel::kv_cache_update::{KVCacheUpdate, KVLayerData},},- utils::attention::fill_attention_bias,};-#[derive(Clone, Copy, Debug, PartialEq, Eq)]-pub struct AttentionBiasUpdate {- pub key: Option<usize>,- pub unmask_col: i32,- pub mask_col: i32,-}-#[derive(Clone)]pub enum KVSlice<B: Backend> {Full {base_prefix_len: usize,- base_positions_len: usize,- positions: Vec<usize>,},Window {window_length: usize,base_ring_offset: usize,base_ring_length: usize,slots: Vec<usize>,- positions: Vec<usize>, // per slot- keys: Array<B>, // [num_groups, slots.len(), head_dim]- values: Array<B>, // [num_groups, slots.len(), head_dim]+ keys: Array<B>, // [num_groups, slots.len(), head_dim]+ values: Array<B>, // [num_groups, slots.len(), head_dim]},}@@ -43,6 +32,7 @@ pub enum KVCacheLayerState {ring_offset: usize,/// Current logical length of the window (<= window_length)ring_length: usize,+ /// Maximum length of the windowwindow_length: usize,},}@@ -56,9 +46,6 @@ pub struct KVCacheLayer<B: Backend> {pub keys: Array<B>,/// [num_groups, max_prefix_length + max_suffix_length, head_dim]pub values: Array<B>,-- pub prefix_token_positions: Vec<usize>,- pub max_suffix_length: usize,}impl<B: Backend> KVCacheLayer<B> {@@ -101,64 +88,6 @@ impl<B: Backend> KVCacheLayer<B> {}}- pub fn fill_attention_bias(- &self,- dst: &mut Array<B>,- suffix_token_positions: &[usize],- suffix_length: usize,- external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,- ) {- let prefix_segment_length = self.prefix_segment_length();- fill_attention_bias(dst, suffix_length, prefix_segment_length, |row_index, column_index| {- if let Some(bias_fn) = external_bias_fn {- bias_fn(row_index, column_index)- } else {- self.bias_should_be_neg_inf(row_index, column_index, suffix_token_positions)- }- });- }-- pub fn bias_should_be_neg_inf(- &self,- row_index: usize,- column_index: usize,- suffix_token_positions: &[usize],- ) -> bool {- let query_position = suffix_token_positions[row_index];- if query_position == INVALID_POSITION {- return true;- }-- let key_position = if column_index >= self.prefix_segment_length() {- suffix_token_positions[column_index - self.prefix_segment_length()]- } else {- match &self.state {- KVCacheLayerState::Full {- ..- } => column_index,- KVCacheLayerState::Windowed {- ..- } => self.prefix_token_positions[column_index],- }- };-- if key_position == INVALID_POSITION {- return true;- }-- if query_position < key_position {- return true;- }-- match &self.state {- KVCacheLayerState::Windowed {- window_length,- ..- } => query_position >= key_position + window_length,- _ => false,- }- }-pub fn update_after_acceptance(&mut self,accepted_suffix_indices: &[usize],@@ -235,64 +164,22 @@ impl<B: Backend> KVCacheLayer<B> {pub fn register_accepted_tokens(&mut self,- token_positions: &[usize],+ number_of_accepted_tokens: usize,) {match &mut self.state {KVCacheLayerState::Full {prefix_len,} => {- self.prefix_token_positions.extend_from_slice(token_positions);- *prefix_len = self.prefix_token_positions.len();+ *prefix_len += number_of_accepted_tokens;},KVCacheLayerState::Windowed {ring_offset,ring_length,window_length,} => {- for &token_pos in token_positions {- if *ring_length < *window_length {- let dst = (*ring_offset + *ring_length) % *window_length;-- self.prefix_token_positions[dst] = token_pos;- *ring_length += 1;- } else {- self.prefix_token_positions[*ring_offset] = token_pos;-- *ring_offset = (*ring_offset + 1) % *window_length;- }- }- },- }- }-- pub fn attention_bias_update_after_acceptance(- &self,- accepted_len: usize,- ) -> Option<AttentionBiasUpdate> {- if accepted_len != 1 {- return None;- }-- match self.state {- KVCacheLayerState::Full {- ..- } => None,- KVCacheLayerState::Windowed {- ring_offset,- ring_length,- window_length,- } => {- let newest_slot = (ring_length > 0)- .then_some((ring_offset + ring_length + window_length - 1) % window_length)- .unwrap_or(0);- let unmask_col = (ring_length > 0).then_some(newest_slot as i32).unwrap_or(-1);- let mask_col = (ring_length == window_length).then_some(ring_offset as i32).unwrap_or(-1);-- Some(AttentionBiasUpdate {- key: Some(window_length),- unmask_col,- mask_col,- })+ let ring_advance = number_of_accepted_tokens.saturating_sub(*window_length - *ring_length);+ *ring_offset = (*ring_offset + ring_advance) % *window_length;+ *ring_length = (*ring_length + number_of_accepted_tokens).min(*window_length);},}}@@ -307,8 +194,6 @@ impl<B: Backend> KVCacheLayer<B> {prefix_len,} => Some(KVSlice::Full {base_prefix_len: prefix_len,- base_positions_len: self.prefix_token_positions.len(),- positions: self.prefix_token_positions.clone(),}),KVCacheLayerState::Windowed {ring_offset,@@ -342,8 +227,6 @@ impl<B: Backend> KVCacheLayer<B> {}).collect();- let positions: Vec<usize> = slots.iter().map(|&s| self.prefix_token_positions[s]).collect();-for (i, &slot) in slots.iter().enumerate() {slice_keys.copy_slice(&self.keys, 1, slot..slot + 1, i);slice_values.copy_slice(&self.values, 1, slot..slot + 1, i);@@ -354,7 +237,6 @@ impl<B: Backend> KVCacheLayer<B> {base_ring_offset: ring_offset,base_ring_length: ring_length,slots,- positions,keys: slice_keys,values: slice_values,})@@ -371,8 +253,6 @@ impl<B: Backend> KVCacheLayer<B> {(KVSlice::Full {base_prefix_len,- base_positions_len,- positions,},KVCacheLayerState::Full {prefix_len,@@ -380,14 +260,9 @@ impl<B: Backend> KVCacheLayer<B> {) => match range {None => {*prefix_len = *base_prefix_len;- self.prefix_token_positions.clone_from(positions);- self.prefix_token_positions.truncate(*base_positions_len);},Some(r) => {- let accepted = r.start;- *prefix_len = base_prefix_len.saturating_add(accepted);- let keep_positions = base_positions_len.saturating_add(accepted);- self.prefix_token_positions.truncate(keep_positions);+ *prefix_len = base_prefix_len.saturating_add(r.start);},},(@@ -396,7 +271,6 @@ impl<B: Backend> KVCacheLayer<B> {base_ring_offset,base_ring_length,slots,- positions,keys,values,},@@ -412,10 +286,6 @@ impl<B: Backend> KVCacheLayer<B> {*ring_offset = *base_ring_offset;*ring_length = *base_ring_length;- for (i, &slot) in slots.iter().enumerate() {- self.prefix_token_positions[slot] = positions[i];- }-for (i, &slot) in slots.iter().enumerate() {self.keys.copy_slice(keys, 1, i..i + 1, slot);self.values.copy_slice(values, 1, i..i + 1, slot);@@ -448,7 +318,6 @@ impl<B: Backend> KVCacheLayer<B> {for (i, &slot) in slots[r.clone()].iter().enumerate() {let src_i = r.start + i;- self.prefix_token_positions[slot] = positions[src_i];self.keys.copy_slice(keys, 1, src_i..src_i + 1, slot);self.values.copy_slice(values, 1, src_i..src_i + 1, slot);}diff --git a/crates/uzu/src/forward_pass/model_shape.rs b/crates/uzu/src/forward_pass/model_shape.rsindex da82e1408..789e1c252 100644--- a/crates/uzu/src/forward_pass/model_shape.rs+++ b/crates/uzu/src/forward_pass/model_shape.rs@@ -169,6 +169,13 @@ impl ModelShape {[suffix_length, self.model_dim]}+ pub fn subtrie_ranges_shape(+ &self,+ suffix_length: usize,+ ) -> [usize; 2] {+ [suffix_length, 3]+ }+pub fn bitmask_shape(&self,suffix_length: usize,diff --git a/crates/uzu/src/forward_pass/scratch_buffers.rs b/crates/uzu/src/forward_pass/scratch_buffers.rsindex 3f2cd6b25..c36e7a4a2 100644--- a/crates/uzu/src/forward_pass/scratch_buffers.rs+++ b/crates/uzu/src/forward_pass/scratch_buffers.rs@@ -1,5 +1,3 @@-use std::collections::{HashMap, HashSet};-use crate::{DataType,array::{Array, ArrayContextExt},@@ -11,6 +9,7 @@ use crate::{pub struct ScratchBuffers<B: Backend> {// 1-Dpub token_ids: Array<B>,+ pub token_subtrie_ranges: Array<B>,pub token_positions: Array<B>,pub token_parents: Array<B>,pub token_bitmask: Array<B>,@@ -18,7 +17,6 @@ pub struct ScratchBuffers<B: Backend> {pub sampling_output: Array<B>,// 2-D- pub attention_window_size_to_bias: HashMap<Option<usize>, Array<B>>,pub logits: Array<B>,pub main: Array<B>,pub shortcut: Array<B>,@@ -72,7 +70,6 @@ impl<B: Backend> ScratchBuffers<B> {context: &B::Context,decoder_config: &DecoderConfig,model_shape: &ModelShape,- max_prefix_len: usize,max_suffix_len: usize,) -> Self {// Helper closure for allocation@@ -97,6 +94,11 @@ impl<B: Backend> ScratchBuffers<B> {Self {// 1-Dtoken_ids: alloc(&[max_suffix_len], DataType::U64, "token_ids"),+ token_subtrie_ranges: alloc(+ &model_shape.subtrie_ranges_shape(max_suffix_len),+ DataType::U32,+ "token_subtrie_ranges",+ ),token_positions: alloc(&[max_suffix_len], DataType::I32, "token_positions"),token_parents: alloc(&[max_suffix_len], DataType::I32, "token_parents"),token_bitmask: alloc(&model_shape.bitmask_shape(max_suffix_len), DataType::U32, "token_bitmask"),@@ -104,23 +106,6 @@ impl<B: Backend> ScratchBuffers<B> {sampling_output: alloc(&[max_suffix_len], DataType::U32, "sampling_output"),// 2-D- attention_window_size_to_bias: model_shape- .sliding_window_length_per_layer- .iter()- .copied()- .collect::<HashSet<_>>()- .into_iter()- .map(|window_size| {- let label = match window_size {- Some(ws) => {- format!("attention_bias_for_window_size_{ws}")- },- None => "attention_bias_for_window_size_none".to_string(),- };- let shape = [max_suffix_len, max_suffix_len + max_prefix_len];- (window_size, alloc(&shape, act_ty, &label))- })- .collect(),logits: alloc(&model_shape.logits_shape(max_suffix_len), act_ty, "logits"),main: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "main"),shortcut: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "shortcut"),diff --git a/crates/uzu/src/forward_pass/state/hash_map_id.rs b/crates/uzu/src/forward_pass/state/hash_map_id.rsdeleted file mode 100644index 4d9231e7e..000000000--- a/crates/uzu/src/forward_pass/state/hash_map_id.rs+++ /dev/null@@ -1,4 +0,0 @@-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub enum HashMapId {- AttentionBias,-}diff --git a/crates/uzu/src/forward_pass/state/mod.rs b/crates/uzu/src/forward_pass/state/mod.rsindex d61b747a6..531093517 100644--- a/crates/uzu/src/forward_pass/state/mod.rs+++ b/crates/uzu/src/forward_pass/state/mod.rs@@ -1,5 +1,4 @@mod array_id;-mod hash_map_id;mod rope_type;mod common_aux_buffers;@@ -11,7 +10,6 @@ mod state;pub use array_id::ArrayId;pub use common_aux_buffers::CommonAuxBuffers;-pub use hash_map_id::HashMapId;pub use language_model_generator_aux_buffers::LanguageModelGeneratorAuxBuffers;pub use rope_buffers::RopeBuffers;pub use rope_type::RopeType;diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rsindex 1318e2570..e2cf57288 100644--- a/crates/uzu/src/forward_pass/state/state.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -1,6 +1,8 @@#[cfg(feature = "tracing")]use std::ops::{Deref, DerefMut};-use std::{cell::RefCell, collections::HashMap, rc::Rc};+use std::{cell::RefCell, rc::Rc};++use ndarray::ArrayView2;#[cfg(feature = "tracing")]use crate::backends::common::Encoder;@@ -15,10 +17,9 @@ use crate::{cache_layers::CacheLayers,model_shape::ModelShape,scratch_buffers::ScratchBuffers,- state::{ArrayId, CommonAuxBuffers, HashMapId, LanguageModelGeneratorAuxBuffers, RopeType, SharedBuffers},+ state::{ArrayId, CommonAuxBuffers, LanguageModelGeneratorAuxBuffers, RopeType, SharedBuffers},},session::parameter::SamplingMethod,- utils::attention::fill_attention_bias,};pub enum ForwardPassMode<B: Backend> {@@ -52,10 +53,10 @@ pub struct ClassifierModeState<B: Backend> {pub struct ForwardPassState<B: Backend> {context: Rc<B::Context>,token_ids: Array<B>,+ pub token_subtrie_ranges: Option<Array<B>>,token_positions: Array<B>,token_parents: Array<B>,token_bitmask: Option<Array<B>>,- attention_bias: HashMap<Option<usize>, Array<B>>,pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,common_aux: CommonAuxBuffers<B>,llm_aux: Option<LanguageModelGeneratorAuxBuffers<B>>,@@ -146,6 +147,7 @@ impl<B: Backend> ForwardPassState<B> {cache_layers: Rc<RefCell<CacheLayers<B>>>,shared_buffers: Rc<RefCell<SharedBuffers<B>>>,token_ids: &[u64],+ token_subtrie_ranges: Option<&[[u32; 3]]>,token_positions: &[usize],token_bitmask: Option<&[u32]>,token_seeds: &[u64],@@ -153,9 +155,7 @@ impl<B: Backend> ForwardPassState<B> {sampling_start: usize,sampling_length: usize,is_prefilling: bool,- external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,skip_token_ids_copy: bool,- should_fill_attention_bias: bool,async_positions: Option<(Rc<RefCell<B::Buffer>>, usize)>,async_seeds: Option<(Rc<RefCell<B::Buffer>>, usize)>,) -> Self {@@ -170,6 +170,18 @@ impl<B: Backend> ForwardPassState<B> {Self::init_token_ids(scratch, token_ids)};+ // Token subtrie ranges+ let token_subtrie_ranges_cell = token_subtrie_ranges.map(|subtrie_ranges| {+ let subtrie_ranges_shape = model_shape.subtrie_ranges_shape(suffix_length);+ let mut subtrie_ranges_array = scratch.token_subtrie_ranges.view(&subtrie_ranges_shape);+ subtrie_ranges_array.as_slice_mut::<u32>().fill(0);+ subtrie_ranges_array.copy_from_view(+ ArrayView2::from_shape((subtrie_ranges.len(), 3), bytemuck::cast_slice::<_, u32>(subtrie_ranges))+ .unwrap(),+ );+ subtrie_ranges_array+ });+// Token positions - use async buffer if providedlet token_positions_cell = if let Some((async_buf, offset)) = async_positions {unsafe {@@ -209,16 +221,6 @@ impl<B: Backend> ForwardPassState<B> {// Sampling outputlet sampling_output = Some(scratch.sampling_output.view(&[suffix_length]));- // Attention bias (causal + sliding window)- let attention_bias = Self::init_llm_attention_bias(- scratch,- &cache_layers,- suffix_length,- token_positions,- external_bias_fn,- should_fill_attention_bias,- );-// Common aux bufferslet common_aux = CommonAuxBuffers::new(scratch, model_shape, suffix_length);@@ -245,10 +247,10 @@ impl<B: Backend> ForwardPassState<B> {Self {context,token_ids: token_ids_cell,+ token_subtrie_ranges: token_subtrie_ranges_cell,token_positions: token_positions_cell,token_parents: token_parents_cell,token_bitmask: token_bitmask_cell,- attention_bias,shared_buffers,common_aux,llm_aux,@@ -256,42 +258,6 @@ impl<B: Backend> ForwardPassState<B> {}}- fn init_llm_attention_bias(- scratch: &ScratchBuffers<B>,- cache_layers: &Rc<RefCell<CacheLayers<B>>>,- suffix_length: usize,- token_positions: &[usize],- external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,- should_fill_attention_bias: bool,- ) -> HashMap<Option<usize>, Array<B>> {- let cache_ref = cache_layers.borrow();- let mut attention_bias_map: HashMap<Option<usize>, Array<B>> = scratch- .attention_window_size_to_bias- .iter()- .map(|(window_size, buffer)| {- let prefix_length = window_size.unwrap_or(cache_ref.max_prefix_length());- let attention_bias_shape = [suffix_length, suffix_length + prefix_length];- let array = buffer.view(&attention_bias_shape);- (*window_size, array)- })- .collect();- drop(cache_ref);-- // Use cache_layers' fill_attention_bias which properly handles- // both causal masking and sliding window constraints- // Skip fill for async decode passes after the first one (bias already set)- if should_fill_attention_bias {- cache_layers.borrow().fill_attention_bias(- &mut attention_bias_map,- token_positions,- suffix_length,- external_bias_fn,- );- }-- attention_bias_map- }-// ========================================================================// Classifier Constructor// ========================================================================@@ -304,7 +270,6 @@ impl<B: Backend> ForwardPassState<B> {shared_buffers: Rc<RefCell<SharedBuffers<B>>>,token_ids: &[u64],token_positions: &[usize],- bidirectional_attention: bool,num_labels: usize,) -> Self {let suffix_length = token_ids.len();@@ -314,9 +279,6 @@ impl<B: Backend> ForwardPassState<B> {let token_positions_cell = Self::init_token_positions(scratch, token_positions);let token_parents_cell = Self::init_token_parents(scratch, token_positions, 0, 0);- // Attention bias (bidirectional or causal)- let attention_bias = Self::init_classifier_attention_bias(scratch, suffix_length, bidirectional_attention);-// Common aux bufferslet common_aux = CommonAuxBuffers::new(scratch, model_shape, suffix_length);@@ -330,10 +292,10 @@ impl<B: Backend> ForwardPassState<B> {Self {context,token_ids: token_ids_cell,+ token_subtrie_ranges: None,token_positions: token_positions_cell,token_parents: token_parents_cell,token_bitmask: None,- attention_bias,shared_buffers,common_aux,llm_aux: None,@@ -341,40 +303,6 @@ impl<B: Backend> ForwardPassState<B> {}}- fn init_classifier_attention_bias(- scratch: &ScratchBuffers<B>,- suffix_length: usize,- bidirectional_attention: bool,- ) -> HashMap<Option<usize>, Array<B>> {- let mut attention_bias_map: HashMap<Option<usize>, Array<B>> = scratch- .attention_window_size_to_bias- .iter()- .map(|(window_size, buffer)| {- let attention_bias_shape = [suffix_length, suffix_length];- let array = buffer.view(&attention_bias_shape);- (*window_size, array)- })- .collect();-- for (window, bias_array) in attention_bias_map.iter_mut() {- if bidirectional_attention {- if let Some(window_size) = window {- let half_window = (window_size / 2) as isize;- fill_attention_bias(bias_array, suffix_length, 0, |row, col| {- let distance = (row as isize) - (col as isize);- distance.abs() > half_window- });- } else {- fill_attention_bias(bias_array, suffix_length, 0, |_row, _col| false);- }- } else {- fill_attention_bias(bias_array, suffix_length, 0, |row, col| row < col);- }- }-- attention_bias_map- }-#[cfg(feature = "tracing")]fn init_classifier_buffers(context: &B::Context,@@ -658,17 +586,6 @@ impl<B: Backend> ForwardPassState<B> {// Public API Methods (formerly trait methods)// ========================================================================- pub fn hashmaps(- &self,- ids: &[HashMapId],- ) -> Box<[HashMap<Option<usize>, Array<B>>]> {- ids.iter()- .map(|id| match id {- HashMapId::AttentionBias => self.attention_bias.clone(),- })- .collect()- }-pub fn active_suffix_length(&self) -> usize {match &self.mode {ForwardPassMode::LanguageModelGenerator(state) => state.active_suffix_length,diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 5ad622507..8b10b09ae 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -18,13 +18,13 @@ use super::{use crate::{backends::common::{Backend, Buffer, CommandBuffer, Context, Encoder, Executable,- kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},+ kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},},config::ModelMetadata,encodable_block::EncodingParameters,forward_pass::{cache_layers::{CacheLayer, CacheLayersSlice},- kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION},+ kv_cache_layer::INVALID_POSITION,state::ForwardPassState,},session::{@@ -39,6 +39,7 @@ use crate::{#[derive(Debug, Clone)]struct Task<'a> {token_ids: &'a [u64],+ token_subtrie_ranges: Option<&'a [[u32; 3]]>,token_positions: &'a [usize],token_bitmask: Option<&'a [u32]>,token_seeds: &'a [u64],@@ -152,7 +153,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let speculator = &self.decoding_config.speculator_config.speculator;let suffix_length = if sample_suffix {- self.decoding_config.generate_suffix_length().saturating_sub(1)+ self.decoding_config.generate_suffix_length().saturating_sub(1).min(prefill_size - tokens_length)} else {prefill_size - tokens_length};@@ -171,6 +172,10 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let token_ids =tokens.iter().copied().take(tokens_length - 1).chain(flat_trie.token_ids()).chunks(prefill_step_size);+ let token_subtrie_ranges = repeat_n(None, tokens_length - 1)+ .chain(flat_trie.token_subtrie_ranges().map(Some))+ .chunks(prefill_step_size);+let token_positions = (prefix_offset..prefix_offset + tokens_length - 1).chain(flat_trie.token_positions().map(|trie_position| prefix_offset + tokens_length - 1 + trie_position)).chunks(prefill_step_size);@@ -184,13 +189,38 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let mut run_times: Vec<f64> = Vec::new();// Process each prefill step and update the KV cache.- for (step, (step_token_ids, step_token_positions, step_token_bitmasks, step_token_seeds)) in- izip!(&token_ids, &token_positions, &token_bitmasks, &token_seeds).enumerate()+ for (+ step,+ (step_token_ids, step_token_subtrie_ranges, step_token_positions, step_token_bitmasks, step_token_seeds),+ ) in izip!(&token_ids, &token_subtrie_ranges, &token_positions, &token_bitmasks, &token_seeds).enumerate(){let tokens_start_index = step * prefill_step_size;let tokens_end_index = tokens_start_index + prefill_step_size;let step_token_ids = step_token_ids.collect::<Box<[u64]>>();+ let step_token_subtrie_ranges = step_token_subtrie_ranges.collect::<Box<[Option<[u32; 3]>]>>();+ let step_token_subtrie_ranges: Option<Box<[[u32; 3]]>> =+ if let Some(trie_start) = step_token_subtrie_ranges.iter().position(|e| e.is_some()) {+ Some(+ step_token_subtrie_ranges+ .iter()+ .enumerate()+ .map(|(i, me)| {+ if let Some([subtrie_start, subtrie_end, height]) = me {+ [+ trie_start as u32 + subtrie_start,+ trie_start as u32 + subtrie_end,+ trie_start as u32 + height,+ ]+ } else {+ [i as u32, step_token_subtrie_ranges.len() as u32 - 1, i as u32]+ }+ })+ .collect(),+ )+ } else {+ None+ };let step_token_positions = step_token_positions.collect::<Box<[usize]>>();let step_token_seeds = step_token_seeds.collect::<Box<[u64]>>();@@ -241,6 +271,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let task = Task {token_ids: &step_token_ids,+ token_subtrie_ranges: step_token_subtrie_ranges.as_deref(),token_positions: &step_token_positions,token_bitmask: step_token_bitmask.as_deref(),token_seeds: &step_token_seeds,@@ -251,8 +282,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {is_prefilling: !should_sample_after_step,};- let (state, run_time) =- self.run_model(task, self.allow_pre_encode(), sampling_method, self.should_fill_attention_bias())?;+ let (state, run_time) = self.run_model(task, self.allow_pre_encode(), sampling_method)?;if should_capture {self.gpu_capture.stop_capture(&self.context.context, "prefill").map_err(|_| Error::CaptureFailed)?;@@ -260,28 +290,18 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {// Register the accepted prompt tokens from this step.let step_end_token_index = std::cmp::min(tokens_end_index, tokens_length);- let tokens_processed_this_step = step_end_token_index - tokens_start_index;+ let mut tokens_processed_this_step = step_end_token_index - tokens_start_index;++ if step == prefill_steps - 1 && sample_suffix {+ tokens_processed_this_step = tokens_processed_this_step.saturating_sub(1);+ }if tokens_processed_this_step > 0 {- let mut positions_for_step: Vec<usize> =- (tokens_start_index..step_end_token_index).map(|idx| idx + prefix_offset).collect();- if step == prefill_steps - 1 && sample_suffix {- // Exclude the last token because it belongs to the suffix for sampling.- positions_for_step.pop();- }-- if !positions_for_step.is_empty() {- let accept_indices_for_step: Vec<usize> = (0..positions_for_step.len()).collect();- if !accept_indices_for_step.is_empty() {- self.update_cache_layers(&accept_indices_for_step, None, true)?;- }+ self.update_cache_layers(&(0..tokens_processed_this_step).collect::<Vec<usize>>(), None, true)?;- self.context.cache_layers.borrow_mut().register_accepted_tokens(&positions_for_step);+ self.context.cache_layers.borrow_mut().register_accepted_tokens(tokens_processed_this_step);- if let Some(&last_idx) = positions_for_step.last() {- self.registered_prefix_len = last_idx + 1;- }- }+ self.registered_prefix_len = prefix_offset + tokens_start_index + tokens_processed_this_step;}last_state = Some(state);@@ -342,6 +362,11 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let token_ids =flat_trie.token_ids().chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();+ let token_subtrie_ranges = flat_trie+ .token_subtrie_ranges()+ .chain(repeat_n([u32::MAX, u32::MAX, u32::MAX], suffix_length - active_suffix_length))+ .collect::<Box<[[u32; 3]]>>();+let token_bitmask: Option<Box<[u32]>> = compiled_grammar.is_some().then(|| {let single_token_bitmask_size = self.context.model_shape.bitmask_shape(1)[1];flat_trie@@ -372,6 +397,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let task = Task {token_ids: &token_ids,+ token_subtrie_ranges: Some(&token_subtrie_ranges),token_positions: &token_positions,token_bitmask: token_bitmask.as_deref(),token_seeds: &token_seeds,@@ -382,8 +408,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {is_prefilling: false,};- let (mut state, run_time) =- self.run_model(task, self.allow_pre_encode(), sampling_method, self.should_fill_attention_bias())?;+ let (mut state, run_time) = self.run_model(task, self.allow_pre_encode(), sampling_method)?;let sampled_tokens = self.read_sampling_output(&mut state)?;@@ -412,17 +437,6 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {tokens_to_generate: usize,) {let prefill_count = self.tokens.len();- let first_decode_position = prefill_count.saturating_sub(1);-- // Fill attention bias from KV cache state for the first decode position.- // Windowed buffers get the correct mask; GPU patches maintain it for subsequent passes.- // Full-attention buffers also get filled (masking beyond-prefix positions).- self.context.cache_layers.borrow().fill_attention_bias_scratch(- &mut self.context.scratch_buffers.attention_window_size_to_bias,- &[first_decode_position],- 1,- &self.context.context,- );self.context.async_buffers.prepare_positions(prefill_count, tokens_to_generate);self.context.async_buffers.prepare_seeds(&self.context.seed, prefill_count, tokens_to_generate);@@ -462,6 +476,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let task = Task {token_ids: &[last_token],+ token_subtrie_ranges: None,token_positions: &[token_position],token_bitmask: None,token_seeds: &[0], // Ignored, using async buffer@@ -475,7 +490,6 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let async_positions = Some((async_positions_buffer.clone(), pass_idx));let async_seeds = Some((async_seeds_buffer.clone(), pass_idx));- let should_fill_attention_bias = false; // we fill it once in preparelet skip_token_ids_copy = pass_idx > 0;let is_first_decode = !is_continuation;@@ -492,6 +506,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.context.cache_layers.clone(),self.context.shared_buffers.clone(),&task.token_ids,+ task.token_subtrie_ranges,&task.token_positions,task.token_bitmask,&task.token_seeds,@@ -499,9 +514,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {/*sampling_start=*/ 0,/*sampling_length=*/ task.active_suffix_length,task.is_prefilling,- None,skip_token_ids_copy,- should_fill_attention_bias,async_positions,async_seeds,);@@ -552,26 +565,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {&mut encoder,&self.context.kv_cache_update,);- self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);-- if let Some(mask_update) = &self.context.mask_update {- let updates: Vec<AttentionBiasUpdate> =- self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);- for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {- if let Some(update) = updates.iter().find(|u| &u.key == window_size) {- if update.unmask_col >= 0 || update.mask_col >= 0 {- let mask_buf_rc = mask_buffer.buffer();- let mut mask_buf_borrow = mask_buf_rc.borrow_mut();- mask_update.encode(- mask_buf_borrow.deref_mut(),- update.unmask_col,- update.mask_col,- &mut encoder,- );- }- }- }- }+ self.context.cache_layers.borrow_mut().register_accepted_tokens(1);// Signal event for next passlet next_counter = current_counter + 1;@@ -671,7 +665,6 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {dst.values.copy_slice(&src.values, 1, 0..copy_rows, 0);}dst.state = src.state.clone();- dst.prefix_token_positions = src.prefix_token_positions.clone();},(CacheLayer::StateSpace(src), CacheLayer::StateSpace(dst)) => {dst.conv_state.copy_from_array(&src.conv_state);@@ -711,7 +704,6 @@ impl<B: Backend> LanguageModelGenerator<B> {task: Task,allow_pre_encode: bool,sampling_method: SamplingMethod,- should_fill_attention_bias: bool,) -> Result<(ForwardPassState<B>, f64), Error> {let run_start = Instant::now();@@ -723,6 +715,7 @@ impl<B: Backend> LanguageModelGenerator<B> {self.context.cache_layers.clone(),self.context.shared_buffers.clone(),task.token_ids,+ task.token_subtrie_ranges,task.token_positions,task.token_bitmask,task.token_seeds,@@ -730,9 +723,7 @@ impl<B: Backend> LanguageModelGenerator<B> {task.sampling_start,task.sampling_length,task.is_prefilling,- None,false,- should_fill_attention_bias,None,None,);@@ -879,19 +870,9 @@ impl<B: Backend> LanguageModelGenerator<B> {let desired_prefix_len = self.tokens.len() - 1;if desired_prefix_len > self.registered_prefix_len {- let positions: Vec<usize> = (self.registered_prefix_len..desired_prefix_len).collect();- if !positions.is_empty() {- self.context.cache_layers.borrow_mut().register_accepted_tokens(&positions);- }+ let number_of_accepted_tokens = desired_prefix_len - self.registered_prefix_len;+ self.context.cache_layers.borrow_mut().register_accepted_tokens(number_of_accepted_tokens);self.registered_prefix_len = desired_prefix_len;}}-- fn should_fill_attention_bias(&self) -> bool {- let sliding_window_sizes = self.context.model_shape.sliding_window_length_per_layer.clone();- let has_sliding_window = sliding_window_sizes.iter().any(|size| size.is_some());- let has_speculative_suffix = self.decoding_config.generate_suffix_length() > 1;-- has_sliding_window || has_speculative_suffix- }}diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rsindex 1849f66d3..d3d67cc36 100644--- a/crates/uzu/src/language_model/language_model_generator_context.rs+++ b/crates/uzu/src/language_model/language_model_generator_context.rs@@ -9,7 +9,7 @@ use crate::{DataType,backends::common::{Backend, Buffer, Context, Kernels,- kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate},+ kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate},},config::{DecoderConfig, LanguageModelConfig, ModelMetadata},encodable_block::{Decoder, Sampling},@@ -128,8 +128,6 @@ pub struct LanguageModelGeneratorContext<B: Backend> {/// Kernels for copying sampled tokens in async pipelinepub token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,pub token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,- /// Kernel for updating attention mask between async passes- pub mask_update: Option<<B::Kernels as Kernels>::MaskUpdateKernel>,/// Pre-allocated buffers for async generationpub async_buffers: AsyncBuffers<B>,}@@ -164,8 +162,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));shared_buffers.borrow_mut().update_data(&root_loader_view);- let scratch_buffers =- ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_prefix_length, max_suffix_length);+ let scratch_buffers = ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_suffix_length);let executables = Decoder::new(context.as_ref(), &decoder_config, &root_loader_view);@@ -191,16 +188,6 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {let token_copy_results = <B::Kernels as Kernels>::TokenCopyToResultsKernel::new(&context).map_err(|e| Error::UnableToCreateContext(e.into()))?;- // Create mask update kernel if model has attention layers- let mask_update = if decoder_config.has_attention_layers() {- Some(- <B::Kernels as Kernels>::MaskUpdateKernel::new(&context, intermediate_data_type)- .map_err(|e| Error::UnableToCreateContext(e.into()))?,- )- } else {- None- };-let async_batch_size = decoding_config.async_batch_size.resolve::<B>(model_path, context.as_ref());let async_buffers = AsyncBuffers::new(context.as_ref(), max_prefix_length, async_batch_size);@@ -220,7 +207,6 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {seed,token_copy_sampled,token_copy_results,- mask_update,async_buffers,};diff --git a/crates/uzu/src/session/tts_session/decoder_runtime.rs b/crates/uzu/src/session/tts_session/decoder_runtime.rsindex ae287aa1c..9e0d49386 100644--- a/crates/uzu/src/session/tts_session/decoder_runtime.rs+++ b/crates/uzu/src/session/tts_session/decoder_runtime.rs@@ -19,7 +19,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {transformer_subtree: &str,embedding_subtree: &str,readout_subtree: &str,- max_prefix_length: usize,max_suffix_length: usize,) -> Result<Self, Error> {let weights_path = model_path.join("model.safetensors");@@ -34,8 +33,7 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {&root_loader_view,transformer_subtree,)?;- let scratch_buffers =- ScratchBuffers::new(context.as_ref(), decoder_config, model_shape, max_prefix_length, max_suffix_length);+ let scratch_buffers = ScratchBuffers::new(context.as_ref(), decoder_config, model_shape, max_suffix_length);let executables = Decoder::new_with_embedding_and_readout_subtrees(context.as_ref(),&decoder_config,@@ -95,12 +93,10 @@ impl<B: Backend> TokenDecoderContext<B> {embedding_subtree: &str,readout_subtree: &str,runtime_config: &TextDecoderRuntimeConfig,- ) -> Result<(Self, DataType, bool), Error> {+ ) -> Result<(Self, DataType), Error> {let model_shape = ModelShape::from_decoder_config(&decoder_config);let max_prefix_length = decoder_config.context_length;let max_suffix_length = text_decoder_prefill_step_size(runtime_config, decoder_config.context_length).max(32);- let should_fill_attention_bias =- model_shape.sliding_window_length_per_layer.iter().any(|value| value.is_some());let activation_data_type = model_shape.activation_data_type();let loaded_model = TokenDecoderLoadedModel::<B>::load(&context,@@ -110,7 +106,6 @@ impl<B: Backend> TokenDecoderContext<B> {transformer_subtree,embedding_subtree,readout_subtree,- max_prefix_length,max_suffix_length,)?;let async_chain_capacity = max_suffix_length.max(1);@@ -145,7 +140,6 @@ impl<B: Backend> TokenDecoderContext<B> {async_chain_capacity,},activation_data_type,- should_fill_attention_bias,))}@@ -200,7 +194,6 @@ pub(super) struct TokenDecoderRunner<B: Backend> {tensor_add_scale: <B::Kernels as Kernels>::TensorAddScaleKernel,single_token_vocab_masks: HashMap<usize, Box<[u32]>>,two_token_vocab_masks: HashMap<usize, Box<[u32]>>,- should_fill_attention_bias: bool,next_position: usize,instrumentation: RunnerInstrumentation,}@@ -219,7 +212,7 @@ impl<B: Backend> TokenDecoderRunner<B> {readout_subtree: &str,runtime_config: &TextDecoderRuntimeConfig,) -> Result<Self, Error> {- let (ctx, activation_data_type, should_fill_attention_bias) = TokenDecoderContext::new(+ let (ctx, activation_data_type) = TokenDecoderContext::new(context,model_path,decoder_config,@@ -248,7 +241,6 @@ impl<B: Backend> TokenDecoderRunner<B> {tensor_add_scale,single_token_vocab_masks: HashMap::new(),two_token_vocab_masks: HashMap::new(),- should_fill_attention_bias,next_position: 0,instrumentation: RunnerInstrumentation::default(),})@@ -281,6 +273,7 @@ impl<B: Backend> TokenDecoderRunner<B> {self.ctx.cache_layers.clone(),self.ctx.shared_buffers.clone(),token_ids,+ None,&positions,None, // no bitmask&token_seeds,@@ -288,9 +281,7 @@ impl<B: Backend> TokenDecoderRunner<B> {0, // sampling_start: irrelevant0, // sampling_length: no samplingfalse,- None,false,- self.should_fill_attention_bias,None,None,);@@ -507,6 +498,7 @@ impl<B: Backend> TokenDecoderRunner<B> {cache_layers_rc.clone(),shared_buffers_rc.clone(),&token_ids,+ None,&[self.next_position + pass],token_bitmask,&[0],@@ -514,9 +506,7 @@ impl<B: Backend> TokenDecoderRunner<B> {0,1,false,- None,pass > 0 || results_offset_slots > 0,- self.should_fill_attention_bias,Some((positions_rc.clone(), pass)),Some((seeds_rc.clone(), pass)),);@@ -568,7 +558,7 @@ impl<B: Backend> TokenDecoderRunner<B> {}self.ctx.cache_layers.borrow_mut().update_after_acceptance(&[0], None, encoder, &self.ctx.kv_cache_update);- self.ctx.cache_layers.borrow_mut().register_accepted_tokens(&[self.next_position + pass]);+ self.ctx.cache_layers.borrow_mut().register_accepted_tokens(1);}Ok(())}@@ -712,6 +702,7 @@ impl<B: Backend> TokenDecoderRunner<B> {self.ctx.cache_layers.clone(),self.ctx.shared_buffers.clone(),token_ids,+ None,positions,token_bitmask,token_seeds,@@ -719,9 +710,7 @@ impl<B: Backend> TokenDecoderRunner<B> {sampling_start,sampling_length,false,- None,false,- self.should_fill_attention_bias,None,None,);@@ -808,12 +797,11 @@ impl<B: Backend> TokenDecoderRunner<B> {positions_storage = [self.next_position, self.next_position + 1];&positions_storage[..]} else {- let v: Vec<usize> = (self.next_position..self.next_position + token_count).collect();- self.ctx.cache_layers.borrow_mut().register_accepted_tokens(&v);+ self.ctx.cache_layers.borrow_mut().register_accepted_tokens(token_count);self.next_position = self.next_position.saturating_add(token_count);return;};- self.ctx.cache_layers.borrow_mut().register_accepted_tokens(positions);+ self.ctx.cache_layers.borrow_mut().register_accepted_tokens(positions.len());self.next_position = self.next_position.saturating_add(token_count);}diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex 9dd4de189..ce7d4c719 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -256,6 +256,7 @@ impl<B: Backend> TraceValidator<B> {ctx.cache_layers.clone(),ctx.shared_buffers.clone(),&token_ids,+ None,&token_positions,None,&token_seeds,@@ -263,9 +264,7 @@ impl<B: Backend> TraceValidator<B> {/*sampling_start=*/ 0,/*sampling_length=*/ token_ids.len(),false,- None,false,- true,None,None,);@@ -813,13 +812,8 @@ impl<B: Backend> TraceValidator<B> {}let decoder_config = &context.decoder_config;- context.scratch_buffers = ScratchBuffers::new(- context.context.as_ref(),- decoder_config,- &context.model_shape,- resolved_prefix_length,- desired_suffix_length,- );+ context.scratch_buffers =+ ScratchBuffers::new(context.context.as_ref(), decoder_config, &context.model_shape, desired_suffix_length);context.cache_layers = Rc::new(RefCell::new(CacheLayers::new(context.context.as_ref(),diff --git a/crates/uzu/src/trie.rs b/crates/uzu/src/trie.rsindex a2d869615..7cdb7bed6 100644--- a/crates/uzu/src/trie.rs+++ b/crates/uzu/src/trie.rs@@ -31,9 +31,16 @@ pub struct TrieNode {next: Vec<TrieNode>,}+#[derive(Debug)]+struct FlatTrieNode<'a> {+ node: &'a TrieNode,+ subtrie_range: (usize, usize),+ height: usize,+}+#[derive(Debug)]pub struct FlatTrie<'a> {- tokens: Box<[(&'a TrieNode, usize)]>,+ tokens: Box<[FlatTrieNode<'a>]>,}impl TrieNode {@@ -170,20 +177,21 @@ impl TrieNode {}pub fn linearize(&self) -> FlatTrie<'_> {- let mut tokens = vec![(self, 0)];+ let mut tokens = vec![FlatTrieNode::new(self, (0, 0), 0)];- let mut stack = vec![(self, 0)];- while let Some((cur_node, next_child_idx)) = stack.last_mut() {- let Some(next_node) = cur_node.next.get(*next_child_idx) else {+ let mut stack = vec![(0, 0)];+ while let Some((cur_node_idx, next_child_idx)) = stack.last_mut() {+ let Some(next_node) = tokens[*cur_node_idx].node.next.get(*next_child_idx) else {+ tokens[*cur_node_idx].subtrie_range.1 = tokens.len() - 1;stack.pop();continue;};*next_child_idx += 1;- tokens.push((next_node, stack.len()));+ tokens.push(FlatTrieNode::new(next_node, (tokens.len(), tokens.len()), stack.len()));if !next_node.next.is_empty() {- stack.push((next_node, 0));+ stack.push((tokens.len() - 1, 0));}}@@ -191,8 +199,22 @@ impl TrieNode {}}+impl<'a> FlatTrieNode<'a> {+ fn new(+ node: &'a TrieNode,+ subtrie_range: (usize, usize),+ height: usize,+ ) -> Self {+ Self {+ node,+ subtrie_range,+ height,+ }+ }+}+impl<'a> FlatTrie<'a> {- pub fn new(tokens: Box<[(&'a TrieNode, usize)]>) -> Self {+ fn new(tokens: Box<[FlatTrieNode<'a>]>) -> Self {Self {tokens,}@@ -203,30 +225,38 @@ impl<'a> FlatTrie<'a> {}pub fn token_ids(&self) -> impl Iterator<Item = u64> {- self.tokens.iter().map(|&(n, _p)| n.token)+ self.tokens.iter().map(|n| n.node.token)+ }++ pub fn token_subtrie_ranges(&self) -> impl Iterator<Item = [u32; 3]> {+ self.tokens.iter().map(|n| {+ let (start, end) = n.subtrie_range;++ [start as u32, end as u32, n.height as u32]+ })}pub fn token_positions(&self) -> impl Iterator<Item = usize> {- self.tokens.iter().map(|&(_n, p)| p)+ self.tokens.iter().map(|n| n.height)}pub fn token_masks(&self) -> impl Iterator<Item = Option<&[u32]>> {- self.tokens.iter().map(|&(n, _p)| n.mask.as_deref())+ self.tokens.iter().map(|n| n.node.mask.as_deref())}pub fn token_seeds(&self) -> impl Iterator<Item = u64> {- self.tokens.iter().map(|&(n, _p)| n.seed)+ self.tokens.iter().map(|n| n.node.seed)}pub fn root(&self) -> Option<&TrieNode> {- self.tokens.first().map(|&(n, _p)| n)+ self.tokens.first().map(|n| n.node)}pub fn index(&self,node: &'a TrieNode,) -> Option<usize> {- self.tokens.iter().position(|&(n, _)| std::ptr::eq(n, node))+ self.tokens.iter().position(|n| std::ptr::eq(n.node, node))}pub fn accept(diff --git a/crates/uzu/src/utils/attention.rs b/crates/uzu/src/utils/attention.rsdeleted file mode 100644index a50c60a80..000000000--- a/crates/uzu/src/utils/attention.rs+++ /dev/null@@ -1,40 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;--use crate::{ArrayElement, DataType, array::Array, backends::common::Backend};--pub fn fill_attention_bias(- dst: &mut Array<impl Backend>,- suffix_length: usize,- prefix_segment_length: usize,- should_be_neg_inf: impl Fn(usize, usize) -> bool,-) {- let fill_typed_for_dtype = match dst.data_type() {- DataType::F16 => fill_typed::<f16>,- DataType::BF16 => fill_typed::<bf16>,- DataType::F32 => fill_typed::<f32>,- DataType::F64 => fill_typed::<f64>,- _ => panic!("Unsupported data type for attention bias fill"),- };-- fill_typed_for_dtype(dst, suffix_length, prefix_segment_length, should_be_neg_inf);-}--fn fill_typed<T: ArrayElement + Float>(- dst: &mut Array<impl Backend>,- suffix_length: usize,- prefix_segment_length: usize,- should_be_neg_inf: impl Fn(usize, usize) -> bool,-) {- let cols = suffix_length + prefix_segment_length;- let slice = dst.as_slice_mut::<T>();- for row in 0..suffix_length {- for col in 0..cols {- slice[row * cols + col] = if should_be_neg_inf(row, col) {- T::neg_infinity()- } else {- T::zero()- };- }- }-}diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rsindex 90c8993e4..530a66f61 100644--- a/crates/uzu/src/utils/mod.rs+++ b/crates/uzu/src/utils/mod.rs@@ -1,4 +1,3 @@-pub mod attention;pub mod env_utils;#[cfg(metal_backend)]pub mod model_size;diff --git a/crates/uzu/tests/unit/encodable_block/attention_test.rs b/crates/uzu/tests/unit/encodable_block/attention_test.rsindex d60d905be..82bf895b3 100644--- a/crates/uzu/tests/unit/encodable_block/attention_test.rs+++ b/crates/uzu/tests/unit/encodable_block/attention_test.rs@@ -4,7 +4,7 @@ use std::mem::size_of;use bytemuck;use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};-use ndarray::{Array3, Array4, s};+use ndarray::{Array4, s};use objc2::{rc::Retained, runtime::ProtocolObject};use test_tag::tag;@@ -23,12 +23,12 @@ use crate::{};fn reference_attention(- queries: &Array4<f32>, // [batch, num_heads, seq_len, head_dim]- keys: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]- values: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]- mask: Option<&Array3<f32>>, // [batch, seq_len, seq_len] or None+ queries: &Array4<f32>, // [batch, num_heads, seq_len, head_dim]+ keys: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]+ values: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]sinks: Option<&[f32]>,scale: f32,+ is_causal: bool,) -> Array4<f32> {let (batch_size, num_heads, seq_len, head_dim) = queries.dim();let (_, num_kv_heads, kv_seq_len, _) = keys.dim();@@ -51,9 +51,12 @@ fn reference_attention(// Efficient matrix multiplication: scores = Q @ K^Tlet mut scores = q.dot(&k.t());- if let Some(mask_data) = mask {- let mask_slice = mask_data.slice(s![b, .., ..kv_seq_len.min(seq_len)]);- scores.slice_mut(s![.., ..kv_seq_len.min(seq_len)]).zip_mut_with(&mask_slice, |s, &m| *s += m);+ if is_causal {+ for i in 0..seq_len {+ for j in (i + 1)..kv_seq_len {+ scores[[i, j]] = f32::NEG_INFINITY;+ }+ }}// Softmax with optional sink logit@@ -84,9 +87,12 @@ fn reference_attention(// Efficient matrix multiplication: scores = Q @ K^Tlet mut scores = q.dot(&k.t());- if let Some(mask_data) = mask {- let mask_slice = mask_data.slice(s![b, .., ..kv_seq_len.min(seq_len)]);- scores.slice_mut(s![.., ..kv_seq_len.min(seq_len)]).zip_mut_with(&mask_slice, |s, &m| *s += m);+ if is_causal {+ for i in 0..seq_len {+ for j in (i + 1)..kv_seq_len {+ scores[[i, j]] = f32::NEG_INFINITY;+ }+ }}// Softmax with optional sink logit@@ -120,7 +126,7 @@ fn create_test_data(seq_len: usize,head_dim: usize,seed: u64,-) -> (Array4<f32>, Array4<f32>, Array4<f32>, Array3<f32>) {+) -> (Array4<f32>, Array4<f32>, Array4<f32>) {use rand::{RngExt, SeedableRng, rngs::StdRng};let mut rng = StdRng::seed_from_u64(seed);@@ -131,15 +137,7 @@ fn create_test_data(let values = Array4::from_shape_fn((batch_size, num_kv_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));- let mask = Array3::from_shape_fn((batch_size, seq_len, seq_len), |(_, i, j)| {- if j <= i {- 0.0- } else {- -1e9- }- });-- (queries, keys, values, mask)+ (queries, keys, values)}/// Convert ndarray to Metal buffer layout expected by our kernel@@ -217,49 +215,6 @@ fn create_value_cache_buffer(.expect("Failed to create buffer")}-fn create_mask_buffer(- mask: &Array3<f32>,- num_heads: usize,- context: &<Metal as Backend>::Context,-) -> Retained<ProtocolObject<dyn MTLBuffer>> {- let (_batch_size, seq_len, _) = mask.dim();-- // Create mask buffer - the kernel expects mask layout: [num_heads, seq_len, seq_len]- let mut mask_data = vec![0.0f32; num_heads * seq_len * seq_len];- for h in 0..num_heads {- for i in 0..seq_len {- for j in 0..seq_len {- let mask_idx = h * seq_len * seq_len + i * seq_len + j;- mask_data[mask_idx] = mask[[0, i, j]];- }- }- }-- context- .device- .new_buffer_with_data(bytemuck::cast_slice(&mask_data), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer")-}--fn create_mask_2d_buffer(- mask: &Array3<f32>,- context: &<Metal as Backend>::Context,-) -> Retained<ProtocolObject<dyn MTLBuffer>> {- let (_batch_size, seq_len, _) = mask.dim();-- let mut mask_data = vec![0.0f32; seq_len * seq_len];- for i in 0..seq_len {- for j in 0..seq_len {- mask_data[i * seq_len + j] = mask[[0, i, j]];- }- }-- context- .device- .new_buffer_with_data(bytemuck::cast_slice(&mask_data), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer")-}-fn create_sinks_buffer(sinks: &[f32],context: &<Metal as Backend>::Context,@@ -297,7 +252,6 @@ fn run_single_pass_attention(queries: &Array4<f32>,keys: &Array4<f32>,values: &Array4<f32>,- mask: Option<&Array3<f32>>,sinks: Option<&[f32]>,scale: f32,) -> Result<Array4<f32>, Box<dyn std::error::Error>> {@@ -308,7 +262,6 @@ fn run_single_pass_attention(let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);- let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));let mut output_buffer = context@@ -318,15 +271,6 @@ fn run_single_pass_attention(let mut encoder = Encoder::new(context).expect("Failed to create encoder");- let mut mask_kv_seq_stride: Option<u32> = None;- let mut mask_q_seq_stride: Option<u32> = None;- let mut mask_head_stride: Option<u32> = None;- if mask_buffer.is_some() {- mask_kv_seq_stride = Some(1);- mask_q_seq_stride = Some(seq_len as u32);- mask_head_stride = Some(0);- }-kernel.encode(&query_buffer,&key_cache_buffer,@@ -338,11 +282,10 @@ fn run_single_pass_attention(head_dim as u32,(seq_len * head_dim) as u32,head_dim as u32,+ None,scale,- mask_buffer.as_ref().map(|b| b),- mask_kv_seq_stride,- mask_q_seq_stride,- mask_head_stride,+ None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,+ None,sinks_buffer.as_ref().map(|b| b),num_heads as u32,seq_len as u32,@@ -364,7 +307,6 @@ fn run_single_pass_attention_with_is_causal(queries: &Array4<f32>,keys: &Array4<f32>,values: &Array4<f32>,- mask: Option<&Array3<f32>>,sinks: Option<&[f32]>,scale: f32,_is_causal: bool,@@ -376,8 +318,6 @@ fn run_single_pass_attention_with_is_causal(let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);- let has_mask = mask.is_some();- let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));let mut output_buffer = context@@ -387,14 +327,6 @@ fn run_single_pass_attention_with_is_causal(let mut encoder = Encoder::new(context).expect("Failed to create encoder");- let mut mask_kv_seq_stride: Option<u32> = None;- let mut mask_q_seq_stride: Option<u32> = None;- let mut mask_head_stride: Option<u32> = None;- if has_mask {- mask_kv_seq_stride = Some(1);- mask_q_seq_stride = Some(seq_len as u32);- mask_head_stride = Some(0);- }kernel.encode(&query_buffer,&key_cache_buffer,@@ -406,11 +338,10 @@ fn run_single_pass_attention_with_is_causal(head_dim as u32,(seq_len * head_dim) as u32,head_dim as u32,+ None,scale,- mask_buffer.as_ref().map(|b| b),- mask_kv_seq_stride,- mask_q_seq_stride,- mask_head_stride,+ None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,+ None,sinks_buffer.as_ref().map(|b| b),num_heads as u32,seq_len as u32,@@ -432,7 +363,6 @@ fn run_gemm_attention(queries: &Array4<f32>,keys: &Array4<f32>,values: &Array4<f32>,- mask: Option<&Array3<f32>>,sinks: Option<&[f32]>,scale: f32,is_causal: bool,@@ -444,7 +374,6 @@ fn run_gemm_attention(let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);- let mask_buffer = mask.map(|m| create_mask_2d_buffer(m, context));let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));let mut output_buffer = context@@ -459,7 +388,7 @@ fn run_gemm_attention(keys_buffer: &key_cache_buffer,values_buffer: &value_cache_buffer,output_buffer: &mut output_buffer,- mask_buffer: mask_buffer.as_ref(),+ trie_buffer: None,sinks_buffer: sinks_buffer.as_ref(),num_heads,num_groups: num_kv_heads,@@ -467,7 +396,9 @@ fn run_gemm_attention(sequence_length: seq_len,segment_prefix_length: 0,max_sequence_length: seq_len,+ ring_params: None,head_dim,+ sliding_window_size: None,is_causal,scale,};@@ -542,14 +473,14 @@ fn test_single_pass_attention_basic() {let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);- println!("Testing reference implementation without mask...");- let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);+ println!("Testing reference implementation...");+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);let ref_max = reference_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));let ref_min = reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));- println!("Reference output range (no mask): [{}, {}]", ref_min, ref_max);+ println!("Reference output range: [{}, {}]", ref_min, ref_max);println!("Reference sample values: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());let is_causal = false; // Non-causal attention for this test@@ -559,13 +490,13 @@ fn test_single_pass_attention_basic() {head_dim as u32,false,false,- false,is_causal,+ false,+ false,).expect("Failed to create attention single pass metal");- let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale)- {+ let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale) {Ok(output) => output,Err(e) => {panic!("Failed to run single-pass attention: {:?}", e);@@ -589,14 +520,14 @@ fn test_gemm_attention_basic() {let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);let kernel = AttentionGemmBlock::new(DataType::F32);- let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);let kernel_output =- run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, None, scale, /*is_causal=*/ false)+ run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, scale, /*is_causal=*/ false).expect("run gemm attention");let tolerance = 1e-2;@@ -616,14 +547,14 @@ fn test_gemm_attention_f32_head_dim_128() {let head_dim = 128;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);let kernel = AttentionGemmBlock::new(DataType::F32);- let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);let kernel_output =- run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, None, scale, /*is_causal=*/ false)+ run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, scale, /*is_causal=*/ false).expect("run gemm attention f32 head_dim=128");let tolerance = 1e-2;@@ -644,40 +575,31 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {let scale = 1.0 / (head_dim as f32).sqrt();let is_causal = true;- let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);- // CPU reference (uses the same additive mask)- let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);+ // CPU reference+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, is_causal);// vector attention path (single-pass)let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(&context,DataType::F32,head_dim as u32,- true,- true,+ false,false,is_causal,+ false,+ false,).expect("Failed to create attention single pass metal");- let vector_output = run_single_pass_attention_with_is_causal(- &kernel,- &context,- &queries,- &keys,- &values,- Some(&mask),- None,- scale,- is_causal,- )- .expect("run vector attention");+ let vector_output =+ run_single_pass_attention_with_is_causal(&kernel, &context, &queries, &keys, &values, None, scale, is_causal)+ .expect("run vector attention");// matrix attention path (gemm)let gemm_kernel = AttentionGemmBlock::new(DataType::F32);- let matrix_output =- run_gemm_attention(&gemm_kernel, &context, &queries, &keys, &values, Some(&mask), None, scale, is_causal)- .expect("run matrix attention");+ let matrix_output = run_gemm_attention(&gemm_kernel, &context, &queries, &keys, &values, None, scale, is_causal)+ .expect("run matrix attention");// Compare both to CPUlet tol_cpu = 5e-2;@@ -696,53 +618,6 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {);}-#[test]-fn test_single_pass_attention_with_mask() {- let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");-- let batch_size = 1;- let num_heads = 4;- let num_kv_heads = 4;- let seq_len = 8;- let head_dim = 64;- let scale = 1.0 / (head_dim as f32).sqrt();-- let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);-- let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(- &context,- DataType::F32,- head_dim as u32,- true,- true,- false,- false,- )- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");- let kernel_output =- match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&mask), None, scale) {- Ok(output) => output,- Err(e) => {- panic!("Failed to run single-pass attention with mask: {:?}", e);- },- };-- let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);-- println!("Mask values:");- for i in 0..seq_len.min(4) {- for j in 0..seq_len.min(4) {- print!("{:8.1} ", mask[[0, i, j]]);- }- println!();- }-- let tolerance = 1e-2;- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention with mask") {- panic!("{}", e);- }-}-#[test]fn test_single_pass_attention_with_sinks() {let context = <Metal as Backend>::Context::new().expect("Failed to create <Metal as Backend>::Context");@@ -754,7 +629,7 @@ fn test_single_pass_attention_with_sinks() {let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 444);+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 444);let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();println!("Using sinks: {:?}", sinks);@@ -763,17 +638,18 @@ fn test_single_pass_attention_with_sinks() {&context,DataType::F32,head_dim as u32,+ true,+ false,false,false,- true,false,).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");- let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);+ let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);let kernel_output =- match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, Some(&sinks), scale) {+ match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale) {Ok(output) => output,Err(e) => {panic!("Failed to run single-pass attention with sinks: {:?}", e);@@ -797,7 +673,7 @@ fn test_single_pass_attention_with_sinks_long_sequence() {let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 777);+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 777);let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();println!("Long sequence sinks: {:?}", sinks);@@ -806,17 +682,18 @@ fn test_single_pass_attention_with_sinks_long_sequence() {&context,DataType::F32,head_dim as u32,+ true,+ false,false,false,- true,false,).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");- let reference_output = reference_attention(&queries, &keys, &values, None, Some(&sinks), scale);+ let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);let kernel_output =- match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, Some(&sinks), scale) {+ match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale) {Ok(output) => output,Err(e) => {panic!("Failed to run single-pass attention long sequence with sinks: {:?}", e);@@ -845,7 +722,7 @@ fn test_single_pass_attention_gqa() {let head_dim = 64;let scale = 1.0 / (head_dim as f32).sqrt();- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);let is_causal = false; // Non-causal attention for this testlet kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(@@ -854,20 +731,20 @@ fn test_single_pass_attention_gqa() {head_dim as u32,false,false,- false,is_causal,+ false,+ false,).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");- let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, None, scale)- {+ let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale) {Ok(output) => output,Err(e) => {panic!("Failed to run single-pass attention GQA: {:?}", e);},};- let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);let tolerance = 1e-2;if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention GQA") {@@ -882,7 +759,6 @@ fn run_two_pass_attention(queries: &Array4<f32>,keys: &Array4<f32>,values: &Array4<f32>,- mask: Option<&Array3<f32>>,sinks: Option<&[f32]>,scale: f32,) -> Result<Array4<f32>, Box<dyn std::error::Error>> {@@ -892,7 +768,6 @@ fn run_two_pass_attention(let queries_buffer = create_query_buffer(queries, context);let keys_buffer = create_key_cache_buffer(keys, seq_len, context);let values_buffer = create_value_cache_buffer(values, seq_len, context);- let mask_buffer = mask.map(|m| create_mask_buffer(m, num_heads, context));let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));let total_blocks_count = 32;@@ -924,14 +799,6 @@ fn run_two_pass_attention(let mut encoder = Encoder::new(context).expect("Failed to create encoder");- let mut mask_kv_seq_stride: Option<u32> = None;- let mut mask_q_seq_stride: Option<u32> = None;- let mut mask_head_stride: Option<u32> = None;- if mask_buffer.is_some() {- mask_kv_seq_stride = Some(1);- mask_q_seq_stride = Some(seq_len as u32);- mask_head_stride = Some(0);- }kernel_pass1.encode(&queries_buffer,&keys_buffer,@@ -945,13 +812,12 @@ fn run_two_pass_attention(head_dim as u32,(seq_len * head_dim) as u32,head_dim as u32,+ None,scale,num_heads as u32,seq_len as u32,- mask_buffer.as_ref().map(|b| b),- mask_kv_seq_stride,- mask_q_seq_stride,- mask_head_stride,+ None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,+ None,sinks_buffer.as_ref().map(|b| b),&mut encoder,);@@ -986,9 +852,9 @@ fn test_two_pass_attention() {let scale = 1.0 / (head_dim as f32).sqrt();let is_causal = false; // Non-causal attention for this test- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);- let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(&context,@@ -996,8 +862,9 @@ fn test_two_pass_attention() {head_dim as u32,false,false,- false,is_causal,+ false,+ false,).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(@@ -1006,22 +873,13 @@ fn test_two_pass_attention() {head_dim as u32,).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");- let kernel_output = match run_two_pass_attention(- &kernel_pass1,- &kernel_pass2,- &context,- &queries,- &keys,- &values,- None,- None,- scale,- ) {- Ok(output) => output,- Err(e) => {- panic!("Failed to run two-pass attention: {:?}", e);- },- };+ let kernel_output =+ match run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale) {+ Ok(output) => output,+ Err(e) => {+ panic!("Failed to run two-pass attention: {:?}", e);+ },+ };let tolerance = 1e-2;if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Two-pass attention") {@@ -1041,17 +899,18 @@ fn test_two_pass_attention_gqa() {let scale = 1.0 / (head_dim as f32).sqrt();let is_causal = false; // Non-causal attention for this test- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);- let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);+ let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(&context,DataType::F32,head_dim as u32,- is_causal,false,false,+ is_causal,+ false,false,).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");@@ -1061,22 +920,13 @@ fn test_two_pass_attention_gqa() {head_dim as u32,).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");- let kernel_output = match run_two_pass_attention(- &kernel_pass1,- &kernel_pass2,- &context,- &queries,- &keys,- &values,- None,- None,- scale,- ) {- Ok(output) => output,- Err(e) => {- panic!("Failed to run two-pass attention GQA: {:?}", e);- },- };+ let kernel_output =+ match run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale) {+ Ok(output) => output,+ Err(e) => {+ panic!("Failed to run two-pass attention GQA: {:?}", e);+ },+ };let tolerance = 1e-2;if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Two-pass attention GQA") {@@ -1106,15 +956,16 @@ fn perf_two_pass_attention() {seq_len - suffix_length,suffix_length);- let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);+ let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(&context,DataType::F32,head_dim as u32,- is_causal,false,false,+ is_causal,+ false,false,).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");@@ -1159,7 +1010,6 @@ fn perf_two_pass_attention() {// ---- Launch and time ----let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- let mask_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;let sinks_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;kernel_pass1.encode(&queries_buffer,@@ -1174,12 +1024,11 @@ fn perf_two_pass_attention() {head_dim as u32,(seq_len * head_dim) as u32,head_dim as u32,+ None,scale,num_heads as u32,suffix_length as u32,- mask_buffer.as_ref().map(|b| b),- None,- None,+ None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,None,sinks_buffer.as_ref().map(|b| b),&mut encoder,diff --git a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rsindex eb4f77a06..ce78d6f79 100644--- a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs+++ b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs@@ -6,7 +6,7 @@ use crate::{common::{Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},metal::Metal,},- forward_pass::kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},+ forward_pass::kv_cache_layer::{KVCacheLayer, KVCacheLayerState},};#[derive(Debug)]@@ -15,13 +15,9 @@ struct Scenario {state: KVCacheLayerState,prefix_capacity: usize,suffix_capacity: usize,- initial_prefix_positions: Vec<usize>,accepted_suffix_indices: Vec<usize>,- accepted_token_positions: Vec<usize>,- suffix_token_positions: Vec<usize>,+ number_of_accepted_tokens: usize,suffix_start: Option<usize>,- expected_prefix_positions: Vec<usize>,- expected_mask: Vec<Vec<bool>>,expected_ring_offset: Option<usize>,expected_ring_length: Option<usize>,expected_prefix_len: Option<usize>,@@ -48,22 +44,10 @@ fn make_test_layer(let keys = context.create_array_zeros(&shape, DataType::F32, "kv_cache_keys");let values = context.create_array_zeros(&shape, DataType::F32, "kv_cache_values");- let prefix_token_positions = match &state {- KVCacheLayerState::Full {- ..- } => Vec::with_capacity(prefix_capacity),- KVCacheLayerState::Windowed {- window_length,- ..- } => vec![INVALID_POSITION; *window_length],- };-KVCacheLayer {state,keys,values,- prefix_token_positions,- max_suffix_length: suffix_capacity,}}@@ -127,22 +111,6 @@ fn expected_after_update(expected}-fn collect_mask(- layer: &KVCacheLayer<Metal>,- suffix_positions: &[usize],-) -> Vec<Vec<bool>> {- let suffix_length = suffix_positions.len();- let prefix_segment_length = layer.prefix_segment_length();-- (0..suffix_length)- .map(|row| {- (0..prefix_segment_length + suffix_length)- .map(|col| layer.bias_should_be_neg_inf(row, col, suffix_positions))- .collect()- })- .collect()-}-fn run_scenario(context: &<Metal as Backend>::Context,scenario: &Scenario,@@ -150,8 +118,6 @@ fn run_scenario(let mut layer =make_test_layer(context, scenario.state.clone(), scenario.prefix_capacity, scenario.suffix_capacity);- layer.prefix_token_positions = scenario.initial_prefix_positions.clone();-let state_before_update = layer.state.clone();let (initial_keys, initial_values) = fill_arrays(&mut layer);@@ -170,9 +136,6 @@ fn run_scenario(} => window_length + scenario.suffix_capacity,};- let mask = collect_mask(&layer, &scenario.suffix_token_positions);- assert_eq!(mask, scenario.expected_mask, "{}: bias mask mismatch", scenario.name);-let kv_cache_update = match KVCacheUpdate::new(context, DataType::F32, total_sequence_length) {Ok(update) => update,Err(e) => {@@ -190,7 +153,7 @@ fn run_scenario(encoder.end_encoding().submit().wait_until_completed().unwrap();- layer.register_accepted_tokens(&scenario.accepted_token_positions);+ layer.register_accepted_tokens(scenario.number_of_accepted_tokens);let actual_keys = layer.keys.as_slice::<f32>().to_vec();assert_eq!(actual_keys, expected_keys, "{}: key buffer mismatch", scenario.name);@@ -198,12 +161,6 @@ fn run_scenario(let actual_values = layer.values.as_slice::<f32>().to_vec();assert_eq!(actual_values, expected_values, "{}: value buffer mismatch", scenario.name);- assert_eq!(- layer.prefix_token_positions, scenario.expected_prefix_positions,- "{}: prefix positions mismatch",- scenario.name- );-match &layer.state {KVCacheLayerState::Full {prefix_len,@@ -235,7 +192,7 @@ fn run_scenario(}#[test]-fn kv_cache_state_and_mask_scenarios() {+fn kv_cache_state_scenarios() {let Some(context) = <Metal as Backend>::Context::new().ok() else {return;};@@ -250,17 +207,9 @@ fn kv_cache_state_and_mask_scenarios() {},prefix_capacity: 6,suffix_capacity: 3,- initial_prefix_positions: vec![6, 1, 2, 3, 4, 5],accepted_suffix_indices: vec![0, 1, 2],- accepted_token_positions: vec![7, 8, 9],- suffix_token_positions: vec![7, 8, 9],+ number_of_accepted_tokens: 3,suffix_start: None,- expected_prefix_positions: vec![6, 7, 8, 9, 4, 5],- expected_mask: vec![- vec![false, true, false, false, false, false, false, true, true],- vec![false, true, true, false, false, false, false, false, true],- vec![false, true, true, true, false, false, false, false, false],- ],expected_ring_offset: Some(4),expected_ring_length: Some(6),expected_prefix_len: None,@@ -275,17 +224,9 @@ fn kv_cache_state_and_mask_scenarios() {},prefix_capacity: 6,suffix_capacity: 3,- initial_prefix_positions: vec![0, 1, 2, 3, INVALID_POSITION, INVALID_POSITION],accepted_suffix_indices: vec![0, 1, 2],- accepted_token_positions: vec![4, 5, 6],- suffix_token_positions: vec![4, 5, 6],+ number_of_accepted_tokens: 3,suffix_start: None,- expected_prefix_positions: vec![6, 1, 2, 3, 4, 5],- expected_mask: vec![- vec![false, false, false, false, true, true, false, true, true],- vec![false, false, false, false, true, true, false, false, true],- vec![true, false, false, false, true, true, false, false, false],- ],expected_ring_offset: Some(1),expected_ring_length: Some(6),expected_prefix_len: None,@@ -300,13 +241,9 @@ fn kv_cache_state_and_mask_scenarios() {},prefix_capacity: 6,suffix_capacity: 1,- initial_prefix_positions: vec![11, 12, 13, 14, 15, 10],accepted_suffix_indices: vec![0],- accepted_token_positions: vec![16],- suffix_token_positions: vec![16],+ number_of_accepted_tokens: 1,suffix_start: None,- expected_prefix_positions: vec![11, 12, 13, 14, 15, 16],- expected_mask: vec![vec![false, false, false, false, false, true, false]],expected_ring_offset: Some(0),expected_ring_length: Some(6),expected_prefix_len: None,@@ -319,17 +256,9 @@ fn kv_cache_state_and_mask_scenarios() {},prefix_capacity: 6,suffix_capacity: 3,- initial_prefix_positions: vec![0, 1, 2, 3],accepted_suffix_indices: vec![0, 1, 2],- accepted_token_positions: vec![4, 5, 6],- suffix_token_positions: vec![4, 5, 6],+ number_of_accepted_tokens: 3,suffix_start: None,- expected_prefix_positions: vec![0, 1, 2, 3, 4, 5, 6],- expected_mask: vec![- vec![false, false, false, false, false, true, true],- vec![false, false, false, false, false, false, true],- vec![false, false, false, false, false, false, false],- ],expected_ring_offset: None,expected_ring_length: None,expected_prefix_len: Some(7),@@ -358,14 +287,11 @@ fn kv_cache_slice_apply_contiguous_window() {4,0,);- layer.prefix_token_positions = vec![10, 11, 12, INVALID_POSITION];let (initial_keys, initial_values) = fill_arrays(&mut layer);let slice = layer.slice(&context, 0..2).expect("slice should exist");// Mutate the captured slots.{- layer.prefix_token_positions[0] = 999;- layer.prefix_token_positions[1] = 998;layer.keys.as_slice_mut::<f32>()[0] = -1.0;layer.keys.as_slice_mut::<f32>()[1] = -2.0;layer.values.as_slice_mut::<f32>()[0] = -3.0;@@ -374,12 +300,6 @@ fn kv_cache_slice_apply_contiguous_window() {layer.apply_slice(&slice, None);- assert_eq!(- layer.prefix_token_positions,- vec![10, 11, 12, INVALID_POSITION],- "positions restored for contiguous slice"- );-let keys_after = layer.keys.as_slice::<f32>().to_vec();let values_after = layer.values.as_slice::<f32>().to_vec();assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for contiguous slice");@@ -402,14 +322,11 @@ fn kv_cache_slice_apply_wrap_window() {4,0,);- layer.prefix_token_positions = vec![30, 31, 32, 33];let (initial_keys, initial_values) = fill_arrays(&mut layer);let slice = layer.slice(&context, 2..4).expect("slice should exist");// Captured slots are expected to wrap; mutate them.{- layer.prefix_token_positions[2] = 777;- layer.prefix_token_positions[3] = 778;layer.keys.as_slice_mut::<f32>()[2] = -11.0;layer.keys.as_slice_mut::<f32>()[3] = -12.0;layer.values.as_slice_mut::<f32>()[2] = -13.0;@@ -418,8 +335,6 @@ fn kv_cache_slice_apply_wrap_window() {layer.apply_slice(&slice, None);- assert_eq!(layer.prefix_token_positions, vec![30, 31, 32, 33], "positions restored for wrapped slice");-let keys_after = layer.keys.as_slice::<f32>().to_vec();let values_after = layer.values.as_slice::<f32>().to_vec();assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for wrapped slice");@@ -440,7 +355,6 @@ fn kv_cache_slice_apply_full_restores_metadata() {6,0,);- layer.prefix_token_positions = vec![1, 2, 3];let slice = layer.slice(&context, 0..2).expect("full slice exists");@@ -451,11 +365,9 @@ fn kv_cache_slice_apply_full_restores_metadata() {{*prefix_len = 1;}- layer.prefix_token_positions = vec![9, 9, 9, 9];layer.apply_slice(&slice, None);- assert_eq!(layer.prefix_token_positions, vec![1, 2, 3], "full slice restores positions");if let KVCacheLayerState::Full {prefix_len,} = &layer.statediff --git a/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rsindex e3216cd1b..946684835 100644--- a/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs+++ b/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs@@ -19,7 +19,6 @@ struct Input<T: ArrayElement + Float> {queries: Box<[T]>,keys: Box<[T]>,values: Box<[T]>,- mask: Option<Box<[T]>>,num_heads: usize,num_kv_heads: usize,sequence_length: usize,@@ -36,7 +35,6 @@ fn get_test_data<T: ArrayElement + Float>(suffix_length: usize,head_dim: usize,do_causal: bool,- with_mask: bool,) -> (Input<T>, Vec<T>) {// queries: [num_heads, suffix_length, head_dim] (contiguous, row-major)let q_size = num_heads * suffix_length * head_dim;@@ -59,31 +57,12 @@ fn get_test_data<T: ArrayElement + Float>(values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();}- // mask: [suffix_length, sequence_length] (broadcast over heads)- let mask = if with_mask {- let m_size = suffix_length * sequence_length;- let mut m = vec![T::zero(); m_size];- for q in 0..suffix_length {- for k in 0..sequence_length {- let idx = q * sequence_length + k;- let diff = (k as i32) - (sequence_length as i32 - suffix_length as i32 + q as i32);- if diff.abs() > 3 {- m[idx] = T::from(-1e9f32).unwrap();- }- }- }- Some(m.into_boxed_slice())- } else {- None- };-let scale = 1.0 / (head_dim as f32).sqrt();let input = Input {queries: queries.into_boxed_slice(),keys: keys.into_boxed_slice(),values: values.into_boxed_slice(),- mask,num_heads,num_kv_heads,sequence_length,@@ -109,8 +88,6 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let output_size = input.suffix_length * input.num_heads * input.head_dim;let output_array = context.create_array_uninitialized(&[output_size], T::data_type(), "");- let mask_array = input.mask.as_ref().map(|m| context.create_array_from(&[m.len()], m, ""));-let segment_prefix_length = input.sequence_length - input.suffix_length;{@@ -123,16 +100,12 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let o_buf_rc = output_array.buffer();let mut o_buf = o_buf_rc.borrow_mut();- let mask_buf_rc = mask_array.as_ref().map(|a| a.buffer());- let mask_buf_borrow = mask_buf_rc.as_ref().map(|rc| rc.borrow());- let mask_buffer: Option<&B::Buffer> = mask_buf_borrow.as_ref().map(|b| b.deref());-let args = AttentionGemmArguments::<B> {queries_buffer: q_buf.deref(),keys_buffer: k_buf.deref(),values_buffer: v_buf.deref(),output_buffer: o_buf.deref_mut(),- mask_buffer,+ trie_buffer: None,sinks_buffer: None,num_heads: input.num_heads,num_groups: input.num_kv_heads,@@ -140,7 +113,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {sequence_length: input.sequence_length,segment_prefix_length,max_sequence_length: input.sequence_length,+ ring_params: None,head_dim: input.head_dim,+ sliding_window_size: None,is_causal: input.do_causal,scale: input.scale,};@@ -166,7 +141,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(for_each_non_cpu_backend!(|B| {let output = get_output::<T, B>(input);let msg = format!(- "AttentionGemm failed (backend={}, heads={}, kv_heads={}, seq={}, suffix={}, head_dim={}, causal={}, mask={})",+ "AttentionGemm failed (backend={}, heads={}, kv_heads={}, seq={}, suffix={}, head_dim={}, causal={})",std::any::type_name::<B>(),input.num_heads,input.num_kv_heads,@@ -174,63 +149,53 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(input.suffix_length,input.head_dim,input.do_causal,- input.mask.is_some(),);assert_eq_float::<T>(expected, &output, eps, &msg);});}fn test_basic<T: ArrayElement + Float + Debug + Display>() {- // Non-causal, no mask, single token- let (input, expected) = get_test_data::<T>(4, 4, 8, 1, 64, false, false);+ // Non-causal, single token+ let (input, expected) = get_test_data::<T>(4, 4, 8, 1, 64, false);test_internal(&input, &expected);- // Non-causal, no mask, multiple tokens- let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, false, false);+ // Non-causal, multiple tokens+ let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, false);test_internal(&input, &expected);}fn test_causal<T: ArrayElement + Float + Debug + Display>() {// Causal, single token decode- let (input, expected) = get_test_data::<T>(4, 4, 16, 1, 64, true, false);+ let (input, expected) = get_test_data::<T>(4, 4, 16, 1, 64, true);test_internal(&input, &expected);// Causal, multi-token prefill- let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, true, false);+ let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, true);test_internal(&input, &expected);}fn test_gqa<T: ArrayElement + Float + Debug + Display>() {// GQA: 8 query heads, 2 kv heads- let (input, expected) = get_test_data::<T>(8, 2, 8, 1, 64, false, false);+ let (input, expected) = get_test_data::<T>(8, 2, 8, 1, 64, false);test_internal(&input, &expected);// GQA causal- let (input, expected) = get_test_data::<T>(8, 2, 8, 4, 64, true, false);- test_internal(&input, &expected);-}--fn test_mask<T: ArrayElement + Float + Debug + Display>() {- let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, false, true);- test_internal(&input, &expected);-- // Causal + mask- let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, true, true);+ let (input, expected) = get_test_data::<T>(8, 2, 8, 4, 64, true);test_internal(&input, &expected);}fn test_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: usize) {- let (input, expected) = get_test_data::<T>(4, 4, 8, 2, head_dim, true, false);+ let (input, expected) = get_test_data::<T>(4, 4, 8, 2, head_dim, true);test_internal(&input, &expected);}fn test_unaligned<T: ArrayElement + Float + Debug + Display>() {// suffix_length not aligned to BQ=32- let (input, expected) = get_test_data::<T>(4, 4, 40, 7, 64, true, false);+ let (input, expected) = get_test_data::<T>(4, 4, 40, 7, 64, true);test_internal(&input, &expected);// sequence_length not aligned to BK- let (input, expected) = get_test_data::<T>(4, 4, 13, 4, 64, false, false);+ let (input, expected) = get_test_data::<T>(4, 4, 13, 4, 64, false);test_internal(&input, &expected);}@@ -282,22 +247,6 @@ fn test_gqa_bf16() {test_gqa::<bf16>();}-// Mask tests-#[test]-fn test_mask_f32() {- test_mask::<f32>();-}--#[test]-fn test_mask_f16() {- test_mask::<f16>();-}--#[test]-fn test_mask_bf16() {- test_mask::<bf16>();-}-// Head dim 128#[test]fn test_head_dim_128_f32() {diff --git a/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rsindex 8fc02f929..ffc99f37b 100644--- a/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs+++ b/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs@@ -19,7 +19,6 @@ struct Input<T: ArrayElement + Float> {queries: Box<[T]>,keys: Box<[T]>,values: Box<[T]>,- mask: Option<Box<[T]>>,num_heads: u32,gqa_factor: u32,sequence_length: u32,@@ -27,7 +26,6 @@ struct Input<T: ArrayElement + Float> {head_dim: u32,scale: f32,do_causal: bool,- has_mask: bool,}fn get_input<T: ArrayElement + Float>(@@ -37,7 +35,6 @@ fn get_input<T: ArrayElement + Float>(suffix_length: u32,head_dim: u32,do_causal: bool,- with_mask: bool,) -> Input<T> {let gqa_factor = num_heads / num_kv_heads;@@ -62,32 +59,12 @@ fn get_input<T: ArrayElement + Float>(values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();}- // mask: [sequence_length * suffix_length] (broadcast over heads with stride 0)- let mask = if with_mask {- let m_size = (suffix_length * sequence_length) as usize;- let mut m = vec![T::zero(); m_size];- for q in 0..suffix_length {- for k in 0..sequence_length {- let idx = (q * sequence_length + k) as usize;- // Create a sliding window mask: allow only nearby keys- let diff = (k as i32) - (sequence_length as i32 - suffix_length as i32 + q as i32);- if diff.abs() > 3 {- m[idx] = T::from(-1e9f32).unwrap();- }- }- }- Some(m.into_boxed_slice())- } else {- None- };-let scale = 1.0 / (head_dim as f32).sqrt();Input {queries: queries.into_boxed_slice(),keys: keys.into_boxed_slice(),values: values.into_boxed_slice(),- mask,num_heads,gqa_factor,sequence_length,@@ -95,22 +72,21 @@ fn get_input<T: ArrayElement + Float>(head_dim,scale,do_causal,- has_mask: with_mask,}}fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");- let float_mask = input.has_mask;let kernel = <<B as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(&context,T::data_type(),input.head_dim,- float_mask,- input.has_mask,+ false,false,input.do_causal,+ false,+ false,).expect("Failed to create AttentionSinglePassKernel");@@ -121,15 +97,6 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let output_size = (input.suffix_length * input.num_heads * input.head_dim) as usize;let output_array = context.create_array_uninitialized(&[output_size], T::data_type(), "");- let mask_array = input.mask.as_ref().map(|m| context.create_array_from(&[m.len()], m, ""));- let mask_buf_rc = mask_array.as_ref().map(|a| a.buffer());- let mask_buf_borrow = mask_buf_rc.as_ref().map(|rc| rc.borrow());- let mask_buffer: Option<&B::Buffer> = mask_buf_borrow.as_ref().map(|b| b.deref());-- let mask_kv_seq_stride: Option<u32> = input.has_mask.then_some(1);- let mask_q_seq_stride: Option<u32> = input.has_mask.then_some(input.sequence_length);- let mask_head_stride: Option<u32> = input.has_mask.then_some(0);-let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(queries_array.buffer().borrow().deref(),@@ -142,11 +109,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {input.head_dim,input.sequence_length * input.head_dim,input.head_dim,+ None,input.scale,- mask_buffer,- mask_kv_seq_stride,- mask_q_seq_stride,- mask_head_stride,+ None::<&B::Buffer>,+ None,None::<&B::Buffer>,input.num_heads,input.suffix_length,@@ -170,68 +136,56 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(for_each_non_cpu_backend!(|B| {let output = get_output::<T, B>(input);let msg = format!(- "AttentionSinglePass failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={}, mask={})",+ "AttentionSinglePass failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={})",std::any::type_name::<B>(),input.num_heads,input.sequence_length,input.suffix_length,input.head_dim,input.do_causal,- input.has_mask,);assert_eq_float::<T>(expected, &output, eps, &msg);});}fn test_basic<T: ArrayElement + Float + Debug + Display>() {- // Non-causal, no mask, single token- let input = get_input::<T>(4, 4, 8, 1, 64, false, false);+ // Non-causal, single token+ let input = get_input::<T>(4, 4, 8, 1, 64, false);let expected = get_output::<T, Cpu>(&input);test_internal(&input, &expected);- // Non-causal, no mask, multiple tokens- let input = get_input::<T>(4, 4, 8, 4, 64, false, false);+ // Non-causal, multiple tokens+ let input = get_input::<T>(4, 4, 8, 4, 64, false);let expected = get_output::<T, Cpu>(&input);test_internal(&input, &expected);}fn test_causal<T: ArrayElement + Float + Debug + Display>() {// Causal, single token decode- let input = get_input::<T>(4, 4, 16, 1, 64, true, false);+ let input = get_input::<T>(4, 4, 16, 1, 64, true);let expected = get_output::<T, Cpu>(&input);test_internal(&input, &expected);// Causal, multi-token prefill- let input = get_input::<T>(4, 4, 8, 4, 64, true, false);+ let input = get_input::<T>(4, 4, 8, 4, 64, true);let expected = get_output::<T, Cpu>(&input);test_internal(&input, &expected);}fn test_gqa<T: ArrayElement + Float + Debug + Display>() {// GQA: 8 query heads, 2 kv heads- let input = get_input::<T>(8, 2, 8, 1, 64, false, false);+ let input = get_input::<T>(8, 2, 8, 1, 64, false);let expected = get_output::<T, Cpu>(&input);test_internal(&input, &expected);// GQA causal- let input = get_input::<T>(8, 2, 8, 4, 64, true, false);- let expected = get_output::<T, Cpu>(&input);- test_internal(&input, &expected);-}--fn test_mask<T: ArrayElement + Float + Debug + Display>() {- let input = get_input::<T>(4, 4, 8, 4, 64, false, true);- let expected = get_output::<T, Cpu>(&input);- test_internal(&input, &expected);-- // Causal + mask- let input = get_input::<T>(4, 4, 8, 4, 64, true, true);+ let input = get_input::<T>(8, 2, 8, 4, 64, true);let expected = get_output::<T, Cpu>(&input);test_internal(&input, &expected);}fn test_head_dim_128<T: ArrayElement + Float + Debug + Display>() {- let input = get_input::<T>(4, 4, 8, 2, 128, true, false);+ let input = get_input::<T>(4, 4, 8, 2, 128, true);let expected = get_output::<T, Cpu>(&input);test_internal(&input, &expected);}@@ -284,22 +238,6 @@ fn test_gqa_bf16() {test_gqa::<bf16>();}-// Mask tests-#[test]-fn test_mask_f32() {- test_mask::<f32>();-}--#[test]-fn test_mask_f16() {- test_mask::<f16>();-}--#[test]-fn test_mask_bf16() {- test_mask::<bf16>();-}-// Head dim 128#[test]fn test_head_dim_128_f32() {diff --git a/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rsindex f9b2936fa..b6cec168c 100644--- a/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs+++ b/crates/uzu/tests/unit/kernel/attention/attention_two_pass_test.rs@@ -26,7 +26,6 @@ struct FirstPassInput<T: ArrayElement + Float> {queries: Box<[T]>,keys: Box<[T]>,values: Box<[T]>,- mask: Option<Box<[T]>>,num_heads: u32,gqa_factor: u32,sequence_length: u32,@@ -34,7 +33,6 @@ struct FirstPassInput<T: ArrayElement + Float> {head_dim: u32,scale: f32,do_causal: bool,- has_mask: bool,}struct FirstPassOutput {@@ -50,7 +48,6 @@ fn get_first_pass_input<T: ArrayElement + Float>(suffix_length: u32,head_dim: u32,do_causal: bool,- with_mask: bool,) -> FirstPassInput<T> {let gqa_factor = num_heads / num_kv_heads;@@ -72,30 +69,12 @@ fn get_first_pass_input<T: ArrayElement + Float>(values[i] = T::from((i as f32 * 0.11 + 2.0).sin() * 0.5).unwrap();}- let mask = if with_mask {- let m_size = (suffix_length * sequence_length) as usize;- let mut m = vec![T::zero(); m_size];- for q in 0..suffix_length {- for k in 0..sequence_length {- let idx = (q * sequence_length + k) as usize;- let diff = (k as i32) - (sequence_length as i32 - suffix_length as i32 + q as i32);- if diff.abs() > 3 {- m[idx] = T::from(-1e9f32).unwrap();- }- }- }- Some(m.into_boxed_slice())- } else {- None- };-let scale = 1.0 / (head_dim as f32).sqrt();FirstPassInput {queries: queries.into_boxed_slice(),keys: keys.into_boxed_slice(),values: values.into_boxed_slice(),- mask,num_heads,gqa_factor,sequence_length,@@ -103,22 +82,21 @@ fn get_first_pass_input<T: ArrayElement + Float>(head_dim,scale,do_causal,- has_mask: with_mask,}}fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassInput<T>) -> FirstPassOutput {let context = B::Context::new().expect("Failed to create Context");- let float_mask = input.has_mask;let kernel = <<B as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(&context,T::data_type(),input.head_dim,- float_mask,- input.has_mask,+ false,false,input.do_causal,+ false,+ false,).expect("Failed to create AttentionTwoPass1Kernel");@@ -135,15 +113,6 @@ fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassIlet sums_array = context.create_array_uninitialized(&[sums_size], DataType::F32, "");let maxs_array = context.create_array_uninitialized(&[maxs_size], DataType::F32, "");- let mask_array = input.mask.as_ref().map(|m| context.create_array_from(&[m.len()], m, ""));- let mask_buf_rc = mask_array.as_ref().map(|a| a.buffer());- let mask_buf_borrow = mask_buf_rc.as_ref().map(|rc| rc.borrow());- let mask_buffer: Option<&B::Buffer> = mask_buf_borrow.as_ref().map(|b| b.deref());-- let mask_kv_seq_stride: Option<u32> = input.has_mask.then_some(1);- let mask_q_seq_stride: Option<u32> = input.has_mask.then_some(input.sequence_length);- let mask_head_stride: Option<u32> = input.has_mask.then_some(0);-let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(queries_array.buffer().borrow().deref(),@@ -158,13 +127,12 @@ fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassIinput.head_dim,input.sequence_length * input.head_dim,input.head_dim,+ None,input.scale,input.num_heads,input.suffix_length,- mask_buffer,- mask_kv_seq_stride,- mask_q_seq_stride,- mask_head_stride,+ None::<&B::Buffer>,+ None,None::<&B::Buffer>,&mut encoder,);@@ -268,14 +236,13 @@ fn test_first_pass_internal<T: ArrayElement + Float + Debug + Display>(for_each_non_cpu_backend!(|B| {let output = get_first_pass_output::<T, B>(input);let msg = format!(- "AttentionTwoPass1 failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={}, mask={})",+ "AttentionTwoPass1 failed (backend={}, heads={}, seq={}, suffix={}, head_dim={}, causal={})",std::any::type_name::<B>(),input.num_heads,input.sequence_length,input.suffix_length,input.head_dim,input.do_causal,- input.has_mask,);assert_eq_float::<f32>(&expected.partials, &output.partials, eps, &format!("{msg} [partials]"));assert_eq_float::<f32>(&expected.sums, &output.sums, eps, &format!("{msg} [sums]"));@@ -284,47 +251,37 @@ fn test_first_pass_internal<T: ArrayElement + Float + Debug + Display>(}fn test_first_pass_basic<T: ArrayElement + Float + Debug + Display>() {- let input = get_first_pass_input::<T>(4, 4, 8, 1, 64, false, false);+ let input = get_first_pass_input::<T>(4, 4, 8, 1, 64, false);let expected = get_first_pass_output::<T, Cpu>(&input);test_first_pass_internal(&input, &expected);- let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, false, false);+ let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, false);let expected = get_first_pass_output::<T, Cpu>(&input);test_first_pass_internal(&input, &expected);}fn test_first_pass_causal<T: ArrayElement + Float + Debug + Display>() {- let input = get_first_pass_input::<T>(4, 4, 16, 1, 64, true, false);+ let input = get_first_pass_input::<T>(4, 4, 16, 1, 64, true);let expected = get_first_pass_output::<T, Cpu>(&input);test_first_pass_internal(&input, &expected);- let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, true, false);+ let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, true);let expected = get_first_pass_output::<T, Cpu>(&input);test_first_pass_internal(&input, &expected);}fn test_first_pass_gqa<T: ArrayElement + Float + Debug + Display>() {- let input = get_first_pass_input::<T>(8, 2, 8, 1, 64, false, false);+ let input = get_first_pass_input::<T>(8, 2, 8, 1, 64, false);let expected = get_first_pass_output::<T, Cpu>(&input);test_first_pass_internal(&input, &expected);- let input = get_first_pass_input::<T>(8, 2, 8, 4, 64, true, false);- let expected = get_first_pass_output::<T, Cpu>(&input);- test_first_pass_internal(&input, &expected);-}--fn test_first_pass_mask<T: ArrayElement + Float + Debug + Display>() {- let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, false, true);- let expected = get_first_pass_output::<T, Cpu>(&input);- test_first_pass_internal(&input, &expected);-- let input = get_first_pass_input::<T>(4, 4, 8, 4, 64, true, true);+ let input = get_first_pass_input::<T>(8, 2, 8, 4, 64, true);let expected = get_first_pass_output::<T, Cpu>(&input);test_first_pass_internal(&input, &expected);}fn test_first_pass_head_dim_128<T: ArrayElement + Float + Debug + Display>() {- let input = get_first_pass_input::<T>(4, 4, 8, 2, 128, true, false);+ let input = get_first_pass_input::<T>(4, 4, 8, 2, 128, true);let expected = get_first_pass_output::<T, Cpu>(&input);test_first_pass_internal(&input, &expected);}@@ -417,21 +374,6 @@ fn test_first_pass_gqa_bf16() {test_first_pass_gqa::<bf16>();}-#[test]-fn test_first_pass_mask_f32() {- test_first_pass_mask::<f32>();-}--#[test]-fn test_first_pass_mask_f16() {- test_first_pass_mask::<f16>();-}--#[test]-fn test_first_pass_mask_bf16() {- test_first_pass_mask::<bf16>();-}-#[test]fn test_first_pass_head_dim_128_f32() {test_first_pass_head_dim_128::<f32>();diff --git a/crates/uzu/tests/unit/kernel/mask_update_test.rs b/crates/uzu/tests/unit/kernel/mask_update_test.rsdeleted file mode 100644index f4113d5ab..000000000--- a/crates/uzu/tests/unit/kernel/mask_update_test.rs+++ /dev/null@@ -1,240 +0,0 @@-use std::{- fmt::{Debug, Display},- ops::DerefMut,-};--use half::{bf16, f16};-use num_traits::Float;-use uzu::{- ArrayContextExt, ArrayElement,- backends::{- common::{Backend, Context, Encoder, Kernels, kernel::MaskUpdateKernel},- cpu::Cpu,- },-};--use crate::common::assert::assert_eq_float;--struct Input<T: ArrayElement + Float> {- mask: Box<[T]>,- unmask_col: i32,- mask_col: i32,-}--fn make_mask<T: ArrayElement + Float>(size: usize) -> Vec<T> {- let mut mask = Vec::with_capacity(size);- for i in 0..size {- mask.push(T::from(i as f32 * 0.5).unwrap());- }- mask-}--fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {- let context = B::Context::new().expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::MaskUpdateKernel::new(&context, T::data_type())- .expect("Failed to create MaskUpdateKernel");-- let len = input.mask.len();- let mask_array = context.create_array_from(&[len], &input.mask, "");-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");- kernel.encode(mask_array.buffer().borrow_mut().deref_mut(), input.unmask_col, input.mask_col, &mut encoder);- encoder.end_encoding().submit().wait_until_completed().unwrap();-- mask_array.as_slice().to_vec()-}--/// Unmask one column and mask another.-fn get_test_data_unmask_and_mask<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {- let size = 8;- let mask = make_mask::<T>(size);-- let input = Input {- mask: mask.into_boxed_slice(),- unmask_col: 2,- mask_col: 5,- };-- let expected = get_output::<T, Cpu>(&input);- (input, expected)-}--/// Only unmask (mask_col negative).-fn get_test_data_unmask_only<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {- let size = 8;- let mask = make_mask::<T>(size);-- let input = Input {- mask: mask.into_boxed_slice(),- unmask_col: 3,- mask_col: -1,- };-- let expected = get_output::<T, Cpu>(&input);- (input, expected)-}--/// Only mask (unmask_col negative).-fn get_test_data_mask_only<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {- let size = 8;- let mask = make_mask::<T>(size);-- let input = Input {- mask: mask.into_boxed_slice(),- unmask_col: -1,- mask_col: 4,- };-- let expected = get_output::<T, Cpu>(&input);- (input, expected)-}--/// Both columns negative — data should remain unchanged.-fn get_test_data_no_op<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {- let size = 8;- let mask = make_mask::<T>(size);-- let expected = mask.clone();-- let input = Input {- mask: mask.into_boxed_slice(),- unmask_col: -1,- mask_col: -1,- };-- (input, expected)-}--/// Larger mask to exercise more threads.-fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {- let size = 1024;- let mask = make_mask::<T>(size);-- let input = Input {- mask: mask.into_boxed_slice(),- unmask_col: 100,- mask_col: 900,- };-- let expected = get_output::<T, Cpu>(&input);- (input, expected)-}--fn test_internal<T: ArrayElement + Float + Debug + Display>(- input: &Input<T>,- expected: &[T],- test_name: &str,-) {- let eps = 1e-5;- let expected_norm = expected.to_vec();- for_each_non_cpu_backend!(|B| {- let actual = get_output::<T, B>(input);- let msg = format!("MaskUpdate {} failed with backend={}", test_name, std::any::type_name::<B>());- assert_eq_float::<T>(&expected_norm, &actual, eps, &msg);- });-}--fn test_unmask_and_mask_internal<T: ArrayElement + Float + Debug + Display>() {- let (input, expected) = get_test_data_unmask_and_mask::<T>();- test_internal::<T>(&input, &expected, "unmask_and_mask");-}--fn test_unmask_only_internal<T: ArrayElement + Float + Debug + Display>() {- let (input, expected) = get_test_data_unmask_only::<T>();- test_internal::<T>(&input, &expected, "unmask_only");-}--fn test_mask_only_internal<T: ArrayElement + Float + Debug + Display>() {- let (input, expected) = get_test_data_mask_only::<T>();- test_internal::<T>(&input, &expected, "mask_only");-}--fn test_no_op_internal<T: ArrayElement + Float + Debug + Display>() {- let (input, expected) = get_test_data_no_op::<T>();- test_internal::<T>(&input, &expected, "no_op");-}--fn test_large_internal<T: ArrayElement + Float + Debug + Display>() {- let (input, expected) = get_test_data_large::<T>();- test_internal::<T>(&input, &expected, "large");-}--// Unmask and mask tests-#[test]-fn test_unmask_and_mask_f32() {- test_unmask_and_mask_internal::<f32>();-}--#[test]-fn test_unmask_and_mask_f16() {- test_unmask_and_mask_internal::<f16>();-}--#[test]-fn test_unmask_and_mask_bf16() {- test_unmask_and_mask_internal::<bf16>();-}--// Unmask only tests-#[test]-fn test_unmask_only_f32() {- test_unmask_only_internal::<f32>();-}--#[test]-fn test_unmask_only_f16() {- test_unmask_only_internal::<f16>();-}--#[test]-fn test_unmask_only_bf16() {- test_unmask_only_internal::<bf16>();-}--// Mask only tests-#[test]-fn test_mask_only_f32() {- test_mask_only_internal::<f32>();-}--#[test]-fn test_mask_only_f16() {- test_mask_only_internal::<f16>();-}--#[test]-fn test_mask_only_bf16() {- test_mask_only_internal::<bf16>();-}--// No-op tests-#[test]-fn test_no_op_f32() {- test_no_op_internal::<f32>();-}--#[test]-fn test_no_op_f16() {- test_no_op_internal::<f16>();-}--#[test]-fn test_no_op_bf16() {- test_no_op_internal::<bf16>();-}--// Large dimension tests-#[test]-fn test_large_f32() {- test_large_internal::<f32>();-}--#[test]-fn test_large_f16() {- test_large_internal::<f16>();-}--#[test]-fn test_large_bf16() {- test_large_internal::<bf16>();-}diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rsindex 34e21cfb0..178642bf1 100644--- a/crates/uzu/tests/unit/kernel/mod.rs+++ b/crates/uzu/tests/unit/kernel/mod.rs@@ -9,7 +9,6 @@ mod audio;mod embedding;mod kv_cache_update_test;mod layer_norm_test;-mod mask_update_test;mod matmul;mod moe;mod pooling;